9 安装flannel

k8s设计了网络模型，但却将他的实现交给了网络插件，CNI网络插件最主要的功能
就是实现POD资源能够跨主机进行通信

最常见的CNI网络插件：
Flannel、Calico、Cancl、OpenContrail、NSX-T、Kube-router

flannel下载地址
https://github.com/coreos/flannel/releases
wget https://github.com/coreos/flannel/releases/download/v0.12.0/flannel-v0.12.0-linux-amd64.tar.gz

---

长传安装包

[root@rstx-203 ~]# cd /opt/src/
[root@rstx-203 src]# ls
flannel-v0.12.0-linux-amd64.tar.gz

[root@rstx-203 src]# mkdir /opt/flannel-v0.12.0

[root@rstx-203 src]# tar xf flannel-v0.12.0-linux-amd64.tar.gz -C /opt/flannel-v0.12.0/

创建软链接

[root@rstx-203 src]# ln -s /opt/flannel-v0.12.0/ /opt/flannel

[root@rstx-203 src]# cd ..
[root@rstx-203 opt]# ll
总用量 0
drwx--x--x 4 root root  28 12月 17 10:52 containerd
lrwxrwxrwx 1 root root  21 12月 18 09:48 flannel -> /opt/flannel-v0.12.0/
drwxr-xr-x 2 root root  64 12月 18 09:47 flannel-v0.12.0
lrwxrwxrwx 1 root root  24 12月 17 10:35 kubernetes -> /opt/kubernetes-v1.15.2/
drwxr-xr-x 4 root root  50 12月 17 10:35 kubernetes-v1.15.2
drwxr-xr-x 2 root root 100 12月 18 09:46 src

---

拷贝证书

[root@rstx-203 opt]# cd flannel
[root@rstx-203 flannel]# mkdir certs
[root@rstx-203 flannel]# cd certs/

[root@rstx-203 certs]# scp rstx-203.host.com:/opt/certs/ca.pem .
[root@rstx-203 certs]# scp rstx-203.host.com:/opt/certs/client.pem . 
[root@rstx-203 certs]# scp rstx-203.host.com:/opt/certs/client-key.pem .

[root@rstx-203 certs]# cd ..

---

[root@rstx-203 flannel]# vi subnet.env

FLANNEL_NETWORK=172.7.0.0/16    #pod的网段
FLANNEL_SUBNET=172.7.203.1/24   #本机运行pod的网段
FLANNEL_MTU=1500
FLANNEL_IPMASQ=false

创建启动脚本

[root@rstx4-203 flannel]# cat flanneld.sh 
#!/bin/sh
./flanneld 
  --public-ip=192.168.1.203 
  --etcd-endpoints=https://192.168.1.202:2379,https://192.168.1.203:2379,https://192.168.1.204:2379 
  --etcd-keyfile=./certs/client-key.pem 
  --etcd-certfile=./certs/client.pem 
  --etcd-cafile=./certs/ca.pem 
  --iface=enp3s0 
  --subnet-file=./subnet.env 
  --healthz-port=2401




./flanneld 
  --public-ip=192.168.1.203 				# 本机IP
  --etcd-endpoints=https://192.168.1.202:2379,https://192.168.1.203:2379,https://192.168.1.204:2379 	# etcd的IP
  --etcd-keyfile=./certs/client-key.pem 
  --etcd-certsfile=./certs/client.pem 
  --etcd-cafile=./certs/ca.pem 
  --iface=enp2s0 							# 注意更改网卡名称
  --subnet-file=./subnet.env 
  --healthz-port=2401


[root@rstx-203 flannel]# chmod +x flanneld.sh 

[root@rstx-203 flannel]# mkdir -p /data/logs/flanneld

---

操作etcd，增加host-gw模型

[root@rstx-203 ~]# cd /opt/etcd

Flannel的host-gw模型，所有node ip必须在同一个物理网关设备下才能使用
[root@rstx-203etcd]# ./etcdctl set /coreos.com/network/config '{"Network": "172.7.0.0/16", "Backend": {"Type": "host-gw"}}'
{"Network": "172.7.0.0/16", "Backend": {"Type": "host-gw"}}

查看etcd集群

[root@rstx-203 etcd]# ./etcdctl member list
1221562a073d0c1a: name=etcd-server-203 peerURLs=https://192.168.1.203:2380 clientURLs=http://127.0.0.1:2379,https://192.168.1.203:2379 isLeader=true
89fe55dc5058e687: name=etcd-server-204 peerURLs=https://192.168.1.204:2380 clientURLs=http://127.0.0.1:2379,https://192.168.1.204:2379 isLeader=false
af26e2bc543fbfba: name=etcd-server-202 peerURLs=https://192.168.1.202:2380 clientURLs=http://127.0.0.1:2379,https://192.168.1.202:2379 isLeader=false

查看flanneld网络

[root@rstx-203 etcd]# ./etcdctl get /coreos.com/network/config
{"Network": "172.7.0.0/16", "Backend": {"Type": "host-gw"}}

---

[root@rstx-203 flannel]# vi /etc/supervisord.d/flannel.ini

[program:flanneld-203]
command=/opt/flannel/flanneld.sh                             ; the program (relative uses PATH, can take args)
numprocs=1                                                   ; number of processes copies to start (def 1)
directory=/opt/flannel                                       ; directory to cwd to before exec (def no cwd)
autostart=true                                               ; start at supervisord start (default: true)
autorestart=true                                             ; retstart at unexpected quit (default: true)
startsecs=30                                                 ; number of secs prog must stay running (def. 1)
startretries=3                                               ; max # of serial start failures (default 3)
exitcodes=0,2                                                ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT                                              ; signal used to kill process (default TERM)
stopwaitsecs=10                                              ; max num secs to wait b4 SIGKILL (default 10)
user=root                                                    ; setuid to this UNIX account to run the program
redirect_stderr=true                                         ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/flanneld/flanneld.stdout.log       ; stderr log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB                                 ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4                                     ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB                                  ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false                                  ; emit events on stdout writes (default false)
killasgroup=true
stopasgroup=true

[root@rstx-203 flannel]# supervisorctl update
flanneld-7-121: added process group
[root@rstx-203 flannel]# supervisorctl status
flanneld-7-121                    RUNNING   pid 9463, uptime 0:00:54
kube-kubelet-7-121                RUNNING   pid 6683, uptime 18:24:51
kube-proxy-7-121                  RUNNING   pid 32574, uptime 17:50:07

---

详解flanneld工作原理

[root@rstx-203 etcd]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.1.1     0.0.0.0         UG    0      0        0 eth0
169.254.0.0     0.0.0.0         255.255.0.0     U     1002   0        0 eth0
172.7.67.0      192.168.1.204   255.255.255.0   UG    0      0        0 eth0
172.7.203.0     0.0.0.0         255.255.255.0   U     0      0        0 docker0
192.168.1.0     0.0.0.0         255.255.255.0   U     0      0        0 eth0

flannel原理就是：给宿主机添加一个静态路由，到达pod ip

Flannel的host-gw模型，所有node ip必须在同一个物理网管设备下才能使用

---

flannel的其他网络模型

Vxlan模型
'{"Network": "172.7.0.0/16", "Backend": {"Type": "VxLAN"}}'
[root@hdss7-14 etcd]# ./etcdctl set /coreos.com/network/config '{"Network": "172.7.0.0/16", "Backend": {"Type": "VxLAN"}}'

直接路由模型
'{"Network": "172.7.0.0/16", "Backend": {"Type": "VxLAN","Directrouting": true}}'

更改flanneld的网络模型

[root@rstx-203 flannel]# supervisorctl stop flanneld-7-11
[root@rstx-203 flannel]# ps aux | grep flanneld 
[root@rstx-203 flannel]# kill 

删除路由

[root@rstx-203 flannel]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.16.1    0.0.0.0         UG    100    0        0 enp2s0
172.7.11.0      192.168.16.11   255.255.255.0   UG    0      0        0 enp2s0
172.7.12.0      0.0.0.0         255.255.255.0   U     0      0        0 docker0
172.7.13.0      192.168.16.13   255.255.255.0   UG    0      0        0 enp2s0
172.7.14.0      192.168.16.14   255.255.255.0   UG    0      0        0 enp2s0
172.17.0.0      0.0.0.0         255.255.0.0     U     0      0        0 br-7957b5310092
192.168.16.0    0.0.0.0         255.255.255.0   U     100    0        0 enp2s0

[root@rstx-203 flannel]# route del -net 172.7.12.0/24 gw 192.168.16.12
[root@rstx-203 flannel]# route del -net 172.7.11.0/24 gw 192.168.16.11

[root@rstx-203 flannel]# ./etcdctl rm /coreos.com/network/config

[root@rstx-203 etcd]# ./etcdctl set /coreos.com/network/config '{"Network": "172.7.0.0/16", "Backend": {"Type": "VxLAN"}}'

[root@rstx-203 flannel]# supervisorctl start flanneld-7-11

---

直接路由模型 -- 当node不在同一个物理网关下，走vxaln模型,在同一个网关下，走host-gw模型
'{"Network": "172.7.0.0/16", "Backend": {"Type": "VxLAN","Directrouting": true}}'

iptables优化

所有节点运行 -- 不优化的话，iptables会占有大量的资源

~]# yum install iptables-services -y

~]# systemctl start iptables 

~]# systemctl enable iptables 

~]# iptables-save | grep -i postrouting

# 此条命令。重启docker会自动添加上，需要写个脚本，每次开机自动自动运行
~]# iptables -t nat -D POSTROUTING -s 172.7.203.0/24 ! -o docker0 -j MASQUERADE		# 删除172.7.203.0/24 为此主机上运行的pod网段

~]# iptables -t nat -I POSTROUTING -s 172.7.203.0/24 ! -d 172.7.0.0/16 ! -o docker0 -j MASQUERADE	# 添加172.7.203.0/24 为此主机上运行的pod网段

~]# iptables-save |grep -i postrouting

~]# iptables-save > /etc/sysconfig/iptables

192.168.1.203主机上，来源是172.7.203.0/24段的docker的ip，目标ip不是172.7.0.0/16段，网络发包不从docker0桥设备上出站的，才进行SNAT转换，pod之间看到的是真实的pod ip

删除 filter表上的两个规则

[root@rstx-203 ~]# iptables-save | grep -i reject

[root@rstx-203 ~]# iptables -t filter -D INPUT -j REJECT --reject-with icmp-host-prohibited
[root@rstx-203 ~]# iptables -t filter -D FORWARD -j REJECT --reject-with icmp-host-prohibited

~]# iptables-save > /etc/sysconfig/iptables

[root@rstx-203 ~]# service iptables save