k8s设计了网络模型，但却将他的实现交给了网络插件，CNI网络插件最主要的功能
就是实现POD资源能够跨主机进行通信
最常见的CNI网络插件：
Flannel、Calico、Cancl、OpenContrail、NSX-T、Kube-router
flannel下载地址
https://github.com/coreos/flannel/releases
wget https://github.com/coreos/flannel/releases/download/v0.12.0/flannel-v0.12.0-linux-amd64.tar.gz
长传安装包
[root@rstx-203 ~]# cd /opt/src/
[root@rstx-203 src]# ls
flannel-v0.12.0-linux-amd64.tar.gz
[root@rstx-203 src]# mkdir /opt/flannel-v0.12.0
[root@rstx-203 src]# tar xf flannel-v0.12.0-linux-amd64.tar.gz -C /opt/flannel-v0.12.0/
创建软链接
[root@rstx-203 src]# ln -s /opt/flannel-v0.12.0/ /opt/flannel
[root@rstx-203 src]# cd ..
[root@rstx-203 opt]# ll
总用量 0
drwx--x--x 4 root root 28 12月 17 10:52 containerd
lrwxrwxrwx 1 root root 21 12月 18 09:48 flannel -> /opt/flannel-v0.12.0/
drwxr-xr-x 2 root root 64 12月 18 09:47 flannel-v0.12.0
lrwxrwxrwx 1 root root 24 12月 17 10:35 kubernetes -> /opt/kubernetes-v1.15.2/
drwxr-xr-x 4 root root 50 12月 17 10:35 kubernetes-v1.15.2
drwxr-xr-x 2 root root 100 12月 18 09:46 src
拷贝证书
[root@rstx-203 opt]# cd flannel
[root@rstx-203 flannel]# mkdir certs
[root@rstx-203 flannel]# cd certs/
[root@rstx-203 certs]# scp rstx-203.host.com:/opt/certs/ca.pem .
[root@rstx-203 certs]# scp rstx-203.host.com:/opt/certs/client.pem .
[root@rstx-203 certs]# scp rstx-203.host.com:/opt/certs/client-key.pem .
[root@rstx-203 certs]# cd ..
[root@rstx-203 flannel]# vi subnet.env
FLANNEL_NETWORK=172.7.0.0/16 #pod的网段
FLANNEL_SUBNET=172.7.203.1/24 #本机运行pod的网段
FLANNEL_MTU=1500
FLANNEL_IPMASQ=false
创建启动脚本
[root@rstx4-203 flannel]# cat flanneld.sh
#!/bin/sh
./flanneld
--public-ip=192.168.1.203
--etcd-endpoints=https://192.168.1.202:2379,https://192.168.1.203:2379,https://192.168.1.204:2379
--etcd-keyfile=./certs/client-key.pem
--etcd-certfile=./certs/client.pem
--etcd-cafile=./certs/ca.pem
--iface=enp3s0
--subnet-file=./subnet.env
--healthz-port=2401
./flanneld
--public-ip=192.168.1.203 # 本机IP
--etcd-endpoints=https://192.168.1.202:2379,https://192.168.1.203:2379,https://192.168.1.204:2379 # etcd的IP
--etcd-keyfile=./certs/client-key.pem
--etcd-certsfile=./certs/client.pem
--etcd-cafile=./certs/ca.pem
--iface=enp2s0 # 注意更改网卡名称
--subnet-file=./subnet.env
--healthz-port=2401
[root@rstx-203 flannel]# chmod +x flanneld.sh
[root@rstx-203 flannel]# mkdir -p /data/logs/flanneld
操作etcd，增加host-gw模型
[root@rstx-203 ~]# cd /opt/etcd
Flannel的host-gw模型，所有node ip必须在同一个物理网关设备下才能使用
[root@rstx-203etcd]# ./etcdctl set /coreos.com/network/config '{"Network": "172.7.0.0/16", "Backend": {"Type": "host-gw"}}'
{"Network": "172.7.0.0/16", "Backend": {"Type": "host-gw"}}
查看etcd集群
[root@rstx-203 etcd]# ./etcdctl member list
1221562a073d0c1a: name=etcd-server-203 peerURLs=https://192.168.1.203:2380 clientURLs=http://127.0.0.1:2379,https://192.168.1.203:2379 isLeader=true
89fe55dc5058e687: name=etcd-server-204 peerURLs=https://192.168.1.204:2380 clientURLs=http://127.0.0.1:2379,https://192.168.1.204:2379 isLeader=false
af26e2bc543fbfba: name=etcd-server-202 peerURLs=https://192.168.1.202:2380 clientURLs=http://127.0.0.1:2379,https://192.168.1.202:2379 isLeader=false
查看flanneld网络
[root@rstx-203 etcd]# ./etcdctl get /coreos.com/network/config
{"Network": "172.7.0.0/16", "Backend": {"Type": "host-gw"}}
[root@rstx-203 flannel]# vi /etc/supervisord.d/flannel.ini
[program:flanneld-203]
command=/opt/flannel/flanneld.sh ; the program (relative uses PATH, can take args)
numprocs=1 ; number of processes copies to start (def 1)
directory=/opt/flannel ; directory to cwd to before exec (def no cwd)
autostart=true ; start at supervisord start (default: true)
autorestart=true ; retstart at unexpected quit (default: true)
startsecs=30 ; number of secs prog must stay running (def. 1)
startretries=3 ; max # of serial start failures (default 3)
exitcodes=0,2 ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT ; signal used to kill process (default TERM)
stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10)
user=root ; setuid to this UNIX account to run the program
redirect_stderr=true ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/flanneld/flanneld.stdout.log ; stderr log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4 ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false ; emit events on stdout writes (default false)
killasgroup=true
stopasgroup=true
[root@rstx-203 flannel]# supervisorctl update
flanneld-7-121: added process group
[root@rstx-203 flannel]# supervisorctl status
flanneld-7-121 RUNNING pid 9463, uptime 0:00:54
kube-kubelet-7-121 RUNNING pid 6683, uptime 18:24:51
kube-proxy-7-121 RUNNING pid 32574, uptime 17:50:07
详解flanneld工作原理
[root@rstx-203 etcd]# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 192.168.1.1 0.0.0.0 UG 0 0 0 eth0
169.254.0.0 0.0.0.0 255.255.0.0 U 1002 0 0 eth0
172.7.67.0 192.168.1.204 255.255.255.0 UG 0 0 0 eth0
172.7.203.0 0.0.0.0 255.255.255.0 U 0 0 0 docker0
192.168.1.0 0.0.0.0 255.255.255.0 U 0 0 0 eth0
flannel原理就是：给宿主机添加一个静态路由，到达pod ip
Flannel的host-gw模型，所有node ip必须在同一个物理网管设备下才能使用
flannel的其他网络模型
Vxlan模型
'{"Network": "172.7.0.0/16", "Backend": {"Type": "VxLAN"}}'
[root@hdss7-14 etcd]# ./etcdctl set /coreos.com/network/config '{"Network": "172.7.0.0/16", "Backend": {"Type": "VxLAN"}}'
直接路由模型
'{"Network": "172.7.0.0/16", "Backend": {"Type": "VxLAN","Directrouting": true}}'
更改flanneld的网络模型
[root@rstx-203 flannel]# supervisorctl stop flanneld-7-11
[root@rstx-203 flannel]# ps aux | grep flanneld
[root@rstx-203 flannel]# kill
删除路由
[root@rstx-203 flannel]# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 192.168.16.1 0.0.0.0 UG 100 0 0 enp2s0
172.7.11.0 192.168.16.11 255.255.255.0 UG 0 0 0 enp2s0
172.7.12.0 0.0.0.0 255.255.255.0 U 0 0 0 docker0
172.7.13.0 192.168.16.13 255.255.255.0 UG 0 0 0 enp2s0
172.7.14.0 192.168.16.14 255.255.255.0 UG 0 0 0 enp2s0
172.17.0.0 0.0.0.0 255.255.0.0 U 0 0 0 br-7957b5310092
192.168.16.0 0.0.0.0 255.255.255.0 U 100 0 0 enp2s0
[root@rstx-203 flannel]# route del -net 172.7.12.0/24 gw 192.168.16.12
[root@rstx-203 flannel]# route del -net 172.7.11.0/24 gw 192.168.16.11
[root@rstx-203 flannel]# ./etcdctl rm /coreos.com/network/config
[root@rstx-203 etcd]# ./etcdctl set /coreos.com/network/config '{"Network": "172.7.0.0/16", "Backend": {"Type": "VxLAN"}}'
[root@rstx-203 flannel]# supervisorctl start flanneld-7-11
直接路由模型 -- 当node不在同一个物理网关下，走vxaln模型,在同一个网关下，走host-gw模型
'{"Network": "172.7.0.0/16", "Backend": {"Type": "VxLAN","Directrouting": true}}'
iptables优化
所有节点运行 -- 不优化的话，iptables会占有大量的资源
~]# yum install iptables-services -y
~]# systemctl start iptables
~]# systemctl enable iptables
~]# iptables-save | grep -i postrouting
# 此条命令。重启docker会自动添加上，需要写个脚本，每次开机自动自动运行
~]# iptables -t nat -D POSTROUTING -s 172.7.203.0/24 ! -o docker0 -j MASQUERADE # 删除172.7.203.0/24 为此主机上运行的pod网段
~]# iptables -t nat -I POSTROUTING -s 172.7.203.0/24 ! -d 172.7.0.0/16 ! -o docker0 -j MASQUERADE # 添加172.7.203.0/24 为此主机上运行的pod网段
~]# iptables-save |grep -i postrouting
~]# iptables-save > /etc/sysconfig/iptables
192.168.1.203主机上，来源是172.7.203.0/24段的docker的ip，目标ip不是172.7.0.0/16段，网络发包不从docker0桥设备上出站的，才进行SNAT转换，pod之间看到的是真实的pod ip
删除 filter表上的两个规则
[root@rstx-203 ~]# iptables-save | grep -i reject
[root@rstx-203 ~]# iptables -t filter -D INPUT -j REJECT --reject-with icmp-host-prohibited
[root@rstx-203 ~]# iptables -t filter -D FORWARD -j REJECT --reject-with icmp-host-prohibited
~]# iptables-save > /etc/sysconfig/iptables
[root@rstx-203 ~]# service iptables save