etcd 删除

vim /etc/sysconfig/flanneld 

FLANNEL_ETCD_ENDPOINTS="https://192.168.30.241:2379,https://192.168.30.242:2379,https://192.168.30.243:2379"
FLANNEL_ETCD_PREFIX="/kube-centos/network"
FLANNEL_OPTIONS="-etcd-cafile=/etc/kubernetes/ssl/ca.pem -etcd-certfile=/etc/kubernetes/ssl/etcd.pem -etcd-keyfile=/etc/kubernetes/ssl/etcd-key.pem --iface=ens32"

flannel  systemd文件

[Unit]
Description=Flanneld overlay address etcd agent
After=network.target
After=network-online.target
Wants=network-online.target
After=etcd.service
Before=docker.service

[Service]
Type=notify
EnvironmentFile=/etc/sysconfig/flanneld
EnvironmentFile=-/etc/sysconfig/docker-network
ExecStart=/usr/bin/flanneld-start $FLANNEL_OPTIONS
ExecStartPost=/usr/libexec/flannel/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker
Restart=on-failure

[Install]
WantedBy=multi-user.target
RequiredBy=docker.service
~                             

[root@etcd1 ~]# etcdctl --cert-file=/etc/etcd/ssl/etcd.pem --key-file=/etc/etcd/ssl/etcd-key.pem --ca-file=/etc/etcd/ssl/ca.pem --endpoints=https://192.168.30.241:2379,https://192.168.30.242:2379,https://192.168.30.243:2379 rm /calico --recursive
[root@etcd1 ~]# 
[root@etcd1 ~]# etcdctl --cert-file=/etc/etcd/ssl/etcd.pem --key-file=/etc/etcd/ssl/etcd-key.pem --ca-file=/etc/etcd/ssl/ca.pem --endpoints=https://192.168.30.241:2379,https://192.168.30.242:2379,https://192.168.30.243:2379 ls /

etcdctl --cert-file=/etc/etcd/ssl/etcd.pem --key-file=/etc/etcd/ssl/etcd-key.pem --ca-file=/etc/etcd/ssl/ca.pem -endpoints=https://192.168.30.11:2379,https://192.168.30.12:2379,https://192.168.30.13:2379   mkdir /kube-centos/network

[root@etcd1 ~]# etcdctl --cert-file=/etc/etcd/ssl/etcd.pem --key-file=/etc/etcd/ssl/etcd-key.pem --ca-file=/etc/etcd/ssl/ca.pem -endpoints=https://192.168.30.11:2379,https://192.168.30.12:2379,https://192.168.30.13:2379     mk /kube-centos/network/config '{"Network":"10.233.0.0/16","SubnetLen":24,"Backend":{"Type":"vxlan"}}'

[root@etcd1 ~]# etcdctl --cert-file=/etc/etcd/ssl/etcd.pem --key-file=/etc/etcd/ssl/etcd-key.pem --ca-file=/etc/etcd/ssl/ca.pem --endpoint "https://192.168.30.241:2379,https://192.168.30.242:2379,https://192.168.30.243:2379" ls /flannel/network/subnets
/flannel/network/subnets/10.233.5.0-25
/flannel/network/subnets/10.233.14.128-25
/flannel/network/subnets/10.233.8.128-25
/flannel/network/subnets/10.233.22.0-25

# 修改其他配置

docker systemd文件

[Unit]
Description=Docker Application Container Engine
Documentation=http://docs.docker.com
After=network.target
Wants=docker-storage-setup.service
Requires=docker-cleanup.timer

[Service]
Type=notify
NotifyAccess=all
EnvironmentFile=-/run/containers/registries.conf
EnvironmentFile=-/etc/sysconfig/docker
EnvironmentFile=-/etc/sysconfig/docker-storage
EnvironmentFile=-/etc/sysconfig/docker-network
Environment=GOTRACEBACK=crash
EnvironmentFile=-/run/flannel/docker
EnvironmentFile=-/run/docker_opts.env
EnvironmentFile=-/run/flannel/subnet.env
Environment=DOCKER_HTTP_HOST_COMPAT=1
Environment=PATH=/usr/libexec/docker:/usr/bin:/usr/sbin
ExecStart=/usr/bin/dockerd-current 
          --add-runtime docker-runc=/usr/libexec/docker/docker-runc-current 
          --default-runtime=docker-runc 
          --exec-opt native.cgroupdriver=systemd 
          --userland-proxy-path=/usr/libexec/docker/docker-proxy-current 
          --bip=${FLANNEL_SUBNET} 
          --mtu=${FLANNEL_MTU} 
          $OPTIONS 
          $DOCKER_STORAGE_OPTIONS 
          $DOCKER_NETWORK_OPTIONS 
          $ADD_REGISTRY 
          $BLOCK_REGISTRY 
          $INSECURE_REGISTRY
          $REGISTRIES
ExecReload=/bin/kill -s HUP $MAINPID
LimitNOFILE=1048576
LimitNPROC=1048576
LimitCORE=infinity
TimeoutStartSec=0
Restart=on-abnormal
MountFlags=slave
KillMode=process

[Install]
WantedBy=multi-user.target
~

设置docker0网桥的IP地址

source /run/flannel/subnet.env
ifconfig docker0 $FLANNEL_SUBNET

是docker daemon restart的时候，往iptables的nat表的DOCKER chain里面加了一条MASQUERADE规则。
解决办法: 
docker的启动参数DOCKER_OPTS里面加入--ip-masq=false



 iptables -P FORWARD ACCEPT

iptalbes 的有多个MARK 模块..用法各不相同..一直没有完全明白..希望高手解释一下各功能的使用及区别....

-m mark
-m connmark
-j MARK
-j CONNMARK
-j CONNSECMARK
-j SECMARK

解释：

小写的是数据包匹配模块，大写的是数据包修改模块。

带 CONN 的是连接的标记，不带的是标记数据包的。

带 SEC 的是用于处理 IPSEC 数据的，不带的是处理一般数据的。

关闭防火墙

systemctl stop firewalld

systemctl disable firewalld

创建/etc/sysctl.d/k8s.conf文件，添加如下内容：

net.bridge.bridge-nf-call-ip6tables = 1

net.bridge.bridge-nf-call-iptables = 1

vm.swappiness=0

执行sysctl -p /etc/sysctl.d/k8s.conf使修改生效

删除swap

swapoff -a

vi /etc/fstab

注释掉如下行

/home/swap swap swap default 0 0

删除 rm /home/swap

禁用SELINUX：

setenforce 0

vi /etc/selinux/configSELINUX=disabled

安装Docker（使用阿里云镜像）

# step 1: 安装必要的一些系统工具

sudo yum install -y yum-utils device-mapper-persistent-data lvm2

# Step 2: 添加软件源信息

sudo yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

# Step 3: 更新并安装 Docker-CE

sudo yum makecache fast

yum install -y --setopt=obsoletes=0

docker-ce-17.03.2.ce-1.el7.centos

docker-ce-selinux-17.03.2.ce-1.el7.centos

systemctl start docker

systemctl enable docker

 

 

 

node

echo 1 > /proc/sys/net/ipv4/ip_forward