root@ubuntu:~# kubectl get pod --all-namespaces -o wide NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES kube-system calico-kube-controllers-5978c5f6b5-tk6pg 1/1 Running 0 2d19h 10.244.243.194 ubuntu <none> <none> kube-system calico-node-6fwpp 1/1 Running 0 2d19h 10.10.16.47 cloud <none> <none> kube-system calico-node-hdkcz 0/1 Running 0 2d19h 10.10.16.81 bogon <none> <none> kube-system calico-node-xldz2 1/1 Running 0 2d19h 10.10.16.82 ubuntu <none> <none> kube-system coredns-66bff467f8-krldv 0/1 CrashLoopBackOff 793 2d19h 10.244.243.195 ubuntu <none> <none> kube-system coredns-66bff467f8-t9qcf 0/1 CrashLoopBackOff 794 2d19h 10.244.243.193 ubuntu <none> <none> kube-system etcd-ubuntu 1/1 Running 4 2d19h 10.10.16.82 ubuntu <none> <none> kube-system kube-apiserver-ubuntu 1/1 Running 7 2d19h 10.10.16.82 ubuntu <none> <none> kube-system kube-controller-manager-ubuntu 1/1 Running 5 2d19h 10.10.16.82 ubuntu <none> <none> kube-system kube-proxy-798sq 1/1 Running 0 2d19h 10.10.16.47 cloud <none> <none> kube-system kube-proxy-8hh62 1/1 Running 0 2d19h 10.10.16.82 ubuntu <none> <none> kube-system kube-proxy-l268b 1/1 Running 0 2d19h 10.10.16.81 bogon <none> <none> kube-system kube-scheduler-ubuntu 1/1 Running 7 2d19h 10.10.16.82 ubuntu <none> <none> root@ubuntu:~#
找到原因当部署在Kubernetes中的CoreDNS Pod检测到循环时，CoreDNS Pod将开始“CrashLoopBackOff”。这是因为每当CoreDNS检测到循环并退出时，Kubernetes将尝试重新启动Pod。
找到 /etc/resolv.conf 里面的文件
# Dynamic resolv.conf(5) file for glibc resolver(3) generated by resolvconf(8)
# DO NOT EDIT THIS FILE BY HAND -- YOUR CHANGES WILL BE OVERWRITTEN
nameserver 127.0.1.1
发现那么sever 127.0.1.1是指向本地，修改文件为
# Dynamic resolv.conf(5) file for glibc resolver(3) generated by resolvconf(8)
# DO NOT EDIT THIS FILE BY HAND -- YOUR CHANGES WILL BE OVERWRITTEN
#nameserver 127.0.1.1
nameserver 8.8.8.8
nameserver 8.8.4.4
然后重启服务
# systemctl daemon-reload
# systemctl restart docker
还是一样，没有用
root@ubuntu:~# kubectl get pod --all-namespaces -o wide NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES kube-system calico-kube-controllers-5978c5f6b5-tk6pg 1/1 Running 0 2d19h 10.244.243.194 ubuntu <none> <none> kube-system calico-node-6fwpp 1/1 Running 0 2d19h 10.10.16.47 cloud <none> <none> kube-system calico-node-hdkcz 0/1 Running 0 2d19h 10.10.16.81 bogon <none> <none> kube-system calico-node-xldz2 1/1 Running 0 2d19h 10.10.16.82 ubuntu <none> <none> kube-system coredns-66bff467f8-krldv 0/1 CrashLoopBackOff 794 2d19h 10.244.243.195 ubuntu <none> <none> kube-system coredns-66bff467f8-t9qcf 0/1 CrashLoopBackOff 794 2d19h 10.244.243.193 ubuntu <none> <none> kube-system etcd-ubuntu 1/1 Running 4 2d19h 10.10.16.82 ubuntu <none> <none> kube-system kube-apiserver-ubuntu 1/1 Running 7 2d19h 10.10.16.82 ubuntu <none> <none> kube-system kube-controller-manager-ubuntu 1/1 Running 5 2d19h 10.10.16.82 ubuntu <none> <none> kube-system kube-proxy-798sq 1/1 Running 0 2d19h 10.10.16.47 cloud <none> <none> kube-system kube-proxy-8hh62 1/1 Running 0 2d19h 10.10.16.82 ubuntu <none> <none> kube-system kube-proxy-l268b 1/1 Running 0 2d19h 10.10.16.81 bogon <none> <none> kube-system kube-scheduler-ubuntu 1/1 Running 7 2d19h 10.10.16.82 ubuntu <none> <none>
root@ubuntu:~# kubectl logs -f coredns-66bff467f8-t9qcf Error from server (NotFound): pods "coredns-66bff467f8-t9qcf" not found root@ubuntu:~# kubectl logs -f coredns-66bff467f8-t9qcf -n kube-system .:53 [INFO] plugin/reload: Running configuration MD5 = 4e235fcc3696966e76816bcd9034ebc7 CoreDNS-1.7.1 linux/arm64, go1.15.2, aa82ca6 [FATAL] plugin/loop: Loop (127.0.0.1:35549 -> :53) detected for zone ".", see https://coredns.io/plugins/loop#troubleshooting. Query: "HINFO 5420486584157748416.693057189719862717." root@ubuntu:~# kubectl logs -f coredns-66bff467f8-t9qcf -n kube-system .:53 [INFO] plugin/reload: Running configuration MD5 = 4e235fcc3696966e76816bcd9034ebc7 CoreDNS-1.7.1 linux/arm64, go1.15.2, aa82ca6 [FATAL] plugin/loop: Loop (127.0.0.1:35549 -> :53) detected for zone ".", see https://coredns.io/plugins/loop#troubleshooting. Query: "HINFO 5420486584157748416.693057189719862717."
删除coredns的pod，重新分配试试
root@ubuntu:~# kubectl get pod --all-namespaces -o wide NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES kube-system calico-kube-controllers-5978c5f6b5-tk6pg 1/1 Running 0 2d19h 10.244.243.194 ubuntu <none> <none> kube-system calico-node-6fwpp 1/1 Running 0 2d19h 10.10.16.47 cloud <none> <none> kube-system calico-node-hdkcz 0/1 Running 0 2d19h 10.10.16.81 bogon <none> <none> kube-system calico-node-xldz2 1/1 Running 0 2d19h 10.10.16.82 ubuntu <none> <none> kube-system coredns-66bff467f8-hlbzk 1/1 Running 0 18s 10.244.29.1 bogon <none> <none> kube-system coredns-66bff467f8-t9qcf 0/1 CrashLoopBackOff 795 2d19h 10.244.243.193 ubuntu <none> <none> kube-system etcd-ubuntu 1/1 Running 4 2d19h 10.10.16.82 ubuntu <none> <none> kube-system kube-apiserver-ubuntu 1/1 Running 7 2d19h 10.10.16.82 ubuntu <none> <none> kube-system kube-controller-manager-ubuntu 1/1 Running 5 2d19h 10.10.16.82 ubuntu <none> <none> kube-system kube-proxy-798sq 1/1 Running 0 2d19h 10.10.16.47 cloud <none> <none> kube-system kube-proxy-8hh62 1/1 Running 0 2d19h 10.10.16.82 ubuntu <none> <none> kube-system kube-proxy-l268b 1/1 Running 0 2d19h 10.10.16.81 bogon <none> <none> kube-system kube-scheduler-ubuntu 1/1 Running 7 2d19h 10.10.16.82 ubuntu <none> <none> root@ubuntu:~# kubectl delete pod coredns-66bff467f8-t9qcf -n kube-system pod "coredns-66bff467f8-t9qcf" deleted root@ubuntu:~# kubectl get pod --all-namespaces -o wide NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES kube-system calico-kube-controllers-5978c5f6b5-tk6pg 1/1 Running 0 2d19h 10.244.243.194 ubuntu <none> <none> kube-system calico-node-6fwpp 1/1 Running 0 2d19h 10.10.16.47 cloud <none> <none> kube-system calico-node-hdkcz 0/1 Running 0 2d19h 10.10.16.81 bogon <none> <none> kube-system calico-node-xldz2 1/1 Running 0 2d19h 10.10.16.82 ubuntu <none> <none> kube-system coredns-66bff467f8-hlbzk 1/1 Running 0 62s 10.244.29.1 bogon <none> <none> kube-system coredns-66bff467f8-zx85v 1/1 Running 0 31s 10.244.41.1 cloud <none> <none> kube-system etcd-ubuntu 1/1 Running 4 2d19h 10.10.16.82 ubuntu <none> <none> kube-system kube-apiserver-ubuntu 1/1 Running 7 2d19h 10.10.16.82 ubuntu <none> <none> kube-system kube-controller-manager-ubuntu 1/1 Running 5 2d19h 10.10.16.82 ubuntu <none> <none> kube-system kube-proxy-798sq 1/1 Running 0 2d19h 10.10.16.47 cloud <none> <none> kube-system kube-proxy-8hh62 1/1 Running 0 2d19h 10.10.16.82 ubuntu <none> <none> kube-system kube-proxy-l268b 1/1 Running 0 2d19h 10.10.16.81 bogon <none> <none> kube-system kube-scheduler-ubuntu 1/1 Running 7 2d19h 10.10.16.82 ubuntu <none> <none> root@ubuntu:~#
# Enable IPIP - name: CALICO_IPV4POOL_IPIP value: "Always"
采用的是ipip模式
root@ubuntu:~# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: enahisic2i0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000 link/ether 48:57:02:64:e7:ab brd ff:ff:ff:ff:ff:ff inet 10.10.16.82/24 brd 10.10.16.255 scope global enahisic2i0 valid_lft forever preferred_lft forever inet 10.10.16.250/32 scope global enahisic2i0 valid_lft forever preferred_lft forever inet6 fe80::4a57:2ff:fe64:e7ab/64 scope link valid_lft forever preferred_lft forever 3: enahisic2i1: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000 link/ether 48:57:02:64:e7:ac brd ff:ff:ff:ff:ff:ff 4: enahisic2i2: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000 link/ether 48:57:02:64:e7:ad brd ff:ff:ff:ff:ff:ff 5: enahisic2i3: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000 link/ether 48:57:02:64:e7:ae brd ff:ff:ff:ff:ff:ff 6: virbr0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default qlen 1000 link/ether 52:54:00:3e:ab:e5 brd ff:ff:ff:ff:ff:ff inet 192.168.122.1/24 brd 192.168.122.255 scope global virbr0 valid_lft forever preferred_lft forever 7: virbr0-nic: <BROADCAST,MULTICAST> mtu 1500 qdisc fq_codel master virbr0 state DOWN group default qlen 1000 link/ether 52:54:00:3e:ab:e5 brd ff:ff:ff:ff:ff:ff 8: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default link/ether 02:42:94:16:24:55 brd ff:ff:ff:ff:ff:ff inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0 valid_lft forever preferred_lft forever 9: tunl0@NONE: <NOARP,UP,LOWER_UP> mtu 1440 qdisc noqueue state UNKNOWN group default qlen 1000 link/ipip 0.0.0.0 brd 0.0.0.0 inet 10.244.243.192/32 scope global tunl0 valid_lft forever preferred_lft forever 13: cali0d27bc8b0f7@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1440 qdisc noqueue state UP group default link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 1 inet6 fe80::ecee:eeff:feee:eeee/64 scope link valid_lft forever preferred_lft forever root@ubuntu:~#