docker pull calico/node:v3.1.4 docker pull calico/cni:v3.1.4 docker pull calico/typha:v3.1.4 docker tag calico/node:v3.1.4 quay.io/calico/node:v3.1.4 docker tag calico/cni:v3.1.4 quay.io/calico/cni:v3.1.4 docker tag calico/typha:v3.1.4 quay.io/calico/typha:v3.1.4
下载执行rbac-kdd.yaml文件
curl https://docs.projectcalico.org/v3.1/getting-started/kubernetes/installation/hosted/rbac-kdd.yaml -O kubectl apply -f rbac-kdd.yaml
root@ubuntu:~# kubectl apply -f rbac-kdd.yaml clusterrole.rbac.authorization.k8s.io/calico-node created clusterrolebinding.rbac.authorization.k8s.io/calico-node created root@ubuntu:~#
下载、配置calico.yaml文件
curl https://docs.projectcalico.org/v3.1/getting-started/kubernetes/installation/hosted/kubernetes-datastore/policy-only/1.7/calico.yaml -O
root@ubuntu:~# kubectl apply -f calico.yaml configmap/calico-config created service/calico-typha created customresourcedefinition.apiextensions.k8s.io/felixconfigurations.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/bgpconfigurations.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/ippools.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/hostendpoints.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/clusterinformations.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/globalnetworkpolicies.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/globalnetworksets.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/networkpolicies.crd.projectcalico.org created serviceaccount/calico-node created unable to recognize "calico.yaml": no matches for kind "Deployment" in version "apps/v1beta1" unable to recognize "calico.yaml": no matches for kind "DaemonSet" in version "extensions/v1beta1" root@ubuntu:~#
caclico的版本太低，需要去https://docs.projectcalico.org/getting-started/kubernetes/quickstart 下载更高版本的yaml文件。
curl https://docs.projectcalico.org/manifests/calico.yaml -O
root@ubuntu:~# curl https://docs.projectcalico.org/manifests/calico.yaml -O curl: symbol lookup error: curl: undefined symbol: curl_multi_poll root@ubuntu:~# kubectl apply -f calico.yaml configmap/calico-config configured customresourcedefinition.apiextensions.k8s.io/bgpconfigurations.crd.projectcalico.org configured customresourcedefinition.apiextensions.k8s.io/bgppeers.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/blockaffinities.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/clusterinformations.crd.projectcalico.org configured customresourcedefinition.apiextensions.k8s.io/felixconfigurations.crd.projectcalico.org configured customresourcedefinition.apiextensions.k8s.io/globalnetworkpolicies.crd.projectcalico.org configured customresourcedefinition.apiextensions.k8s.io/globalnetworksets.crd.projectcalico.org configured customresourcedefinition.apiextensions.k8s.io/hostendpoints.crd.projectcalico.org configured customresourcedefinition.apiextensions.k8s.io/ipamblocks.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/ipamconfigs.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/ipamhandles.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/ippools.crd.projectcalico.org configured customresourcedefinition.apiextensions.k8s.io/kubecontrollersconfigurations.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/networkpolicies.crd.projectcalico.org configured customresourcedefinition.apiextensions.k8s.io/networksets.crd.projectcalico.org created clusterrole.rbac.authorization.k8s.io/calico-kube-controllers created clusterrolebinding.rbac.authorization.k8s.io/calico-kube-controllers created clusterrole.rbac.authorization.k8s.io/calico-node configured clusterrolebinding.rbac.authorization.k8s.io/calico-node configured daemonset.apps/calico-node created serviceaccount/calico-node unchanged deployment.apps/calico-kube-controllers created serviceaccount/calico-kube-controllers created poddisruptionbudget.policy/calico-kube-controllers created root@ubuntu:~# kubectl get pods -n kube-system -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES calico-kube-controllers-5978c5f6b5-6kqtf 1/1 Running 0 47s 10.244.2.12 cloud <none> <none> calico-node-db2m9 0/1 PodInitializing 0 48s 10.10.16.82 ubuntu <none> <none> calico-node-jt86r 0/1 Init:0/3 0 48s 10.10.16.81 bogon <none> <none> calico-node-tq8c4 0/1 Init:2/3 0 48s 10.10.16.47 cloud <none> <none> coredns-66bff467f8-57g2p 1/1 Running 0 54m 10.244.2.4 cloud <none> <none> coredns-66bff467f8-bjvn7 1/1 Running 0 54m 10.244.2.2 cloud <none> <none> etcd-ubuntu 1/1 Running 1 245d 10.10.16.82 ubuntu <none> <none> kube-apiserver-ubuntu 1/1 Running 1 245d 10.10.16.82 ubuntu <none> <none> kube-controller-manager-ubuntu 1/1 Running 3 245d 10.10.16.82 ubuntu <none> <none> kube-proxy-896mz 1/1 Running 0 245d 10.10.16.82 ubuntu <none> <none> kube-proxy-nh2cp 1/1 Running 0 22h 10.10.16.47 cloud <none> <none> kube-proxy-p4qkx 1/1 Running 0 41m 10.10.16.81 bogon <none> <none> kube-scheduler-ubuntu 1/1 Running 5 245d 10.10.16.82 ubuntu <none> <none> root@ubuntu:~#
calicoctl安装
参考https://docs.projectcalico.org/getting-started/clis/calicoctl/install
curl -o calicoctl -O -L "https://github.com/projectcalico/calicoctl/releases/download/v3.19.1/calicoctl-linux-arm64"
root@ubuntu:~# ls /etc/calico/calicoctl.cfg ls: cannot access '/etc/calico/calicoctl.cfg': No such file or directory root@ubuntu:~# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+-------------------+-------+------------+-------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+-------------------+-------+------------+-------------+ | 10.10.16.47 | node-to-node mesh | up | 2021-06-18 | Established | | 14.14.18.89 | node-to-node mesh | start | 2021-06-18 | Passive | +--------------+-------------------+-------+------------+-------------+ IPv6 BGP status No IPv6 peers found. root@ubuntu:~#
三个node，但是
calicoctl node status只显示了两个
root@cloud:~# kubectl get pod --all-namespaces -o wide NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES kube-system calico-kube-controllers-5978c5f6b5-tk6pg 1/1 Running 0 2d19h 10.244.243.194 ubuntu <none> <none> kube-system calico-node-6fwpp 1/1 Running 0 2d19h 10.10.16.47 cloud <none> <none> kube-system calico-node-hdkcz 0/1 Running 0 2d19h 10.10.16.81 bogon <none> <none> kube-system calico-node-xldz2 1/1 Running 0 2d19h 10.10.16.82 ubuntu <none> <none> kube-system coredns-66bff467f8-krldv 0/1 CrashLoopBackOff 791 2d19h 10.244.243.195 ubuntu <none> <none> kube-system coredns-66bff467f8-t9qcf 0/1 CrashLoopBackOff 792 2d19h 10.244.243.193 ubuntu <none> <none> kube-system etcd-ubuntu 1/1 Running 4 2d19h 10.10.16.82 ubuntu <none> <none> kube-system kube-apiserver-ubuntu 1/1 Running 7 2d19h 10.10.16.82 ubuntu <none> <none> kube-system kube-controller-manager-ubuntu 1/1 Running 5 2d19h 10.10.16.82 ubuntu <none> <none> kube-system kube-proxy-798sq 1/1 Running 0 2d19h 10.10.16.47 cloud <none> <none> kube-system kube-proxy-8hh62 1/1 Running 0 2d19h 10.10.16.82 ubuntu <none> <none> kube-system kube-proxy-l268b 1/1 Running 0 2d19h 10.10.16.81 bogon <none> <none> kube-system kube-scheduler-ubuntu 1/1 Running 7 2d19h 10.10.16.82 ubuntu <none> <none>
原来采用的ipip模式
# Enable IPIP - name: CALICO_IPV4POOL_IPIP value: "Always" # Enable or Disabl
root@cloud:~# kubectl get pod --all-namespaces -o wide Unable to connect to the server: x509: certificate signed by unknown authority (possibly because of "crypto/rsa: verification error" while trying to verify candidate authority certificate "kubernetes") root@cloud:~#
master节点没问题
root@ubuntu:~# kubectl get pod --all-namespaces -o wide NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES kube-system calico-kube-controllers-5978c5f6b5-tk6pg 1/1 Running 0 2d16h 10.244.243.194 ubuntu <none> <none> kube-system calico-node-6fwpp 1/1 Running 0 2d16h 10.10.16.47 cloud <none> <none> kube-system calico-node-hdkcz 0/1 Running 0 2d16h 10.10.16.81 bogon <none> <none> kube-system calico-node-xldz2 1/1 Running 0 2d16h 10.10.16.82 ubuntu <none> <none> kube-system coredns-66bff467f8-krldv 0/1 CrashLoopBackOff 764 2d16h 10.244.243.195 ubuntu <none> <none> kube-system coredns-66bff467f8-t9qcf 0/1 CrashLoopBackOff 764 2d16h 10.244.243.193 ubuntu <none> <none> kube-system etcd-ubuntu 1/1 Running 4 2d16h 10.10.16.82 ubuntu <none> <none> kube-system kube-apiserver-ubuntu 1/1 Running 7 2d16h 10.10.16.82 ubuntu <none> <none> kube-system kube-controller-manager-ubuntu 1/1 Running 5 2d16h 10.10.16.82 ubuntu <none> <none> kube-system kube-proxy-798sq 1/1 Running 0 2d16h 10.10.16.47 cloud <none> <none> kube-system kube-proxy-8hh62 1/1 Running 0 2d16h 10.10.16.82 ubuntu <none> <none> kube-system kube-proxy-l268b 1/1 Running 0 2d16h 10.10.16.81 bogon <none> <none> kube-system kube-scheduler-ubuntu 1/1 Running 7 2d16h 10.10.16.82 ubuntu <none> <none> root@ubuntu:~#
解决方法：
kubeadm init初始化完成后提示的操作执行一遍：
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
第一个master节点执行完成后，需将scp /etc/kubernetes/pki/*到其他节点后，其他master节点才能初始化。在kubeadm初始化之前要保证systemctl status kubelet 查看到的状态保持activating (auto-restart)状态，否则kubeadm会报错
更新后可以访问
root@cloud:~# kubectl get pod --all-namespaces -o wide NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES kube-system calico-kube-controllers-5978c5f6b5-tk6pg 1/1 Running 0 2d18h 10.244.243.194 ubuntu <none> <none> kube-system calico-node-6fwpp 1/1 Running 0 2d18h 10.10.16.47 cloud <none> <none> kube-system calico-node-hdkcz 0/1 Running 0 2d18h 10.10.16.81 bogon <none> <none> kube-system calico-node-xldz2 1/1 Running 0 2d18h 10.10.16.82 ubuntu <none> <none> kube-system coredns-66bff467f8-krldv 0/1 CrashLoopBackOff 789 2d19h 10.244.243.195 ubuntu <none> <none> kube-system coredns-66bff467f8-t9qcf 0/1 CrashLoopBackOff 790 2d19h 10.244.243.193 ubuntu <none> <none> kube-system etcd-ubuntu 1/1 Running 4 2d19h 10.10.16.82 ubuntu <none> <none> kube-system kube-apiserver-ubuntu 1/1 Running 7 2d19h 10.10.16.82 ubuntu <none> <none> kube-system kube-controller-manager-ubuntu 1/1 Running 5 2d19h 10.10.16.82 ubuntu <none> <none> kube-system kube-proxy-798sq 1/1 Running 0 2d18h 10.10.16.47 cloud <none> <none> kube-system kube-proxy-8hh62 1/1 Running 0 2d19h 10.10.16.82 ubuntu <none> <none> kube-system kube-proxy-l268b 1/1 Running 0 2d18h 10.10.16.81 bogon <none> <none> kube-system kube-scheduler-ubuntu 1/1 Running 7 2d19h 10.10.16.82 ubuntu <none> <none> root@cloud:~#
root@ubuntu:~# kubectl -n default describe pod web-nginx-7bdc6b976b-glwvh | grep Container Containers: Container ID: docker://c49c2786c61b0aad42dff61c7a98ccfe6a81ead316fc438da6d45758dd4b572a ContainersReady True root@ubuntu:~# kubectl -n default describe pod web-nginx-7bdc6b976b-glwvh | grep Node Node: bogon/10.10.16.81 Node-Selectors: <none> root@ubuntu:~# kubectl -n default describe pod web-nginx-7bdc6b976b-xzsf2 | grep Container Containers: Container ID: docker://4ce7b443f478e6dba7a8cacfc7eca673709a26418e2a65c85a6630778f915437 ContainersReady True Normal Pulled 4m32s kubelet, cloud Container image "nginx" already present on machine root@ubuntu:~# kubectl -n default describe pod web-nginx-7bdc6b976b-xzsf2 | grep Node Node: cloud/10.10.16.47 Node-Selectors: <none> root@ubuntu:~#
ipip
root@ubuntu:~# kubectl get pods -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES web-nginx-7bdc6b976b-glwvh 1/1 Running 0 35m 10.244.29.2 bogon <none> <none> web-nginx-7bdc6b976b-pqk86 1/1 Running 0 35m 10.244.243.196 ubuntu <none> <none> web-nginx-7bdc6b976b-xzsf2 1/1 Running 0 35m 10.244.41.2 cloud <none> <none> root@ubuntu:~#
81节点
[root@bogon ~]# docker inspect c49c2786c61b | grep -i pid "Pid": 18121, "PidMode": "", "PidsLimit": null, [root@bogon ~]# nsenter -n --target 18121 ABRT has detected 1 problem(s). For more info run: abrt-cli list --since 1624247964 [root@bogon ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever 2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000 link/ipip 0.0.0.0 brd 0.0.0.0 4: eth0@if101: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1440 qdisc noqueue state UP group default link/ether da:b3:97:64:94:03 brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 10.244.29.2/32 brd 10.244.29.2 scope global eth0 valid_lft forever preferred_lft forever [root@bogon ~]#
47节点
root@cloud:~# docker inspect 4ce7b443f478 | grep -i pid "Pid": 188682, "PidMode": "", "PidsLimit": null, root@cloud:~# nsenter -n --target ab2a5aa39300 nsenter: failed to parse pid: 'ab2a5aa39300' root@cloud:~# nsenter -n --target 188682 root@cloud:~# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever 2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000 link/ipip 0.0.0.0 brd 0.0.0.0 4: eth0@if39: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1440 qdisc noqueue state UP group default link/ether 72:98:01:99:c5:5a brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 10.244.41.2/32 brd 10.244.41.2 scope global eth0 valid_lft forever preferred_lft forever root@cloud:~#
81 pod上ping
[root@bogon ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever 2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000 link/ipip 0.0.0.0 brd 0.0.0.0 4: eth0@if101: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1440 qdisc noqueue state UP group default link/ether da:b3:97:64:94:03 brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 10.244.29.2/32 brd 10.244.29.2 scope global eth0 valid_lft forever preferred_lft forever [root@bogon ~]# ping 10.244.41.2 PING 10.244.41.2 (10.244.41.2) 56(84) bytes of data. ^C --- 10.244.41.2 ping statistics --- 8 packets transmitted, 0 received, 100% packet loss, time 7313ms [root@bogon ~]# route -n Kernel IP routing table Destination Gateway Genmask Flags Metric Ref Use Iface 0.0.0.0 169.254.1.1 0.0.0.0 UG 0 0 0 eth0 169.254.1.1 0.0.0.0 255.255.255.255 UH 0 0 0 eth0 [root@bogon ~]# ping 10.244.41.2 PING 10.244.41.2 (10.244.41.2) 56(84) bytes of data. ^C --- 10.244.41.2 ping statistics --- 2 packets transmitted, 0 received, 100% packet loss, time 1054ms [root@bogon ~]# ping 10.244.41.2 PING 10.244.41.2 (10.244.41.2) 56(84) bytes of data.
[root@bogon ~]# route -n Kernel IP routing table Destination Gateway Genmask Flags Metric Ref Use Iface 0.0.0.0 10.10.16.254 0.0.0.0 UG 0 0 0 enahisic2i0 10.2.0.1 0.0.0.0 255.255.255.255 UH 0 0 0 v-cali-peer 10.10.16.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i0 10.10.34.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i2 10.10.102.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i1 10.244.29.0 0.0.0.0 255.255.255.192 U 0 0 0 * 10.244.29.1 0.0.0.0 255.255.255.255 UH 0 0 0 cali2e486421e22 10.244.29.2 0.0.0.0 255.255.255.255 UH 0 0 0 calib87dcb53769 14.14.18.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i3.310 172.16.100.0 0.0.0.0 255.255.255.0 U 0 0 0 brqf1411bad-10 172.17.0.0 0.0.0.0 255.255.0.0 U 0 0 0 docker0 172.168.104.0 0.0.0.0 255.255.255.0 U 0 0 0 enah2i3.1022 192.168.33.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i1 192.168.122.0 0.0.0.0 255.255.255.0 U 0 0 0 virbr0 [root@bogon ~]# ip link show calib87dcb53769 101: calib87dcb53769@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1440 qdisc noqueue state UP mode DEFAULT group default link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 7 [root@bogon ~]# tcpdump -i calib87dcb53769 icmp -ennvv tcpdump: listening on calib87dcb53769, link-type EN10MB (Ethernet), capture size 262144 bytes 15:48:38.757625 da:b3:97:64:94:03 > ee:ee:ee:ee:ee:ee, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 7929, offset 0, flags [DF], proto ICMP (1), length 84) 10.244.29.2 > 10.244.41.2: ICMP echo request, id 47564, seq 1, length 64 15:48:39.812490 da:b3:97:64:94:03 > ee:ee:ee:ee:ee:ee, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 7983, offset 0, flags [DF], proto ICMP (1), length 84) 10.244.29.2 > 10.244.41.2: ICMP echo request, id 47564, seq 2, length 64 ^C 2 packets captured 2 packets received by filter 0 packets dropped by kernel [root@bogon ~]# tcpdump -i enahisic2i0 icmp -ennvv tcpdump: listening on enahisic2i0, link-type EN10MB (Ethernet), capture size 262144 bytes 15:49:25.235307 48:57:02:64:ea:1b > f4:1d:6b:87:53:2a, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 9916, offset 0, flags [DF], proto ICMP (1), length 84) 10.244.29.2 > 10.244.41.2: ICMP echo request, id 48417, seq 1, length 64 15:49:26.292527 48:57:02:64:ea:1b > f4:1d:6b:87:53:2a, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 9972, offset 0, flags [DF], proto ICMP (1), length 84) 10.244.29.2 > 10.244.41.2: ICMP echo request, id 48417, seq 2, length 64 15:49:27.332515 48:57:02:64:ea:1b > f4:1d:6b:87:53:2a, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 10024, offset 0, flags [DF], proto ICMP (1), length 84) 10.244.29.2 > 10.244.41.2: ICMP echo request, id 48417, seq 3, length 64 15:49:28.372513 48:57:02:64:ea:1b > f4:1d:6b:87:53:2a, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 10044, offset 0, flags [DF], proto ICMP (1), length 84) 10.244.29.2 > 10.244.41.2: ICMP echo request, id 48417, seq 4, length 64 ^C 4 packets captured 4 packets received by filter 0 packets dropped by kernel [root@bogon ~]# ip a | grep tun 99: tunl0@NONE: <NOARP,UP,LOWER_UP> mtu 1440 qdisc noqueue state UNKNOWN group default qlen 1000 inet 10.244.29.0/32 scope global tunl0 You have new mail in /var/spool/mail/root [root@bogon ~]#
没有走tun10
root@ubuntu:~# kubectl get ipamblocks NAME AGE 10-244-243-192-26 2d20h 10-244-29-0-26 2d20h 10-244-41-0-26 2d20h root@ubuntu:~# kubectl get ipamblocks 10-244-243-192-26 -o yaml | grep node node: ubuntu node: ubuntu node: ubuntu root@ubuntu:~#
从cloud pod ping ，可以访问
root@cloud:~# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever 2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000 link/ipip 0.0.0.0 brd 0.0.0.0 4: eth0@if39: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1440 qdisc noqueue state UP group default link/ether 72:98:01:99:c5:5a brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 10.244.41.2/32 brd 10.244.41.2 scope global eth0 valid_lft forever preferred_lft forever root@cloud:~# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever 2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000 link/ipip 0.0.0.0 brd 0.0.0.0 4: eth0@if39: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1440 qdisc noqueue state UP group default link/ether 72:98:01:99:c5:5a brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 10.244.41.2/32 brd 10.244.41.2 scope global eth0 valid_lft forever preferred_lft forever root@cloud:~# route -n Kernel IP routing table Destination Gateway Genmask Flags Metric Ref Use Iface 0.0.0.0 169.254.1.1 0.0.0.0 UG 0 0 0 eth0 169.254.1.1 0.0.0.0 255.255.255.255 UH 0 0 0 eth0 root@cloud:~# ping 10.244.243.192 PING 10.244.243.192 (10.244.243.192) 56(84) bytes of data. 64 bytes from 10.244.243.192: icmp_seq=1 ttl=63 time=0.358 ms 64 bytes from 10.244.243.192: icmp_seq=2 ttl=63 time=0.197 ms ^C --- 10.244.243.192 ping statistics --- 2 packets transmitted, 2 received, 0% packet loss, time 1012ms rtt min/avg/max/mdev = 0.197/0.277/0.358/0.082 ms root@cloud:~# ping 10.244.243.196 PING 10.244.243.196 (10.244.243.196) 56(84) bytes of data. 64 bytes from 10.244.243.196: icmp_seq=1 ttl=62 time=0.403 ms 64 bytes from 10.244.243.196: icmp_seq=2 ttl=62 time=0.197 ms ^C --- 10.244.243.196 ping statistics --- 2 packets transmitted, 2 received, 0% packet loss, time 1019ms rtt min/avg/max/mdev = 0.197/0.300/0.403/0.103 ms root@cloud:~#
cloud tcpdump
root@cloud:~# route -n Kernel IP routing table Destination Gateway Genmask Flags Metric Ref Use Iface 0.0.0.0 10.10.16.254 0.0.0.0 UG 0 0 0 enahisic2i0 9.251.0.0 172.17.0.1 255.255.0.0 UG 0 0 0 docker0 10.10.16.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i0 10.99.1.231 10.10.16.82 255.255.255.255 UGH 0 0 0 enahisic2i0 10.110.79.116 10.10.16.82 255.255.255.255 UGH 0 0 0 enahisic2i0 10.110.171.213 10.10.16.82 255.255.255.255 UGH 0 0 0 enahisic2i0 10.244.2.0 0.0.0.0 255.255.255.0 U 0 0 0 cni0 10.244.41.0 0.0.0.0 255.255.255.192 U 0 0 0 * 10.244.41.1 0.0.0.0 255.255.255.255 UH 0 0 0 cali027a65c4a41 10.244.41.2 0.0.0.0 255.255.255.255 UH 0 0 0 cali8763f2b9f05 10.244.243.192 10.10.16.82 255.255.255.192 UG 0 0 0 tunl0 172.17.0.0 0.0.0.0 255.255.0.0 U 0 0 0 docker0 root@cloud:~#
root@cloud:~# tcpdump -i tunl0 icmp -ennvv tcpdump: listening on tunl0, link-type RAW (Raw IP), capture size 262144 bytes 16:05:00.343356 ip: (tos 0x0, ttl 63, id 3394, offset 0, flags [DF], proto ICMP (1), length 84) 10.244.41.2 > 10.244.243.196: ICMP echo request, id 39207, seq 134, length 64 16:05:00.343534 ip: (tos 0x0, ttl 63, id 49183, offset 0, flags [none], proto ICMP (1), length 84) 10.244.243.196 > 10.244.41.2: ICMP echo reply, id 39207, seq 134, length 64 16:05:01.367334 ip: (tos 0x0, ttl 63, id 3603, offset 0, flags [DF], proto ICMP (1), length 84) 10.244.41.2 > 10.244.243.196: ICMP echo request, id 39207, seq 135, length 64 16:05:01.367518 ip: (tos 0x0, ttl 63, id 49386, offset 0, flags [none], proto ICMP (1), length 84) 10.244.243.196 > 10.244.41.2: ICMP echo reply, id 39207, seq 135, length 64 ^C 4 packets captured 4 packets received by filter 0 packets dropped by kernel root@cloud:~#
root@cloud:~# tcpdump -i enahisic2i0 "ip proto 4" and host 10.10.16.82 -ennvv tcpdump: listening on enahisic2i0, link-type EN10MB (Ethernet), capture size 262144 bytes 16:10:01.399380 9c:52:f8:67:c4:d3 > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 118: (tos 0x0, ttl 63, id 63884, offset 0, flags [DF], proto IPIP (4), length 104) 10.10.16.47 > 10.10.16.82: (tos 0x0, ttl 63, id 40436, offset 0, flags [DF], proto ICMP (1), length 84) 10.244.41.2 > 10.244.243.196: ICMP echo request, id 39207, seq 428, length 64 16:10:01.399541 48:57:02:64:e7:ab > 9c:52:f8:67:c4:d3, ethertype IPv4 (0x0800), length 118: (tos 0x0, ttl 63, id 16648, offset 0, flags [none], proto IPIP (4), length 104) 10.10.16.82 > 10.10.16.47: (tos 0x0, ttl 63, id 22181, offset 0, flags [none], proto ICMP (1), length 84) 10.244.243.196 > 10.244.41.2: ICMP echo reply, id 39207, seq 428, length 64 16:10:02.423387 9c:52:f8:67:c4:d3 > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 118: (tos 0x0, ttl 63, id 63998, offset 0, flags [DF], proto IPIP (4), length 104) 10.10.16.47 > 10.10.16.82: (tos 0x0, ttl 63, id 40517, offset 0, flags [DF], proto ICMP (1), length 84) 10.244.41.2 > 10.244.243.196: ICMP echo request, id 39207, seq 429, length 64 16:10:02.423551 48:57:02:64:e7:ab > 9c:52:f8:67:c4:d3, ethertype IPv4 (0x0800), length 118: (tos 0x0, ttl 63, id 16878, offset 0, flags [none], proto IPIP (4), length 104) 10.10.16.82 > 10.10.16.47: (tos 0x0, ttl 63, id 22198, offset 0, flags [none], proto ICMP (1), length 84) 10.244.243.196 > 10.244.41.2: ICMP echo reply, id 39207, seq 429, length 64 16:10:03.447413 9c:52:f8:67:c4:d3 > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 118: (tos 0x0, ttl 63, id 64201, offset 0, flags [DF], proto IPIP (4), length 104) 10.10.16.47 > 10.10.16.82: (tos 0x0, ttl 63, id 40654, offset 0, flags [DF], proto ICMP (1), length 84) 10.244.41.2 > 10.244.243.196: ICMP echo request, id 39207, seq 430, length 64 16:10:03.447555 48:57:02:64:e7:ab > 9c:52:f8:67:c4:d3, ethertype IPv4 (0x0800), length 118: (tos 0x0, ttl 63, id 16970, offset 0, flags [none], proto IPIP (4), length 104) 10.10.16.82 > 10.10.16.47: (tos 0x0, ttl 63, id 22403, offset 0, flags [none], proto ICMP (1), length 84) 10.244.243.196 > 10.244.41.2: ICMP echo reply, id 39207, seq 430, length 64 16:10:04.471319 9c:52:f8:67:c4:d3 > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 118: (tos 0x0, ttl 63, id 64332, offset 0, flags [DF], proto IPIP (4), length 104) 10.10.16.47 > 10.10.16.82: (tos 0x0, ttl 63, id 40671, offset 0, flags [DF], proto ICMP (1), length 84) 10.244.41.2 > 10.244.243.196: ICMP echo request, id 39207, seq 431, length 64 16:10:04.471471 48:57:02:64:e7:ab > 9c:52:f8:67:c4:d3, ethertype IPv4 (0x0800), length 118: (tos 0x0, ttl 63, id 17063, offset 0, flags [none], proto IPIP (4), length 104) 10.10.16.82 > 10.10.16.47: (tos 0x0, ttl 63, id 22565, offset 0, flags [none], proto ICMP (1), length 84) 10.244.243.196 > 10.244.41.2: ICMP echo reply, id 39207, seq 431, length 64 ^C 8 packets captured 9 packets received by filter 0 packets dropped by kernel root@cloud:~#
查看bogon上的calico
root@ubuntu:~# kubectl get pods -n kube-system -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES calico-kube-controllers-5978c5f6b5-tk6pg 1/1 Running 0 2d21h 10.244.243.194 ubuntu <none> <none> calico-node-6fwpp 1/1 Running 0 2d21h 10.10.16.47 cloud <none> <none> calico-node-hdkcz 0/1 Running 0 2d21h 10.10.16.81 bogon <none> <none> calico-node-xldz2 1/1 Running 0 2d21h 10.10.16.82 ubuntu <none> <none> coredns-66bff467f8-hlbzk 1/1 Running 0 97m 10.244.29.1 bogon <none> <none> coredns-66bff467f8-zx85v 1/1 Running 0 96m 10.244.41.1 cloud <none> <none> etcd-ubuntu 1/1 Running 4 2d21h 10.10.16.82 ubuntu <none> <none> kube-apiserver-ubuntu 1/1 Running 7 2d21h 10.10.16.82 ubuntu <none> <none> kube-controller-manager-ubuntu 1/1 Running 5 2d21h 10.10.16.82 ubuntu <none> <none> kube-proxy-798sq 1/1 Running 0 2d21h 10.10.16.47 cloud <none> <none> kube-proxy-8hh62 1/1 Running 0 2d21h 10.10.16.82 ubuntu <none> <none> kube-proxy-l268b 1/1 Running 0 2d21h 10.10.16.81 bogon <none> <none> kube-scheduler-ubuntu 1/1 Running 7 2d21h 10.10.16.82 ubuntu <none> <none>
Events: Type Reason Age From Message ---- ------ ---- ---- ------- Warning Unhealthy 62s (x24801 over 2d20h) kubelet, bogon (combined from similar events): Readiness probe failed: 2021-06-21 08:13:24.282 [INFO][46951] confd/health.go 180: Number of node(s) with BGP peering established = 0 calico/node is not ready: BIRD is not ready: BGP not established with 10.10.16.47,10.10.16.82
179端口可以访问
[root@bogon ~]# netstat -pan | grep 179 tcp 0 0 0.0.0.0:179 0.0.0.0:* LISTEN 29055/bird
root@ubuntu:~# telnet 10.10.16.81 179 Trying 10.10.16.81... Connected to 10.10.16.81. Escape character is '^]'. ?Y" @|A
更改 calico.yaml
- name: IP_AUTODETECTION_METHOD value: "interface=enahisic2i0"
root@ubuntu:~# kubectl apply -f calico.yaml
root@ubuntu:~# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+-------------------+-------+----------+-------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+-------------------+-------+----------+-------------+ | 10.10.16.47 | node-to-node mesh | up | 08:24:08 | Established | | 10.10.16.81 | node-to-node mesh | up | 10:59:50 | Established | +--------------+-------------------+-------+----------+-------------+ IPv6 BGP status No IPv6 peers found.
[root@bogon ~]# route -n Kernel IP routing table Destination Gateway Genmask Flags Metric Ref Use Iface 0.0.0.0 10.10.16.254 0.0.0.0 UG 0 0 0 enahisic2i0 10.2.0.1 0.0.0.0 255.255.255.255 UH 0 0 0 v-cali-peer 10.10.16.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i0 10.10.34.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i2 10.10.102.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i1 10.244.2.0 10.10.16.47 255.255.255.0 UG 0 0 0 tunl0 10.244.29.0 0.0.0.0 255.255.255.192 U 0 0 0 * 10.244.29.1 0.0.0.0 255.255.255.255 UH 0 0 0 cali2e486421e22 10.244.29.2 0.0.0.0 255.255.255.255 UH 0 0 0 calib87dcb53769 10.244.41.0 10.10.16.47 255.255.255.192 UG 0 0 0 tunl0 10.244.243.192 10.10.16.82 255.255.255.192 UG 0 0 0 tunl0
都可以ping 通了
[root@bogon ~]# ping 10.244.41.2 PING 10.244.41.2 (10.244.41.2) 56(84) bytes of data. 64 bytes from 10.244.41.2: icmp_seq=1 ttl=62 time=0.407 ms ^C --- 10.244.41.2 ping statistics --- 1 packets transmitted, 1 received, 0% packet loss, time 0ms rtt min/avg/max/mdev = 0.407/0.407/0.407/0.000 ms [root@bogon ~]# ping 10.244.243.196 PING 10.244.243.196 (10.244.243.196) 56(84) bytes of data. 64 bytes from 10.244.243.196: icmp_seq=1 ttl=62 time=0.382 ms 64 bytes from 10.244.243.196: icmp_seq=2 ttl=62 time=0.234 ms 64 bytes from 10.244.243.196: icmp_seq=3 ttl=62 time=0.219 ms 64 bytes from 10.244.243.196: icmp_seq=4 ttl=62 time=0.209 ms ^C --- 10.244.243.196 ping statistics --- 4 packets transmitted, 4 received, 0% packet loss, time 3115ms rtt min/avg/max/mdev = 0.209/0.261/0.382/0.070 ms [root@bogon ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever 2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000 link/ipip 0.0.0.0 brd 0.0.0.0 4: eth0@if101: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1440 qdisc noqueue state UP group default link/ether da:b3:97:64:94:03 brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 10.244.29.2/32 brd 10.244.29.2 scope global eth0 valid_lft forever preferred_lft forever [root@bogon ~]#
bird进程
[root@bogon ~]# ps -elf | grep bird 0 S root 25128 25042 0 80 0 - 18 poll_s 18:59 ? 00:00:00 runsv bird 0 S root 25129 25042 0 80 0 - 18 poll_s 18:59 ? 00:00:00 runsv bird6 4 S root 25311 25128 0 80 0 - 27 poll_s 18:59 ? 00:00:00 bird -R -s /var/run/calico/bird.ctl -d -c /etc/calico/confd/config/bird.cfg 0 S root 25312 25129 0 80 0 - 27 poll_s 18:59 ? 00:00:00 bird6 -R -s /var/run/calico/bird6.ctl -d -c /etc/calico/confd/config/bird6.cfg
[root@bogon ~]# docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 6d6ed3a4025c dd6d43d932df "start_runit" 6 minutes ago Up 6 minutes k8s_calico-node_calico-node-2qjzx_kube-system_9020f771-abbe-44ed-957e-bc8e7a8214b7_0 fbc032ab2217 k8s.gcr.io/pause:3.2 "/pause" 6 minutes ago Up 6 minutes k8s_POD_calico-node-2qjzx_kube-system_9020f771-abbe-44ed-957e-bc8e7a8214b7_0 c49c2786c61b nginx "/docker-entrypoint.…" 4 hours ago Up 4 hours k8s_web2-worker_web-nginx-7bdc6b976b-glwvh_default_ed36eef4-d46d-41cd-9a48-f28eccc42a80_0 cc95c060ac60 k8s.gcr.io/pause:3.2 "/pause" 4 hours ago Up 4 hours k8s_POD_web-nginx-7bdc6b976b-glwvh_default_ed36eef4-d46d-41cd-9a48-f28eccc42a80_0 c3f286f69d98 31084f9a8be6 "/coredns -conf /etc…" 4 hours ago Up 4 hours k8s_coredns_coredns-66bff467f8-hlbzk_kube-system_c2b829bb-17a8-4c90-8485-e81d9dc949f1_0 c0933eac8349 k8s.gcr.io/pause:3.2 "/pause" 4 hours ago Up 4 hours k8s_POD_coredns-66bff467f8-hlbzk_kube-system_c2b829bb-17a8-4c90-8485-e81d9dc949f1_0 282d756684d8 f782b1121865 "/usr/local/bin/kube…" 3 days ago Up 3 days k8s_kube-proxy_kube-proxy-l268b_kube-system_ce9c084b-8915-419c-af4b-ca2e6789f77d_0 72635b27607b k8s.gcr.io/pause:3.2 "/pause" 3 days ago Up 3 days k8s_POD_kube-proxy-l268b_kube-system_ce9c084b-8915-419c-af4b-ca2e6789f77d_0 [root@bogon ~]# ps -elf | grep bird 0 S root 25128 25042 0 80 0 - 18 poll_s 18:59 ? 00:00:00 runsv bird 0 S root 25129 25042 0 80 0 - 18 poll_s 18:59 ? 00:00:00 runsv bird6 4 S root 25311 25128 0 80 0 - 27 poll_s 18:59 ? 00:00:00 bird -R -s /var/run/calico/bird.ctl -d -c /etc/calico/confd/config/bird.cfg 0 S root 25312 25129 0 80 0 - 27 poll_s 18:59 ? 00:00:00 bird6 -R -s /var/run/calico/bird6.ctl -d -c /etc/calico/confd/config/bird6.cfg 0 S root 33417 47750 0 80 0 - 1726 pipe_w 19:06 pts/2 00:00:00 grep --color bird
[root@bogon ~]# cat /etc/calico/confd/config/bird.cfg cat: /etc/calico/confd/config/bird.cfg: No such file or directory [root@bogon ~]# ls /etc/calico/confd/config/ ls: cannot access /etc/calico/confd/config/: No such file or directory [root@bogon ~]# docker exec -it 6d6ed3a4025c bash OCI runtime exec failed: exec failed: container_linux.go:380: starting container process caused: exec: "bash": executable file not found in $PATH: unknown [root@bogon ~]# docker exec -it 6d6ed3a4025c ls bin lib proc sys dev licenses root tmp etc media run usr home mnt sbin var included-source opt srv [root@bogon ~]# docker exec -it 6d6ed3a4025c ls /etc/calico/confd/config/ bird.cfg bird6_aggr.cfg bird_aggr.cfg bird6.cfg bird6_ipam.cfg bird_ipam.cfg [root@bogon ~]# docker exec -it 6d6ed3a4025c cat /etc/calico/confd/config/bird.cfg function apply_communities () { } # Generated by confd include "bird_aggr.cfg"; include "bird_ipam.cfg"; router id 10.10.16.81; # Configure synchronization between routing tables and kernel. protocol kernel { learn; # Learn all alien routes from the kernel persist; # Don't remove routes on bird shutdown scan time 2; # Scan kernel routing table every 2 seconds import all; export filter calico_kernel_programming; # Default is export none graceful restart; # Turn on graceful restart to reduce potential flaps in # routes when reloading BIRD configuration. With a full # automatic mesh, there is no way to prevent BGP from # flapping since multiple nodes update their BGP # configuration at the same time, GR is not guaranteed to # work correctly in this scenario. merge paths on; # Allow export multipath routes (ECMP) } # Watch interface up/down events. protocol device { debug { states }; scan time 2; # Scan interfaces every 2 seconds } protocol direct { debug { states }; interface -"cali*", -"kube-ipvs*", "*"; # Exclude cali* and kube-ipvs* but # include everything else. In # IPVS-mode, kube-proxy creates a # kube-ipvs0 interface. We exclude # kube-ipvs0 because this interface # gets an address for every in use # cluster IP. We use static routes # for when we legitimately want to # export cluster IPs. } # Template for all BGP clients template bgp bgp_template { debug { states }; description "Connection to BGP peer"; local as 64512; multihop; gateway recursive; # This should be the default, but just in case. import all; # Import all routes, since we don't know what the upstream # topology is and therefore have to trust the ToR/RR. export filter calico_export_to_bgp_peers; # Only want to export routes for workloads. add paths on; graceful restart; # See comment in kernel section about graceful restart. connect delay time 2; connect retry time 5; error wait time 5,30; } # ------------- Node-to-node mesh ------------- # For peer /host/bogon/ip_addr_v4 # Skipping ourselves (10.10.16.81) # For peer /host/cloud/ip_addr_v4 protocol bgp Mesh_10_10_16_47 from bgp_template { neighbor 10.10.16.47 as 64512; source address 10.10.16.81; # The local address we use for the TCP connection } # For peer /host/ubuntu/ip_addr_v4 protocol bgp Mesh_10_10_16_82 from bgp_template { neighbor 10.10.16.82 as 64512; source address 10.10.16.81; # The local address we use for the TCP connection passive on; # Mesh is unidirectional, peer will connect to us. } # ------------- Global peers ------------- # No global peers configured. # ------------- Node-specific peers ------------- # No node-specific peers configured.
[root@bogon ~]# vtysh Hello, this is FRRouting (version 7.3-MyOwnFRRVersion). Copyright 1996-2005 Kunihiro Ishiguro, et al. bogon# sh ip route Codes: K - kernel route, C - connected, S - static, R - RIP, O - OSPF, I - IS-IS, B - BGP, E - EIGRP, N - NHRP, T - Table, v - VNC, V - VNC-Direct, A - Babel, D - SHARP, F - PBR, f - OpenFabric, > - selected route, * - FIB route, q - queued route, r - rejected route K>* 0.0.0.0/0 [0/0] via 10.10.16.254, enahisic2i0, 43w3d04h K>* 10.2.0.1/32 [0/0] is directly connected, v-cali-peer, 01w6d07h C * 10.10.16.0/24 is directly connected, enahisic2i0, 05w1d02h C>* 10.10.16.0/24 is directly connected, enahisic2i0, 43w3d04h C>* 10.10.34.0/24 is directly connected, enahisic2i2, 01w3d02h C>* 10.10.102.0/24 is directly connected, enahisic2i1, 44w3d01h K>* 10.244.2.0/24 [0/0] via 10.10.16.47, tunl0 onlink, 00:11:57 K>* 10.244.29.0/26 [0/0] unreachable (blackhole), 3d01h03m C>* 10.244.29.0/32 is directly connected, tunl0, 3d01h07m K>* 10.244.29.1/32 [0/0] is directly connected, cali2e486421e22, 04:30:04 K>* 10.244.29.2/32 [0/0] is directly connected, calib87dcb53769, 03:48:30 K>* 10.244.41.0/26 [0/0] via 10.10.16.47, tunl0 onlink, 00:11:57 K>* 10.244.243.192/26 [0/0] via 10.10.16.82, tunl0 onlink, 00:11:57 C>* 14.14.18.0/24 is directly connected, enahisic2i3.310, 45w6d08h C>* 172.16.100.0/24 is directly connected, brqf1411bad-10, 48w4d09h C>* 172.168.104.0/24 is directly connected, enah2i3.1022, 47w6d22h C>* 192.168.1.3/32 is directly connected, enahisic2i1, 48w1d00h C>* 192.168.1.5/32 is directly connected, enahisic2i1, 48w1d01h C>* 192.168.33.0/24 is directly connected, enahisic2i1, 47w4d09h bogon# sh bgp route % This command is applicable only for unicast ipv4|ipv6 bogon# show bgp neighbor % BGP instance not found bogon# quit You have mail in /var/spool/mail/root [root@bogon ~]# docker exec -it 6d6ed3a4025c "docker exec" requires at least 2 arguments. See 'docker exec --help'. Usage: docker exec [OPTIONS] CONTAINER COMMAND [ARG...] Run a command in a running container
6d6ed3a4025c 容器和host共享net naespace
在Calico IPIP模式下，其实有一个配置项(CALICO_IPV4POOL_IPIP)，可以让calico只有在跨子网的情况下才使用IPIP协议，同一子网中的POD直接通过路由信息直连。该参数为如下，添加到calico-node的环境变量中即可。
- name: CALICO_IPV4POOL_IPIP
value: CrossSubnet
如果是已经部署了calico之后再修改这个参数，还需要修改ippool中的ipipMode参数。
[root@master01 ~]# kubectl get ippool default-ipv4-ippool -oyaml
apiVersion: crd.projectcalico.org/v1
kind: IPPool
metadata:
annotations:
projectcalico.org/metadata: '{"uid":"3797d863-3b04-45cd-b9ba-ac1980a21520","creationTimestamp":"2021-03-30T04:16:40Z"}'
creationTimestamp: "2021-03-30T04:16:40Z"
generation: 1
name: default-ipv4-ippool
resourceVersion: "732"
selfLink: /apis/crd.projectcalico.org/v1/ippools/default-ipv4-ippool
uid: 4c7acedb-0026-4062-994d-e82d261f29cc
spec:
blockSize: 26
cidr: 172.16.0.0/16
ipipMode: CrossSubnet
calio arp 代理
root@cloud:~# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever 2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000 link/ipip 0.0.0.0 brd 0.0.0.0 4: eth0@if39: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1440 qdisc noqueue state UP group default link/ether 72:98:01:99:c5:5a brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 10.244.41.2/32 brd 10.244.41.2 scope global eth0 valid_lft forever preferred_lft forever root@cloud:~# route -n Kernel IP routing table Destination Gateway Genmask Flags Metric Ref Use Iface 0.0.0.0 169.254.1.1 0.0.0.0 UG 0 0 0 eth0 169.254.1.1 0.0.0.0 255.255.255.255 UH 0 0 0 eth0 root@cloud:~# ping 8.8.8.8 PING 8.8.8.8 (8.8.8.8) 56(84) bytes of data. 64 bytes from 8.8.8.8: icmp_seq=1 ttl=105 time=11.8 ms 64 bytes from 8.8.8.8: icmp_seq=2 ttl=105 time=11.7 ms ^C --- 8.8.8.8 ping statistics --- 2 packets transmitted, 2 received, 0% packet loss, time 1001ms rtt min/avg/max/mdev = 11.716/11.797/11.878/0.081 ms root@cloud:~# ip n 10.10.16.47 dev eth0 lladdr ee:ee:ee:ee:ee:ee STALE 169.254.1.1 dev eth0 lladdr ee:ee:ee:ee:ee:ee DELAY root@cloud:~#
169.254.1.1ip 配置在哪个设备上
root@cloud:~# ip a | grep 169.254.1.1 root@cloud:~# ip a | grep 39
root@cloud:~# ip a | grep 169.254.1.1 root@cloud:~# ip a | grep 39 39: cali8763f2b9f05@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1440 qdisc noqueue state UP group default root@cloud:~# ip s sh | grep 39 Error: argument "sh" is wrong: unknown root@cloud:~# ip a sh cali8763f2b9f05 39: cali8763f2b9f05@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1440 qdisc noqueue state UP group default link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 2 inet6 fe80::ecee:eeff:feee:eeee/64 scope link valid_lft forever preferred_lft forever
报文会发送到这个 interface，这个 interface 有一个随机分配的 MAC 地址，但是没有
IP地址，接收到想要 169.254.1.1 MAC 地址的 ARP 请求报文，它会怎么做呢？这个又不是它的 IP，而且它又没有和任何的 bridge 相连可以广播 ARP 报文。
只能抓包看看了，记住要先删除容器中 169.254.1.1 对应的 ARP 表项（使用 ip neigh del 命令），然后运行 ping 的时候在主机上抓包：
root@cloud:~# ip n 10.10.16.47 dev eth0 lladdr ee:ee:ee:ee:ee:ee STALE 169.254.1.1 dev eth0 lladdr ee:ee:ee:ee:ee:ee DELAY root@cloud:~# ip n del 169.254.1.1 dev eth0 lladdr ee:ee:ee:ee:ee:ee root@cloud:~# ip n 10.10.16.47 dev eth0 lladdr ee:ee:ee:ee:ee:ee STALE root@cloud:~#
root@cloud:~# tcpdump -i cali8763f2b9f05 arp -eennvv tcpdump: listening on cali8763f2b9f05, link-type EN10MB (Ethernet), capture size 262144 bytes 19:31:20.393534 72:98:01:99:c5:5a > ff:ff:ff:ff:ff:ff, ethertype ARP (0x0806), length 42: Ethernet (len 6), IPv4 (len 4), Request who-has 169.254.1.1 tell 10.244.41.2, length 28 19:31:20.393562 ee:ee:ee:ee:ee:ee > 72:98:01:99:c5:5a, ethertype ARP (0x0806), length 42: Ethernet (len 6), IPv4 (len 4), Reply 169.254.1.1 is-at ee:ee:ee:ee:ee:ee, length 28 19:31:25.431286 ee:ee:ee:ee:ee:ee > 72:98:01:99:c5:5a, ethertype ARP (0x0806), length 42: Ethernet (len 6), IPv4 (len 4), Request who-has 10.244.41.2 tell 10.10.16.47, length 28 19:31:25.431445 72:98:01:99:c5:5a > ee:ee:ee:ee:ee:ee, ethertype ARP (0x0806), length 42: Ethernet (len 6), IPv4 (len 4), Reply 10.244.41.2 is-at 72:98:01:99:c5:5a, length 28
从前面两个报文可以看到，接收到 ARP 请求后，它直接进行了应答，应答报文中 MAC 地址是
ee:ee:ee:ee:ee:ee
，这正是该 interface 自己的 MAC 地址。换句话说，它把自己的 MAC 地址作为应答返回给容器。容器的后续报文 IP 地址还是目的容器，但是 MAC 地址就变成了主机上该 interface 的地址，也就是说所有的报文都会发给主机，然后主机根据 IP 地址进行转发。
主机这个 interface 不管 ARP 请求的内容，直接用自己的 MAC 地址作为应答的行为被成为 ARP proxy，是 calico 开启的，可以通过下面的命令确认：
root@cloud:~# cat /proc/sys/net/ipv4/conf/cali8763f2b9f05/proxy_arp 1 root@cloud:~#
的来说，可以认为 calico 把主机作为容器的默认网关来使用，所有的报文发到主机，然后主机根据路由表进行转发。和经典的网络架构不同的是，calico 并没有给默认网络配置一个 IP 地址（这样每个网络都会额外消耗一个 IP 资源，而且主机上也会增加对应的 IP 地址和路由信息），而是通过 arp proxy 和修改容器路由表来实现。
主机的 interface 接收到报文之后，下面的事情就容易理解了，所有的报文会根据路由表来走：
root@ubuntu:~/etcd-v3.5.0-linux-arm64# route -n Kernel IP routing table Destination Gateway Genmask Flags Metric Ref Use Iface 0.0.0.0 10.10.16.254 0.0.0.0 UG 0 0 0 enahisic2i0 10.10.16.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i0 10.244.2.0 10.10.16.47 255.255.255.0 UG 0 0 0 tunl0 10.244.29.0 10.10.16.81 255.255.255.192 UG 0 0 0 tunl0 10.244.41.0 10.10.16.47 255.255.255.192 UG 0 0 0 tunl0 10.244.243.192 0.0.0.0 255.255.255.192 U 0 0 0 * 10.244.243.194 0.0.0.0 255.255.255.255 UH 0 0 0 cali0d27bc8b0f7 10.244.243.196 0.0.0.0 255.255.255.255 UH 0 0 0 cali6c2a86772cc 172.17.0.0 0.0.0.0 255.255.0.0 U 0 0 0 docker0 192.168.122.0 0.0.0.0 255.255.255.0 U 0 0 0 virbr0
root@ubuntu:~# tcpdump -i cali6c2a86772cc icmp -eennvv tcpdump: listening on cali6c2a86772cc, link-type EN10MB (Ethernet), capture size 262144 bytes 19:34:56.837542 ee:ee:ee:ee:ee:ee > 06:de:36:95:e9:7a, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 62, id 51016, offset 0, flags [DF], proto ICMP (1), length 84) 10.244.41.2 > 10.244.243.196: ICMP echo request, id 30590, seq 1, length 64 19:34:56.837598 06:de:36:95:e9:7a > ee:ee:ee:ee:ee:ee, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 33480, offset 0, flags [none], proto ICMP (1), length 84) 10.244.243.196 > 10.244.41.2: ICMP echo reply, id 30590, seq 1, length 64 19:34:57.848663 ee:ee:ee:ee:ee:ee > 06:de:36:95:e9:7a, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 62, id 51200, offset 0, flags [DF], proto ICMP (1), length 84) 10.244.41.2 > 10.244.243.196: ICMP echo request, id 30590, seq 2, length 64 19:34:57.848712 06:de:36:95:e9:7a > ee:ee:ee:ee:ee:ee, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 33514, offset 0, flags [none], proto ICMP (1), length 84) 10.244.243.196 > 10.244.41.2: ICMP echo reply, id 30590, seq 2, length 64 19:34:58.872585 ee:ee:ee:ee:ee:ee > 06:de:36:95:e9:7a, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 62, id 51385, offset 0, flags [DF], proto ICMP (1), length 84) 10.244.41.2 > 10.244.243.196: ICMP echo request, id 30590, seq 3, length 64 19:34:58.872621 06:de:36:95:e9:7a > ee:ee:ee:ee:ee:ee, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 33734, offset 0, flags [none], proto ICMP (1), length 84) 10.244.243.196 > 10.244.41.2: ICMP echo reply, id 30590, seq 3, length 64
tcpdump: listening on tunl0, link-type RAW (Raw IP), capture size 262144 bytes 19:35:39.737716 ip: (tos 0x0, ttl 63, id 53135, offset 0, flags [DF], proto ICMP (1), length 84) 10.244.41.2 > 10.244.243.196: ICMP echo request, id 31435, seq 1, length 64 19:35:39.737840 ip: (tos 0x0, ttl 63, id 34986, offset 0, flags [none], proto ICMP (1), length 84) 10.244.243.196 > 10.244.41.2: ICMP echo reply, id 31435, seq 1, length 64 19:35:40.760635 ip: (tos 0x0, ttl 63, id 53351, offset 0, flags [DF], proto ICMP (1), length 84) 10.244.41.2 > 10.244.243.196: ICMP echo request, id 31435, seq 2, length 64 19:35:40.760689 ip: (tos 0x0, ttl 63, id 35170, offset 0, flags [none], proto ICMP (1), length 84) 10.244.243.196 > 10.244.41.2: ICMP echo reply, id 31435, seq 2, length 64
root@ubuntu:~# ip a sh tunl0 没有mac地址 9: tunl0@NONE: <NOARP,UP,LOWER_UP> mtu 1440 qdisc noqueue state UNKNOWN group default qlen 1000 link/ipip 0.0.0.0 brd 0.0.0.0 inet 10.244.243.192/32 scope global tunl0 valid_lft forever preferred_lft forever root@ubuntu:~#
ping 10.244.29.2
root@cloud:~# ping 10.244.29.2 PING 10.244.29.2 (10.244.29.2) 56(84) bytes of data. 64 bytes from 10.244.29.2: icmp_seq=1 ttl=62 time=0.376 ms 64 bytes from 10.244.29.2: icmp_seq=2 ttl=62 time=0.255 ms ^C --- 10.244.29.2 ping statistics --- 2 packets transmitted, 2 received, 0% packet loss, time 1027ms rtt min/avg/max/mdev = 0.255/0.315/0.376/0.063 ms root@cloud:~# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever 2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000 link/ipip 0.0.0.0 brd 0.0.0.0 4: eth0@if39: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1440 qdisc noqueue state UP group default link/ether 72:98:01:99:c5:5a brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 10.244.41.2/32 brd 10.244.41.2 scope global eth0 valid_lft forever preferred_lft forever root@cloud:~#
cali6c2a86772cc的mac也是ee:ee:ee:ee:ee:ee
root@ubuntu:~# ip a sh cali6c2a86772cc 15: cali6c2a86772cc@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1440 qdisc noqueue state UP group default link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet6 fe80::ecee:eeff:feee:eeee/64 scope link valid_lft forever preferred_lft forever root@ubuntu:~#
[root@bogon ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever 2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000 link/ipip 0.0.0.0 brd 0.0.0.0 4: eth0@if101: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1440 qdisc noqueue state UP group default link/ether da:b3:97:64:94:03 brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 10.244.29.2/32 brd 10.244.29.2 scope global eth0 valid_lft forever preferred_lft forever [root@bogon ~]#
da:b3:97:64:94:03是pod的mac
[root@bogon ~]# ip a | grep tun 99: tunl0@NONE: <NOARP,UP,LOWER_UP> mtu 1440 qdisc noqueue state UNKNOWN group default qlen 1000 inet 10.244.29.0/32 scope global tunl0 You have new mail in /var/spool/mail/root [root@bogon ~]# tcpdump -i calib87dcb53769 icmp -ennvv tcpdump: listening on calib87dcb53769, link-type EN10MB (Ethernet), capture size 262144 bytes 19:44:49.021297 ee:ee:ee:ee:ee:ee > da:b3:97:64:94:03, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 62, id 45545, offset 0, flags [DF], proto ICMP (1), length 84) 10.244.41.2 > 10.244.29.2: ICMP echo request, id 42350, seq 1, length 64 19:44:49.021340 da:b3:97:64:94:03 > ee:ee:ee:ee:ee:ee, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 15732, offset 0, flags [none], proto ICMP (1), length 84) 10.244.29.2 > 10.244.41.2: ICMP echo reply, id 42350, seq 1, length 64 19:44:50.048274 ee:ee:ee:ee:ee:ee > da:b3:97:64:94:03, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 62, id 45706, offset 0, flags [DF], proto ICMP (1), length 84) 10.244.41.2 > 10.244.29.2: ICMP echo request, id 42350, seq 2, length 64 19:44:50.048296 da:b3:97:64:94:03 > ee:ee:ee:ee:ee:ee, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 15764, offset 0, flags [none], proto ICMP (1), length 84) 10.244.29.2 > 10.244.41.2: ICMP echo reply, id 42350, seq 2, length 64 ^C 4 packets captured 4 packets received by filter 0 packets dropped by kernel You have mail in /var/spool/mail/root [root@bogon ~]# ip a | grep da:b3:97:64:94:03 [root@bogon ~]# ip n | grep da:b3:97:64:94:03 10.244.29.2 dev calib87dcb53769 lladdr da:b3:97:64:94:03 STALE ----从这里 [root@bogon ~]# route -n Kernel IP routing table Destination Gateway Genmask Flags Metric Ref Use Iface 0.0.0.0 10.10.16.254 0.0.0.0 UG 0 0 0 enahisic2i0 10.2.0.1 0.0.0.0 255.255.255.255 UH 0 0 0 v-cali-peer 10.10.16.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i0 10.10.34.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i2 10.10.102.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i1 10.244.2.0 10.10.16.47 255.255.255.0 UG 0 0 0 tunl0 10.244.29.0 0.0.0.0 255.255.255.192 U 0 0 0 * 10.244.29.1 0.0.0.0 255.255.255.255 UH 0 0 0 cali2e486421e22 10.244.29.2 0.0.0.0 255.255.255.255 UH 0 0 0 calib87dcb53769 10.244.41.0 10.10.16.47 255.255.255.192 UG 0 0 0 tunl0 10.244.243.192 10.10.16.82 255.255.255.192 UG 0 0 0 tunl0 14.14.18.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i3.310 172.16.100.0 0.0.0.0 255.255.255.0 U 0 0 0 brqf1411bad-10 172.17.0.0 0.0.0.0 255.255.0.0 U 0 0 0 docker0 172.168.104.0 0.0.0.0 255.255.255.0 U 0 0 0 enah2i3.1022 192.168.33.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i1 192.168.122.0 0.0.0.0 255.255.255.0 U 0 0 0 virbr0 [root@bogon ~]#
nodeport
root@ubuntu:~# kubectl apply -f web-ngx-svc.yml service/nodeport-svc created root@ubuntu:~# kubectl get svc NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 3d nodeport-svc NodePort 10.107.44.16 <none> 3000:30090/TCP 3s root@ubuntu:~# cat web-ngx-svc.yml apiVersion: v1 kind: Service metadata: name: nodeport-svc spec: type: NodePort selector: app: web-nginx ports: - protocol: TCP port: 3000 targetPort: 80 nodePort: 30090
访问10.10.16.82:30090
root@ubuntu:~# ip a | grep 10.244.243.192 inet 10.244.243.192/32 scope global tunl0 root@ubuntu:~# ip a sh tun10 Device "tun10" does not exist. root@ubuntu:~# ip a sh tunl0 9: tunl0@NONE: <NOARP,UP,LOWER_UP> mtu 1440 qdisc noqueue state UNKNOWN group default qlen 1000 link/ipip 0.0.0.0 brd 0.0.0.0 inet 10.244.243.192/32 scope global tunl0 valid_lft forever preferred_lft forever root@ubuntu:~#
bogon抓包
原地址是 tun10的ip
[root@bogon ~]# tcpdump -i enahisic2i0 "ip proto 4" and host 10.10.16.82 -ennvv tcpdump: listening on enahisic2i0, link-type EN10MB (Ethernet), capture size 262144 bytes 20:02:11.894506 48:57:02:64:e7:ab > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 94: (tos 0x0, ttl 56, id 44165, offset 0, flags [DF], proto IPIP (4), length 80) 10.10.16.82 > 10.10.16.81: (tos 0x0, ttl 56, id 36209, offset 0, flags [DF], proto TCP (6), length 60) 10.244.243.192.36979 > 10.244.29.2.80: Flags [S], cksum 0x98e7 (correct), seq 2450954292, win 64240, options [mss 1460,nop,wscale 8,sackOK,TS val 252756829 ecr 0], length 0 20:02:11.894686 48:57:02:64:ea:1b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 94: (tos 0x0, ttl 63, id 62675, offset 0, flags [DF], proto IPIP (4), length 80) 10.10.16.81 > 10.10.16.82: (tos 0x0, ttl 63, id 0, offset 0, flags [DF], proto TCP (6), length 60) 10.244.29.2.80 > 10.244.243.192.36979: Flags [S.], cksum 0x822b (correct), seq 1370978234, ack 2450954293, win 27760, options [mss 1400,sackOK,TS val 1118476620 ecr 252756829,nop,wscale 7], length 0 20:02:11.896175 48:57:02:64:e7:ab > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 86: (tos 0x0, ttl 56, id 44166, offset 0, flags [DF], proto IPIP (4), length 72) 10.10.16.82 > 10.10.16.81: (tos 0x0, ttl 56, id 36214, offset 0, flags [DF], proto TCP (6), length 52) 10.244.243.192.36979 > 10.244.29.2.80: Flags [.], cksum 0x1b26 (correct), seq 1, ack 1, win 515, options [nop,nop,TS val 252756832 ecr 1118476620], length 0 20:02:11.897450 48:57:02:64:e7:ab > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 603: (tos 0x0, ttl 56, id 44167, offset 0, flags [DF], proto IPIP (4), length 589) 10.10.16.82 > 10.10.16.81: (tos 0x0, ttl 56, id 36215, offset 0, flags [DF], proto TCP (6), length 569) 10.244.243.192.36979 > 10.244.29.2.80: Flags [P.], cksum 0x17d3 (correct), seq 1:518, ack 1, win 515, options [nop,nop,TS val 252756832 ecr 1118476620], length 517: HTTP 20:02:11.897582 48:57:02:64:ea:1b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 86: (tos 0x0, ttl 63, id 62676, offset 0, flags [DF], proto IPIP (4), length 72) 10.10.16.81 > 10.10.16.82: (tos 0x0, ttl 63, id 62171, offset 0, flags [DF], proto TCP (6), length 52) 10.244.29.2.80 > 10.244.243.192.36979: Flags [.], cksum 0x1a3f (correct), seq 1, ack 518, win 226, options [nop,nop,TS val 1118476623 ecr 252756832], length 0 20:02:11.897738 48:57:02:64:ea:1b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 395: (tos 0x0, ttl 63, id 62677, offset 0, flags [DF], proto IPIP (4), length 381) 10.10.16.81 > 10.10.16.82: (tos 0x0, ttl 63, id 62172, offset 0, flags [DF], proto TCP (6), length 361) 10.244.29.2.80 > 10.244.243.192.36979: Flags [P.], cksum 0x7306 (correct), seq 1:310, ack 518, win 226, options [nop,nop,TS val 1118476623 ecr 252756832], length 309: HTTP, length: 309 HTTP/1.1 400 Bad Request Server: nginx/1.21.0 Date: Mon, 21 Jun 2021 12:02:11 GMT Content-Type: text/html Content-Length: 157 Connection: close <html> <head><title>400 Bad Request</title></head> <body> <center><h1>400 Bad Request</h1></center> <hr><center>nginx/1.21.0</center> </body> </html> 20:02:11.897796 48:57:02:64:ea:1b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 86: (tos 0x0, ttl 63, id 62678, offset 0, flags [DF], proto IPIP (4), length 72) 10.10.16.81 > 10.10.16.82: (tos 0x0, ttl 63, id 62173, offset 0, flags [DF], proto TCP (6), length 52) 10.244.29.2.80 > 10.244.243.192.36979: Flags [F.], cksum 0x1909 (correct), seq 310, ack 518, win 226, options [nop,nop,TS val 1118476623 ecr 252756832], length 0 20:02:11.899987 48:57:02:64:e7:ab > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 86: (tos 0x0, ttl 56, id 44168, offset 0, flags [DF], proto IPIP (4), length 72) 10.10.16.82 > 10.10.16.81: (tos 0x0, ttl 56, id 36217, offset 0, flags [DF], proto TCP (6), length 52) 10.244.243.192.36979 > 10.244.29.2.80: Flags [.], cksum 0x17e7 (correct), seq 518, ack 311, win 513, options [nop,nop,TS val 252756835 ecr 1118476623], length 0 20:02:11.900576 48:57:02:64:e7:ab > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 93: (tos 0x0, ttl 56, id 44169, offset 0, flags [DF], proto IPIP (4), length 79) 10.10.16.82 > 10.10.16.81: (tos 0x0, ttl 56, id 36218, offset 0, flags [DF], proto TCP (6), length 59) 10.244.243.192.36979 > 10.244.29.2.80: Flags [P.], cksum 0xb9d1 (correct), seq 518:525, ack 311, win 513, options [nop,nop,TS val 252756836 ecr 1118476623], length 7: HTTP 20:02:11.900672 48:57:02:64:e7:ab > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 86: (tos 0x0, ttl 56, id 44170, offset 0, flags [DF], proto IPIP (4), length 72) 10.10.16.82 > 10.10.16.81: (tos 0x0, ttl 56, id 36219, offset 0, flags [DF], proto TCP (6), length 52) 10.244.243.192.36979 > 10.244.29.2.80: Flags [F.], cksum 0x17de (correct), seq 525, ack 311, win 513, options [nop,nop,TS val 252756836 ecr 1118476623], length 0 20:02:11.900787 48:57:02:64:ea:1b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 86: (tos 0x0, ttl 63, id 62679, offset 0, flags [DF], proto IPIP (4), length 72) 10.10.16.81 > 10.10.16.82: (tos 0x0, ttl 63, id 62174, offset 0, flags [DF], proto TCP (6), length 52) 10.244.29.2.80 > 10.244.243.192.36979: Flags [.], cksum 0x18fa (correct), seq 311, ack 526, win 226, options [nop,nop,TS val 1118476626 ecr 252756836], length 0 20:02:11.901507 48:57:02:64:e7:ab > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 94: (tos 0x0, ttl 56, id 44171, offset 0, flags [DF], proto IPIP (4), length 80) 10.10.16.82 > 10.10.16.81: (tos 0x0, ttl 56, id 36223, offset 0, flags [DF], proto TCP (6), length 60) 10.244.243.192.21048 > 10.244.29.2.80: Flags [S], cksum 0x5e5d (correct), seq 921463837, win 64240, options [mss 1460,nop,wscale 8,sackOK,TS val 252756836 ecr 0], length 0 20:02:11.901642 48:57:02:64:ea:1b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 94: (tos 0x0, ttl 63, id 62680, offset 0, flags [DF], proto IPIP (4), length 80) 10.10.16.81 > 10.10.16.82: (tos 0x0, ttl 63, id 0, offset 0, flags [DF], proto TCP (6), length 60) 10.244.29.2.80 > 10.244.243.192.21048: Flags [S.], cksum 0x8530 (correct), seq 4043545303, ack 921463838, win 27760, options [mss 1400,sackOK,TS val 1118476627 ecr 252756836,nop,wscale 7], length 0 20:02:11.906382 48:57:02:64:e7:ab > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 86: (tos 0x0, ttl 56, id 44172, offset 0, flags [DF], proto IPIP (4), length 72) 10.10.16.82 > 10.10.16.81: (tos 0x0, ttl 56, id 36226, offset 0, flags [DF], proto TCP (6), length 52) 10.244.243.192.21048 > 10.244.29.2.80: Flags [.], cksum 0x1e28 (correct), seq 1, ack 1, win 515, options [nop,nop,TS val 252756842 ecr 1118476627], length 0 20:02:11.907753 48:57:02:64:e7:ab > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 603: (tos 0x0, ttl 56, id 44173, offset 0, flags [DF], proto IPIP (4), length 589) 10.10.16.82 > 10.10.16.81: (tos 0x0, ttl 56, id 36228, offset 0, flags [DF], proto TCP (6), length 569) 10.244.243.192.21048 > 10.244.29.2.80: Flags [P.], cksum 0x1483 (correct), seq 1:518, ack 1, win 515, options [nop,nop,TS val 252756842 ecr 1118476627], length 517: HTTP 20:02:11.907860 48:57:02:64:ea:1b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 86: (tos 0x0, ttl 63, id 62681, offset 0, flags [DF], proto IPIP (4), length 72) 10.10.16.81 > 10.10.16.82: (tos 0x0, ttl 63, id 8121, offset 0, flags [DF], proto TCP (6), length 52) 10.244.29.2.80 > 10.244.243.192.21048: Flags [.], cksum 0x1d3e (correct), seq 1, ack 518, win 226, options [nop,nop,TS val 1118476633 ecr 252756842], length 0 20:02:11.907903 48:57:02:64:ea:1b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 395: (tos 0x0, ttl 63, id 62682, offset 0, flags [DF], proto IPIP (4), length 381) 10.10.16.81 > 10.10.16.82: (tos 0x0, ttl 63, id 8122, offset 0, flags [DF], proto TCP (6), length 361) 10.244.29.2.80 > 10.244.243.192.21048: Flags [P.], cksum 0x7605 (correct), seq 1:310, ack 518, win 226, options [nop,nop,TS val 1118476633 ecr 252756842], length 309: HTTP, length: 309 HTTP/1.1 400 Bad Request Server: nginx/1.21.0 Date: Mon, 21 Jun 2021 12:02:11 GMT Content-Type: text/html Content-Length: 157 Connection: close <html> <head><title>400 Bad Request</title></head> <body> <center><h1>400 Bad Request</h1></center> <hr><center>nginx/1.21.0</center> </body> </html> 20:02:11.907947 48:57:02:64:ea:1b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 86: (tos 0x0, ttl 63, id 62683, offset 0, flags [DF], proto IPIP (4), length 72) 10.10.16.81 > 10.10.16.82: (tos 0x0, ttl 63, id 8123, offset 0, flags [DF], proto TCP (6), length 52) 10.244.29.2.80 > 10.244.243.192.21048: Flags [F.], cksum 0x1c08 (correct), seq 310, ack 518, win 226, options [nop,nop,TS val 1118476633 ecr 252756842], length 0 20:02:11.911831 48:57:02:64:e7:ab > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 86: (tos 0x0, ttl 56, id 44174, offset 0, flags [DF], proto IPIP (4), length 72) 10.10.16.82 > 10.10.16.81: (tos 0x0, ttl 56, id 36231, offset 0, flags [DF], proto TCP (6), length 52) 10.244.243.192.21048 > 10.244.29.2.80: Flags [.], cksum 0x1ae4 (correct), seq 518, ack 311, win 513, options [nop,nop,TS val 252756847 ecr 1118476633], length 0 20:02:11.911953 48:57:02:64:e7:ab > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 93: (tos 0x0, ttl 56, id 44175, offset 0, flags [DF], proto IPIP (4), length 79) 10.10.16.82 > 10.10.16.81: (tos 0x0, ttl 56, id 36232, offset 0, flags [DF], proto TCP (6), length 59) 10.244.243.192.21048 > 10.244.29.2.80: Flags [P.], cksum 0xbcce (correct), seq 518:525, ack 311, win 513, options [nop,nop,TS val 252756848 ecr 1118476633], length 7: HTTP 20:02:11.912147 48:57:02:64:e7:ab > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 86: (tos 0x0, ttl 56, id 44176, offset 0, flags [DF], proto IPIP (4), length 72) 10.10.16.82 > 10.10.16.81: (tos 0x0, ttl 56, id 36233, offset 0, flags [DF], proto TCP (6), length 52) 10.244.243.192.21048 > 10.244.29.2.80: Flags [F.], cksum 0x1adb (correct), seq 525, ack 311, win 513, options [nop,nop,TS val 252756848 ecr 1118476633], length 0 20:02:11.912276 48:57:02:64:ea:1b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 86: (tos 0x0, ttl 63, id 62684, offset 0, flags [DF], proto IPIP (4), length 72) 10.10.16.81 > 10.10.16.82: (tos 0x0, ttl 63, id 8124, offset 0, flags [DF], proto TCP (6), length 52) 10.244.29.2.80 > 10.244.243.192.21048: Flags [.], cksum 0x1bf6 (correct), seq 311, ack 526, win 226, options [nop,nop,TS val 1118476637 ecr 252756848], length 0
pod 抓包
[root@bogon ~]# tcpdump -i eth0 tcp and port 80 -env tcpdump: listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes 20:02:11.894594 ee:ee:ee:ee:ee:ee > da:b3:97:64:94:03, ethertype IPv4 (0x0800), length 74: (tos 0x0, ttl 55, id 36209, offset 0, flags [DF], proto TCP (6), length 60) 10.244.243.192.36979 > 10.244.29.2.http: Flags [S], cksum 0x98e7 (correct), seq 2450954292, win 64240, options [mss 1460,nop,wscale 8,sackOK,TS val 252756829 ecr 0], length 0 20:02:11.894637 da:b3:97:64:94:03 > ee:ee:ee:ee:ee:ee, ethertype IPv4 (0x0800), length 74: (tos 0x0, ttl 64, id 0, offset 0, flags [DF], proto TCP (6), length 60) 10.244.29.2.http > 10.244.243.192.36979: Flags [S.], cksum 0x26d9 (incorrect -> 0x822b), seq 1370978234, ack 2450954293, win 27760, options [mss 1400,sackOK,TS val 1118476620 ecr 252756829,nop,wscale 7], length 0 20:02:11.896260 ee:ee:ee:ee:ee:ee > da:b3:97:64:94:03, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 55, id 36214, offset 0, flags [DF], proto TCP (6), length 52) 10.244.243.192.36979 > 10.244.29.2.http: Flags [.], cksum 0x1b26 (correct), ack 1, win 515, options [nop,nop,TS val 252756832 ecr 1118476620], length 0 20:02:11.897541 ee:ee:ee:ee:ee:ee > da:b3:97:64:94:03, ethertype IPv4 (0x0800), length 583: (tos 0x0, ttl 55, id 36215, offset 0, flags [DF], proto TCP (6), length 569) 10.244.243.192.36979 > 10.244.29.2.http: Flags [P.], cksum 0x17d3 (correct), seq 1:518, ack 1, win 515, options [nop,nop,TS val 252756832 ecr 1118476620], length 517: HTTP 20:02:11.897561 da:b3:97:64:94:03 > ee:ee:ee:ee:ee:ee, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 64, id 62171, offset 0, flags [DF], proto TCP (6), length 52) 10.244.29.2.http > 10.244.243.192.36979: Flags [.], cksum 0x26d1 (incorrect -> 0x1a3f), ack 518, win 226, options [nop,nop,TS val 1118476623 ecr 252756832], length 0 20:02:11.897656 da:b3:97:64:94:03 > ee:ee:ee:ee:ee:ee, ethertype IPv4 (0x0800), length 375: (tos 0x0, ttl 64, id 62172, offset 0, flags [DF], proto TCP (6), length 361) 10.244.29.2.http > 10.244.243.192.36979: Flags [P.], cksum 0x2806 (incorrect -> 0x7306), seq 1:310, ack 518, win 226, options [nop,nop,TS val 1118476623 ecr 252756832], length 309: HTTP, length: 309 HTTP/1.1 400 Bad Request Server: nginx/1.21.0 Date: Mon, 21 Jun 2021 12:02:11 GMT Content-Type: text/html Content-Length: 157 Connection: close <html> <head><title>400 Bad Request</title></head> <body> <center><h1>400 Bad Request</h1></center> <hr><center>nginx/1.21.0</center> </body> </html> 20:02:11.897772 da:b3:97:64:94:03 > ee:ee:ee:ee:ee:ee, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 64, id 62173, offset 0, flags [DF], proto TCP (6), length 52) 10.244.29.2.http > 10.244.243.192.36979: Flags [F.], cksum 0x26d1 (incorrect -> 0x1909), seq 310, ack 518, win 226, options [nop,nop,TS val 1118476623 ecr 252756832], length 0 20:02:11.900090 ee:ee:ee:ee:ee:ee > da:b3:97:64:94:03, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 55, id 36217, offset 0, flags [DF], proto TCP (6), length 52) 10.244.243.192.36979 > 10.244.29.2.http: Flags [.], cksum 0x17e7 (correct), ack 311, win 513, options [nop,nop,TS val 252756835 ecr 1118476623], length 0 20:02:11.900647 ee:ee:ee:ee:ee:ee > da:b3:97:64:94:03, ethertype IPv4 (0x0800), length 73: (tos 0x0, ttl 55, id 36218, offset 0, flags [DF], proto TCP (6), length 59) 10.244.243.192.36979 > 10.244.29.2.http: Flags [P.], cksum 0xb9d1 (correct), seq 518:525, ack 311, win 513, options [nop,nop,TS val 252756836 ecr 1118476623], length 7: HTTP 20:02:11.900742 ee:ee:ee:ee:ee:ee > da:b3:97:64:94:03, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 55, id 36219, offset 0, flags [DF], proto TCP (6), length 52) 10.244.243.192.36979 > 10.244.29.2.http: Flags [F.], cksum 0x17de (correct), seq 525, ack 311, win 513, options [nop,nop,TS val 252756836 ecr 1118476623], length 0 20:02:11.900752 da:b3:97:64:94:03 > ee:ee:ee:ee:ee:ee, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 64, id 62174, offset 0, flags [DF], proto TCP (6), length 52) 10.244.29.2.http > 10.244.243.192.36979: Flags [.], cksum 0x26d1 (incorrect -> 0x18fa), ack 526, win 226, options [nop,nop,TS val 1118476626 ecr 252756836], length 0 20:02:11.901607 ee:ee:ee:ee:ee:ee > da:b3:97:64:94:03, ethertype IPv4 (0x0800), length 74: (tos 0x0, ttl 55, id 36223, offset 0, flags [DF], proto TCP (6), length 60) 10.244.243.192.21048 > 10.244.29.2.http: Flags [S], cksum 0x5e5d (correct), seq 921463837, win 64240, options [mss 1460,nop,wscale 8,sackOK,TS val 252756836 ecr 0], length 0 20:02:11.901622 da:b3:97:64:94:03 > ee:ee:ee:ee:ee:ee, ethertype IPv4 (0x0800), length 74: (tos 0x0, ttl 64, id 0, offset 0, flags [DF], proto TCP (6), length 60) 10.244.29.2.http > 10.244.243.192.21048: Flags [S.], cksum 0x26d9 (incorrect -> 0x8530), seq 4043545303, ack 921463838, win 27760, options [mss 1400,sackOK,TS val 1118476627 ecr 252756836,nop,wscale 7], length 0 20:02:11.906457 ee:ee:ee:ee:ee:ee > da:b3:97:64:94:03, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 55, id 36226, offset 0, flags [DF], proto TCP (6), length 52) 10.244.243.192.21048 > 10.244.29.2.http: Flags [.], cksum 0x1e28 (correct), ack 1, win 515, options [nop,nop,TS val 252756842 ecr 1118476627], length 0 20:02:11.907824 ee:ee:ee:ee:ee:ee > da:b3:97:64:94:03, ethertype IPv4 (0x0800), length 583: (tos 0x0, ttl 55, id 36228, offset 0, flags [DF], proto TCP (6), length 569) 10.244.243.192.21048 > 10.244.29.2.http: Flags [P.], cksum 0x1483 (correct), seq 1:518, ack 1, win 515, options [nop,nop,TS val 252756842 ecr 1118476627], length 517: HTTP 20:02:11.907839 da:b3:97:64:94:03 > ee:ee:ee:ee:ee:ee, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 64, id 8121, offset 0, flags [DF], proto TCP (6), length 52) 10.244.29.2.http > 10.244.243.192.21048: Flags [.], cksum 0x26d1 (incorrect -> 0x1d3e), ack 518, win 226, options [nop,nop,TS val 1118476633 ecr 252756842], length 0 20:02:11.907871 da:b3:97:64:94:03 > ee:ee:ee:ee:ee:ee, ethertype IPv4 (0x0800), length 375: (tos 0x0, ttl 64, id 8122, offset 0, flags [DF], proto TCP (6), length 361) 10.244.29.2.http > 10.244.243.192.21048: Flags [P.], cksum 0x2806 (incorrect -> 0x7605), seq 1:310, ack 518, win 226, options [nop,nop,TS val 1118476633 ecr 252756842], length 309: HTTP, length: 309 HTTP/1.1 400 Bad Request Server: nginx/1.21.0 Date: Mon, 21 Jun 2021 12:02:11 GMT Content-Type: text/html Content-Length: 157 Connection: close <html> <head><title>400 Bad Request</title></head> <body> <center><h1>400 Bad Request</h1></center> <hr><center>nginx/1.21.0</center> </body> </html> 20:02:11.907927 da:b3:97:64:94:03 > ee:ee:ee:ee:ee:ee, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 64, id 8123, offset 0, flags [DF], proto TCP (6), length 52) 10.244.29.2.http > 10.244.243.192.21048: Flags [F.], cksum 0x26d1 (incorrect -> 0x1c08), seq 310, ack 518, win 226, options [nop,nop,TS val 1118476633 ecr 252756842], length 0 20:02:11.911854 ee:ee:ee:ee:ee:ee > da:b3:97:64:94:03, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 55, id 36231, offset 0, flags [DF], proto TCP (6), length 52) 10.244.243.192.21048 > 10.244.29.2.http: Flags [.], cksum 0x1ae4 (correct), ack 311, win 513, options [nop,nop,TS val 252756847 ecr 1118476633], length 0 20:02:11.912062 ee:ee:ee:ee:ee:ee > da:b3:97:64:94:03, ethertype IPv4 (0x0800), length 73: (tos 0x0, ttl 55, id 36232, offset 0, flags [DF], proto TCP (6), length 59) 10.244.243.192.21048 > 10.244.29.2.http: Flags [P.], cksum 0xbcce (correct), seq 518:525, ack 311, win 513, options [nop,nop,TS val 252756848 ecr 1118476633], length 7: HTTP 20:02:11.912237 ee:ee:ee:ee:ee:ee > da:b3:97:64:94:03, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 55, id 36233, offset 0, flags [DF], proto TCP (6), length 52) 10.244.243.192.21048 > 10.244.29.2.http: Flags [F.], cksum 0x1adb (correct), seq 525, ack 311, win 513, options [nop,nop,TS val 252756848 ecr 1118476633], length 0 20:02:11.912246 da:b3:97:64:94:03 > ee:ee:ee:ee:ee:ee, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 64, id 8124, offset 0, flags [DF], proto TCP (6), length 52) 10.244.29.2.http > 10.244.243.192.21048: Flags [.], cksum 0x26d1 (incorrect -> 0x1bf6), ack 526, win 226, options [nop,nop,TS val 1118476637 ecr 252756848], length 0 ^C 22 packets captured 22 packets received by filter 0 packets dropped by kernel You have mail in /var/spool/mail/root [root@bogon ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever 2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000 link/ipip 0.0.0.0 brd 0.0.0.0 4: eth0@if101: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1440 qdisc noqueue state UP group default link/ether da:b3:97:64:94:03 brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 10.244.29.2/32 brd 10.244.29.2 scope global eth0 valid_lft forever preferred_lft forever [root@bogon ~]#
再次访问
ubuntu
root@ubuntu:~# conntrack -L -o ktimestamp | grep 54109 tcp 6 3568 CLOSE_WAIT src=192.168.117.51 dst=10.10.16.82 sport=53838 dport=30090 src=10.244.41.2 dst=10.244.243.192 sport=80 dport=54109 [ASSURED] mark=0 use=1 conntrack v1.4.4 (conntrack-tools): 158 flow entries have been shown. root@ubuntu:~#
cloud
root@cloud:~# tcpdump -i eth0 tcp and port 80 -env tcpdump: listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes 20:17:15.532515 ee:ee:ee:ee:ee:ee > 72:98:01:99:c5:5a, ethertype IPv4 (0x0800), length 74: (tos 0x0, ttl 55, id 36387, offset 0, flags [DF], proto TCP (6), length 60) 10.244.243.192.54109 > 10.244.41.2.80: Flags [S], cksum 0xa3a0 (correct), seq 2322496604, win 64240, options [mss 1460,nop,wscale 8,sackOK,TS val 253660460 ecr 0], length 0 20:17:15.532564 72:98:01:99:c5:5a > ee:ee:ee:ee:ee:ee, ethertype IPv4 (0x0800), length 74: (tos 0x0, ttl 64, id 0, offset 0, flags [DF], proto TCP (6), length 60) 10.244.41.2.80 > 10.244.243.192.54109: Flags [S.], cksum 0x32d9 (incorrect -> 0x8f01), seq 3246101185, ack 2322496605, win 65236, options [mss 1400,sackOK,TS val 2795836932 ecr 253660460,nop,wscale 7], length 0 20:17:15.533832 ee:ee:ee:ee:ee:ee > 72:98:01:99:c5:5a, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 55, id 36388, offset 0, flags [DF], proto TCP (6), length 52) 10.244.243.192.54109 > 10.244.41.2.80: Flags [.], cksum 0xba61 (correct), ack 1, win 515, options [nop,nop,TS val 253660462 ecr 2795836932], length 0 20:18:00.538366 ee:ee:ee:ee:ee:ee > 72:98:01:99:c5:5a, ethertype IPv4 (0x0800), length 55: (tos 0x0, ttl 55, id 36392, offset 0, flags [DF], proto TCP (6), length 41) 10.244.243.192.54109 > 10.244.41.2.80: Flags [.], cksum 0x4c6f (correct), seq 0:1, ack 1, win 515, length 1: HTTP 20:18:00.538399 72:98:01:99:c5:5a > ee:ee:ee:ee:ee:ee, ethertype IPv4 (0x0800), length 78: (tos 0x0, ttl 64, id 64355, offset 0, flags [DF], proto TCP (6), length 64) 10.244.41.2.80 > 10.244.243.192.54109: Flags [.], cksum 0x32dd (incorrect -> 0xc6e9), ack 1, win 510, options [nop,nop,TS val 2795881938 ecr 253660462,nop,nop,sack 1 {0:1}], length 0 20:18:15.594104 72:98:01:99:c5:5a > ee:ee:ee:ee:ee:ee, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 64, id 64356, offset 0, flags [DF], proto TCP (6), length 52) 10.244.41.2.80 > 10.244.243.192.54109: Flags [F.], cksum 0x32d1 (incorrect -> 0xcfc7), seq 1, ack 1, win 510, options [nop,nop,TS val 2795896993 ecr 253660462], length 0 20:18:15.595465 ee:ee:ee:ee:ee:ee > 72:98:01:99:c5:5a, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 55, id 36394, offset 0, flags [DF], proto TCP (6), length 52) 10.244.243.192.54109 > 10.244.41.2.80: Flags [.], cksum 0xe524 (correct), ack 2, win 515, options [nop,nop,TS val 253720523 ecr 2795896993], length 0 ^C 7 packets captured 7 packets received by filter 0 packets dropped by kernel
root@cloud:~# tcpdump -i enahisic2i0 "ip proto 4" and host 10.10.16.82 -ennvv tcpdump: listening on enahisic2i0, link-type EN10MB (Ethernet), capture size 262144 bytes 20:17:15.532429 48:57:02:64:e7:ab > 9c:52:f8:67:c4:d3, ethertype IPv4 (0x0800), length 94: (tos 0x0, ttl 56, id 23484, offset 0, flags [DF], proto IPIP (4), length 80) 10.10.16.82 > 10.10.16.47: (tos 0x0, ttl 56, id 36387, offset 0, flags [DF], proto TCP (6), length 60) 10.244.243.192.54109 > 10.244.41.2.80: Flags [S], cksum 0xa3a0 (correct), seq 2322496604, win 64240, options [mss 1460,nop,wscale 8,sackOK,TS val 253660460 ecr 0], length 0 20:17:15.532608 9c:52:f8:67:c4:d3 > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 94: (tos 0x0, ttl 63, id 20880, offset 0, flags [DF], proto IPIP (4), length 80) 10.10.16.47 > 10.10.16.82: (tos 0x0, ttl 63, id 0, offset 0, flags [DF], proto TCP (6), length 60) 10.244.41.2.80 > 10.244.243.192.54109: Flags [S.], cksum 0x8f01 (correct), seq 3246101185, ack 2322496605, win 65236, options [mss 1400,sackOK,TS val 2795836932 ecr 253660460,nop,wscale 7], length 0 20:17:15.533812 48:57:02:64:e7:ab > 9c:52:f8:67:c4:d3, ethertype IPv4 (0x0800), length 86: (tos 0x0, ttl 56, id 23485, offset 0, flags [DF], proto IPIP (4), length 72) 10.10.16.82 > 10.10.16.47: (tos 0x0, ttl 56, id 36388, offset 0, flags [DF], proto TCP (6), length 52) 10.244.243.192.54109 > 10.244.41.2.80: Flags [.], cksum 0xba61 (correct), seq 1, ack 1, win 515, options [nop,nop,TS val 253660462 ecr 2795836932], length 0 20:18:00.538298 48:57:02:64:e7:ab > 9c:52:f8:67:c4:d3, ethertype IPv4 (0x0800), length 75: (tos 0x0, ttl 56, id 25664, offset 0, flags [DF], proto IPIP (4), length 61) 10.10.16.82 > 10.10.16.47: (tos 0x0, ttl 56, id 36392, offset 0, flags [DF], proto TCP (6), length 41) 10.244.243.192.54109 > 10.244.41.2.80: Flags [.], cksum 0x4c6f (correct), seq 0:1, ack 1, win 515, length 1: HTTP 20:18:00.538434 9c:52:f8:67:c4:d3 > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 29510, offset 0, flags [DF], proto IPIP (4), length 84) 10.10.16.47 > 10.10.16.82: (tos 0x0, ttl 63, id 64355, offset 0, flags [DF], proto TCP (6), length 64) 10.244.41.2.80 > 10.244.243.192.54109: Flags [.], cksum 0xc6e9 (correct), seq 1, ack 1, win 510, options [nop,nop,TS val 2795881938 ecr 253660462,nop,nop,sack 1 {0:1}], length 0 20:18:15.594175 9c:52:f8:67:c4:d3 > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 86: (tos 0x0, ttl 63, id 31181, offset 0, flags [DF], proto IPIP (4), length 72) 10.10.16.47 > 10.10.16.82: (tos 0x0, ttl 63, id 64356, offset 0, flags [DF], proto TCP (6), length 52) 10.244.41.2.80 > 10.244.243.192.54109: Flags [F.], cksum 0xcfc7 (correct), seq 1, ack 1, win 510, options [nop,nop,TS val 2795896993 ecr 253660462], length 0 20:18:15.595435 48:57:02:64:e7:ab > 9c:52:f8:67:c4:d3, ethertype IPv4 (0x0800), length 86: (tos 0x0, ttl 56, id 27742, offset 0, flags [DF], proto IPIP (4), length 72) 10.10.16.82 > 10.10.16.47: (tos 0x0, ttl 56, id 36394, offset 0, flags [DF], proto TCP (6), length 52) 10.244.243.192.54109 > 10.244.41.2.80: Flags [.], cksum 0xe524 (correct), seq 1, ack 2, win 515, options [nop,nop,TS val 253720523 ecr 2795896993], length 0 ^C 7 packets captured 8 packets received by filter 0 packets dropped by kernel
root@cloud:~# conntrack -L -o ktimestamp | grep 54109 tcp 6 86378 ESTABLISHED src=10.244.243.192 dst=10.244.41.2 sport=54109 dport=80 src=10.244.41.2 dst=10.244.243.192 sport=80 dport=54109 [ASSURED] mark=0 use=1 conntrack v1.4.4 (conntrack-tools): 103 flow entries have been shown. root@cloud:~#
bogon
[root@bogon ~]# tcpdump -i eth0 tcp and port 80 -env tcpdump: listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes 20:17:15.535555 ee:ee:ee:ee:ee:ee > da:b3:97:64:94:03, ethertype IPv4 (0x0800), length 74: (tos 0x0, ttl 55, id 36386, offset 0, flags [DF], proto TCP (6), length 60) 10.244.243.192.46814 > 10.244.29.2.http: Flags [S], cksum 0x0f91 (correct), seq 107855084, win 64240, options [mss 1460,nop,wscale 8,sackOK,TS val 253660460 ecr 0], length 0 20:17:15.535594 da:b3:97:64:94:03 > ee:ee:ee:ee:ee:ee, ethertype IPv4 (0x0800), length 74: (tos 0x0, ttl 64, id 0, offset 0, flags [DF], proto TCP (6), length 60) 10.244.29.2.http > 10.244.243.192.46814: Flags [S.], cksum 0x26d9 (incorrect -> 0x0ca5), seq 1752728393, ack 107855085, win 27760, options [mss 1400,sackOK,TS val 1119380254 ecr 253660460,nop,wscale 7], length 0 20:17:15.537118 ee:ee:ee:ee:ee:ee > da:b3:97:64:94:03, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 55, id 36389, offset 0, flags [DF], proto TCP (6), length 52) 10.244.243.192.46814 > 10.244.29.2.http: Flags [.], cksum 0xa5a0 (correct), ack 1, win 515, options [nop,nop,TS val 253660462 ecr 1119380254], length 0 20:18:00.541243 ee:ee:ee:ee:ee:ee > da:b3:97:64:94:03, ethertype IPv4 (0x0800), length 55: (tos 0x0, ttl 55, id 36391, offset 0, flags [DF], proto TCP (6), length 41) 10.244.243.192.46814 > 10.244.29.2.http: Flags [.], cksum 0x20db (correct), seq 0:1, ack 1, win 515, length 1: HTTP 20:18:00.541260 da:b3:97:64:94:03 > ee:ee:ee:ee:ee:ee, ethertype IPv4 (0x0800), length 78: (tos 0x0, ttl 64, id 25878, offset 0, flags [DF], proto TCP (6), length 64) 10.244.29.2.http > 10.244.243.192.46814: Flags [.], cksum 0x26dd (incorrect -> 0x3a31), ack 1, win 217, options [nop,nop,TS val 1119425259 ecr 253660462,nop,nop,sack 1 {0:1}], length 0 20:18:15.597364 da:b3:97:64:94:03 > ee:ee:ee:ee:ee:ee, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 64, id 25879, offset 0, flags [DF], proto TCP (6), length 52) 10.244.29.2.http > 10.244.243.192.46814: Flags [F.], cksum 0x26d1 (incorrect -> 0xbc2b), seq 1, ack 1, win 217, options [nop,nop,TS val 1119440315 ecr 253660462], length 0 20:18:15.598522 ee:ee:ee:ee:ee:ee > da:b3:97:64:94:03, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 55, id 36395, offset 0, flags [DF], proto TCP (6), length 52) 10.244.243.192.46814 > 10.244.29.2.http: Flags [.], cksum 0xd063 (correct), ack 2, win 515, options [nop,nop,TS val 253720523 ecr 1119440315], length 0 ^C 7 packets captured 7 packets received by filter 0 packets dropped by kernel You have mail in /var/spool/mail/root [root@bogon ~]#
[root@bogon ~]# conntrack -L -o ktimestamp | grep 46814 tcp 6 86365 ESTABLISHED src=10.244.243.192 dst=10.244.29.2 sport=46814 dport=80 src=10.244.29.2 dst=10.244.243.192 sport=80 dport=46814 [ASSURED] mark=0 use=1 conntrack v1.4.4 (conntrack-tools): 512 flow entries have been shown. [root@bogon ~]#
[root@bogon ~]# tcpdump -i enahisic2i0 "ip proto 4" and host 10.10.16.82 -ennvv tcpdump: listening on enahisic2i0, link-type EN10MB (Ethernet), capture size 262144 bytes 20:17:15.535214 48:57:02:64:e7:ab > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 94: (tos 0x0, ttl 56, id 13470, offset 0, flags [DF], proto IPIP (4), length 80) 10.10.16.82 > 10.10.16.81: (tos 0x0, ttl 56, id 36386, offset 0, flags [DF], proto TCP (6), length 60) 10.244.243.192.46814 > 10.244.29.2.80: Flags [S], cksum 0x0f91 (correct), seq 107855084, win 64240, options [mss 1460,nop,wscale 8,sackOK,TS val 253660460 ecr 0], length 0 20:17:15.535642 48:57:02:64:ea:1b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 94: (tos 0x0, ttl 63, id 1739, offset 0, flags [DF], proto IPIP (4), length 80) 10.10.16.81 > 10.10.16.82: (tos 0x0, ttl 63, id 0, offset 0, flags [DF], proto TCP (6), length 60) 10.244.29.2.80 > 10.244.243.192.46814: Flags [S.], cksum 0x0ca5 (correct), seq 1752728393, ack 107855085, win 27760, options [mss 1400,sackOK,TS val 1119380254 ecr 253660460,nop,wscale 7], length 0 20:17:15.537093 48:57:02:64:e7:ab > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 86: (tos 0x0, ttl 56, id 13471, offset 0, flags [DF], proto IPIP (4), length 72) 10.10.16.82 > 10.10.16.81: (tos 0x0, ttl 56, id 36389, offset 0, flags [DF], proto TCP (6), length 52) 10.244.243.192.46814 > 10.244.29.2.80: Flags [.], cksum 0xa5a0 (correct), seq 1, ack 1, win 515, options [nop,nop,TS val 253660462 ecr 1119380254], length 0 20:18:00.541043 48:57:02:64:e7:ab > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 75: (tos 0x0, ttl 56, id 13833, offset 0, flags [DF], proto IPIP (4), length 61) 10.10.16.82 > 10.10.16.81: (tos 0x0, ttl 56, id 36391, offset 0, flags [DF], proto TCP (6), length 41) 10.244.243.192.46814 > 10.244.29.2.80: Flags [.], cksum 0x20db (correct), seq 0:1, ack 1, win 515, length 1: HTTP 20:18:00.541300 48:57:02:64:ea:1b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 3472, offset 0, flags [DF], proto IPIP (4), length 84) 10.10.16.81 > 10.10.16.82: (tos 0x0, ttl 63, id 25878, offset 0, flags [DF], proto TCP (6), length 64) 10.244.29.2.80 > 10.244.243.192.46814: Flags [.], cksum 0x3a31 (correct), seq 1, ack 1, win 217, options [nop,nop,TS val 1119425259 ecr 253660462,nop,nop,sack 1 {0:1}], length 0 20:18:15.597437 48:57:02:64:ea:1b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 86: (tos 0x0, ttl 63, id 4311, offset 0, flags [DF], proto IPIP (4), length 72) 10.10.16.81 > 10.10.16.82: (tos 0x0, ttl 63, id 25879, offset 0, flags [DF], proto TCP (6), length 52) 10.244.29.2.80 > 10.244.243.192.46814: Flags [F.], cksum 0xbc2b (correct), seq 1, ack 1, win 217, options [nop,nop,TS val 1119440315 ecr 253660462], length 0 20:18:15.598317 48:57:02:64:e7:ab > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 86: (tos 0x0, ttl 56, id 17366, offset 0, flags [DF], proto IPIP (4), length 72) 10.10.16.82 > 10.10.16.81: (tos 0x0, ttl 56, id 36395, offset 0, flags [DF], proto TCP (6), length 52) 10.244.243.192.46814 > 10.244.29.2.80: Flags [.], cksum 0xd063 (correct), seq 1, ack 2, win 515, options [nop,nop,TS val 253720523 ecr 1119440315], length 0 ^C 7 packets captured 7 packets received by filter 0 packets dropped by kernel You have mail in /var/spool/mail/root [root@bogon ~]#
Calico部署
https://blog.csdn.net/xixihahalelehehe/article/details/105567076