zoukankan      html  css  js  c++  java
  • calico bgp rr

    bgp peer

    查看状态

    calicoctl node status
    

    配置全局 bgp peer(rr)

    cat << EOF | calicoctl create -f -
    apiVersion: projectcalico.org/v3
    kind: BGPPeer
    metadata:
      name: bgppeer-global-3040
    spec:
      peerIP: 172.26.6.1
      asNumber: 64567
    EOF
    
    # 删除
    $ calicoctl delete bgpPeer 172.26.6.1
    

    特定 BGP peer

    $ cat << EOF | calicoctl create -f -
    apiVersion: projectcalico.org/v3
    kind: BGPPeer
    metadata:
      name: bgppeer-node-aabbff
    spec:
      peerIP: aa:bb::ff
      node: node1
      asNumber: 64514
    EOF
    
    calicoctl delete bgpPeer aa:bb::ff --scope=node --node=node1
    calicoctl get bgpPeer

    1、 配置BGP Peer设置RR节点

    root@ubuntu:~# ./calicoctl get bgppeer
    NAME   PEERIP   NODE   ASN   
    
    root@ubuntu:~# ./calicoctl get bgppeer
    NAME   PEERIP   NODE   ASN   
    
    root@ubuntu:~# ./calicoctl  node status
    Calico process is running.
    
    IPv4 BGP status
    No IPv4 peers found.
    
    IPv6 BGP status
    No IPv6 peers found.
    
    root@ubuntu:~# 
    ./calicoctl apply -f bgpconfig.yaml
    root@ubuntu:~# cat bgpconfig.yaml
    apiVersion: projectcalico.org/v3
    kind: BGPConfiguration
    metadata:
      name: default
    spec:
      logSeverityScreen: Info
      nodeToNodeMeshEnabled: false 
      asNumber: 64512

     64512是全局as nu,node 默认as 是64512

    root@ubuntu:~# ./calicoctl get bgppeer
    NAME   PEERIP   NODE   ASN   
    
    root@ubuntu:~# 
    2、配置指定节点充当路由反射器

    从当前节点找两个节点充当路由反射器,作为路由反射器

    为方便让BGPPeer轻松选择节点,通过标签选择器匹配。给路由器反射器节点打标签:

    给选定的节点指定路由反射器ID

    复制代码
    root@ubuntu:~# ./calicoctl get node ubuntu  -o yaml > rr-node.yaml
    root@ubuntu:~# cat rr-node.yaml 
    apiVersion: projectcalico.org/v3
    kind: Node
    metadata:
      annotations:
        projectcalico.org/kube-labels: '{"beta.kubernetes.io/arch":"arm64","beta.kubernetes.io/os":"linux","kubernetes.io/arch":"arm64","kubernetes.io/hostname":"ubuntu","kubernetes.io/os":"linux","node-role.kubernetes.io/master":""}'
      creationTimestamp: "2021-06-18T11:12:32Z"
      labels:
        beta.kubernetes.io/arch: arm64
        beta.kubernetes.io/os: linux
        kubernetes.io/arch: arm64
        kubernetes.io/hostname: ubuntu
        kubernetes.io/os: linux
        node-role.kubernetes.io/master: ""
      name: ubuntu
      resourceVersion: "797539"
      uid: 369ae2bb-756d-446d-a044-9225d9849a13
    spec:
      addresses:
      - address: 10.10.16.82/24
        type: CalicoNodeIP
      - address: 10.10.16.82
        type: InternalIP
      bgp:
        ipv4Address: 10.10.16.82/24
      orchRefs:
      - nodeName: ubuntu
        orchestrator: k8s
    status:
      podCIDRs:
      - 10.244.0.0/24
    root@ubuntu:~# 
    复制代码
      增加  routeReflectorClusterID: 244.0.0.1
    复制代码
    root@ubuntu:~# ./calicoctl apply -f rr-node.yaml
    Successfully applied 1 'Node' resource(s)
    root@ubuntu:~# cat rr-node.yaml 
    apiVersion: projectcalico.org/v3
    kind: Node
    metadata:
      annotations:
        projectcalico.org/kube-labels: '{"beta.kubernetes.io/arch":"arm64","beta.kubernetes.io/os":"linux","kubernetes.io/arch":"arm64","kubernetes.io/hostname":"ubuntu","kubernetes.io/os":"linux","node-role.kubernetes.io/master":""}'
      creationTimestamp: "2021-06-18T11:12:32Z"
      labels:
        beta.kubernetes.io/arch: arm64
        beta.kubernetes.io/os: linux
        kubernetes.io/arch: arm64
        kubernetes.io/hostname: ubuntu
        kubernetes.io/os: linux
        node-role.kubernetes.io/master: ""
      name: ubuntu
      resourceVersion: "797539"
      uid: 369ae2bb-756d-446d-a044-9225d9849a13
    spec:
      addresses:
      - address: 10.10.16.82/24
        type: CalicoNodeIP
      - address: 10.10.16.82
        type: InternalIP
      bgp:
        ipv4Address: 10.10.16.82/24
        routeReflectorClusterID: 244.0.0.1
      orchRefs:
      - nodeName: ubuntu
        orchestrator: k8s
    status:
      podCIDRs:
      - 10.244.0.0/24
    3、 kubectl label node ubuntu route-reflector=true
    root@ubuntu:~# kubectl get node --show-labels
    NAME     STATUS   ROLES    AGE     VERSION   LABELS
    bogon    Ready    worker   3d21h   v1.18.1   beta.kubernetes.io/arch=arm64,beta.kubernetes.io/os=linux,kubernetes.io/arch=arm64,kubernetes.io/hostname=bogon,kubernetes.io/os=linux,node-role.kubernetes.io/worker=worker
    cloud    Ready    worker   3d21h   v1.21.1   beta.kubernetes.io/arch=arm64,beta.kubernetes.io/os=linux,kubernetes.io/arch=arm64,kubernetes.io/hostname=cloud,kubernetes.io/os=linux,node-role.kubernetes.io/worker=worker
    ubuntu   Ready    master   3d21h   v1.18.1   beta.kubernetes.io/arch=arm64,beta.kubernetes.io/os=linux,kubernetes.io/arch=arm64,kubernetes.io/hostname=ubuntu,kubernetes.io/os=linux,node-role.kubernetes.io/master=,route-reflector=true
    root@ubuntu:~# 

    4、集群内RR模式

    开启集群内部RR模式

    此时已经选择并且配置完成节点信息,我们需要下发规则使BGP RR模式生效。

    开启集群内部RR模式

    #创建一个rr-rule.yaml文件,文件内容如下:
    apiVersion: projectcalico.org/v3
    kind: BGPPeer
    metadata:
      name: unbuntu #NodeName
    spec:
      nodeSelector: "all()"
      peerSelector: "has(route-reflector)"
    #应用规则开启集群内部RR模式
    calicoctl create -f rr-rule.yaml
    root@ubuntu:~# ./calicoctl create -f bgp-rr.yaml 
    Successfully created 1 'BGPPeer' resource(s)
    root@ubuntu:~# cat bgp-rr.yaml 
    apiVersion: projectcalico.org/v3
    kind: BGPPeer
    metadata:
      name: unbuntu
    spec:
      nodeSelector: all()
      peerSelector: route-reflector == 'true'
    root@ubuntu:~# 
    root@ubuntu:~# ./calicoctl get bgppeer
    NAME      PEERIP   NODE    ASN   
    unbuntu            all()   0     
    root@ubuntu:~# ./calicoctl  node status
    Calico process is running.
    
    IPv4 BGP status
    +--------------+---------------+-------+----------+-------------+
    | PEER ADDRESS |   PEER TYPE   | STATE |  SINCE   |    INFO     |
    +--------------+---------------+-------+----------+-------------+
    | 10.10.16.47  | node specific | up    | 08:45:50 | Established |
    | 10.10.16.81  | node specific | up    | 08:45:50 | Established |
    +--------------+---------------+-------+----------+-------------+
    
    IPv6 BGP status
    No IPv6 peers found.
    
    root@ubuntu:~# 

    cloud 也只有一个bpg peer

    root@cloud:~# ./calicoctl  node status
    Calico process is running.
    
    IPv4 BGP status
    +--------------+---------------+-------+----------+-------------+
    | PEER ADDRESS |   PEER TYPE   | STATE |  SINCE   |    INFO     |
    +--------------+---------------+-------+----------+-------------+
    | 10.10.16.82  | node specific | up    | 08:45:50 | Established |
    +--------------+---------------+-------+----------+-------------+
    
    IPv6 BGP status
    No IPv6 peers found.
    
    root@cloud:~# 

    bogon 只有一个bgp peer

    You have mail in /var/spool/mail/root
    [root@bogon ~]# ./calicoctl  node status
    Calico process is running.
    
    IPv4 BGP status
    +--------------+---------------+-------+----------+-------------+
    | PEER ADDRESS |   PEER TYPE   | STATE |  SINCE   |    INFO     |
    +--------------+---------------+-------+----------+-------------+
    | 10.10.16.82  | node specific | up    | 08:45:50 | Established |
    +--------------+---------------+-------+----------+-------------+
    
    IPv6 BGP status
    No IPv6 peers found.
    
    You have mail in /var/spool/mail/root
    [root@bogon ~]# 
     
    root@ubuntu:~# kubectl get pods -o wide
    NAME                         READY   STATUS    RESTARTS   AGE     IP               NODE     NOMINATED NODE   READINESS GATES
    web-nginx-7bdc6b976b-7454h   1/1     Running   0          6h29m   10.244.29.4      bogon    <none>           <none>
    web-nginx-7bdc6b976b-cml8v   1/1     Running   0          6h29m   10.244.41.3      cloud    <none>           <none>
    web-nginx-7bdc6b976b-p7m2b   1/1     Running   0          6h29m   10.244.243.197   ubuntu   <none>           <none>
    root@ubuntu:~# 

    cloud pod 

    root@cloud:~# ip a
    1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
        link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
        inet 127.0.0.1/8 scope host lo
           valid_lft forever preferred_lft forever
    2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000
        link/ipip 0.0.0.0 brd 0.0.0.0
    4: eth0@if44: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1440 qdisc noqueue state UP group default 
        link/ether f2:a1:42:ff:d4:86 brd ff:ff:ff:ff:ff:ff link-netnsid 0
        inet 10.244.41.3/32 brd 10.244.41.3 scope global eth0
           valid_lft forever preferred_lft forever
    root@cloud:~# ping 10.244.29.4
    PING 10.244.29.4 (10.244.29.4) 56(84) bytes of data.
    64 bytes from 10.244.29.4: icmp_seq=1 ttl=62 time=0.269 ms
    64 bytes from 10.244.29.4: icmp_seq=2 ttl=62 time=0.131 ms
    ^C
    --- 10.244.29.4 ping statistics ---
    2 packets transmitted, 2 received, 0% packet loss, time 1013ms
    rtt min/avg/max/mdev = 0.131/0.200/0.269/0.069 ms
    root@cloud:~# ping  10.244.243.197
    PING 10.244.243.197 (10.244.243.197) 56(84) bytes of data.
    64 bytes from 10.244.243.197: icmp_seq=1 ttl=62 time=0.275 ms
    64 bytes from 10.244.243.197: icmp_seq=2 ttl=62 time=0.173 ms
    64 bytes from 10.244.243.197: icmp_seq=3 ttl=62 time=0.128 ms
    ^C
    --- 10.244.243.197 ping statistics ---
    3 packets transmitted, 3 received, 0% packet loss, time 2035ms
    rtt min/avg/max/mdev = 0.128/0.192/0.275/0.061 ms
    root@cloud:~# 
    

      bogon pod

    [root@bogon ~]# ping 10.244.243.197
    PING 10.244.243.197 (10.244.243.197) 56(84) bytes of data.
    64 bytes from 10.244.243.197: icmp_seq=1 ttl=62 time=0.296 ms
    64 bytes from 10.244.243.197: icmp_seq=2 ttl=62 time=0.213 ms
    ^C
    --- 10.244.243.197 ping statistics ---
    2 packets transmitted, 2 received, 0% packet loss, time 1020ms
    rtt min/avg/max/mdev = 0.213/0.254/0.296/0.044 ms
    You have mail in /var/spool/mail/root
    [root@bogon ~]# ping 10.244.29.4
    PING 10.244.29.4 (10.244.29.4) 56(84) bytes of data.
    64 bytes from 10.244.29.4: icmp_seq=1 ttl=64 time=0.033 ms
    64 bytes from 10.244.29.4: icmp_seq=2 ttl=64 time=0.039 ms
    ^C
    --- 10.244.29.4 ping statistics ---
    2 packets transmitted, 2 received, 0% packet loss, time 1061ms
    rtt min/avg/max/mdev = 0.033/0.036/0.039/0.003 ms
    [root@bogon ~]# ip a
    1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
        link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
        inet 127.0.0.1/8 scope host lo
           valid_lft forever preferred_lft forever
    2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000
        link/ipip 0.0.0.0 brd 0.0.0.0
    4: eth0@if103: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1440 qdisc noqueue state UP group default 
        link/ether fa:34:76:a4:9d:f8 brd ff:ff:ff:ff:ff:ff link-netnsid 0
        inet 10.244.29.4/32 brd 10.244.29.4 scope global eth0
           valid_lft forever preferred_lft forever
    [root@bogon ~]# 

    bogon 路由

    [root@bogon ~]# route -n
    Kernel IP routing table
    Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
    0.0.0.0         10.10.16.254    0.0.0.0         UG    0      0        0 enahisic2i0
    10.2.0.1        0.0.0.0         255.255.255.255 UH    0      0        0 v-cali-peer
    10.10.16.0      0.0.0.0         255.255.255.0   U     0      0        0 enahisic2i0
    10.10.34.0      0.0.0.0         255.255.255.0   U     0      0        0 enahisic2i2
    10.10.102.0     0.0.0.0         255.255.255.0   U     0      0        0 enahisic2i1
    10.244.2.0      10.10.16.47     255.255.255.0   UG    0      0        0 enahisic2i0
    10.244.29.0     0.0.0.0         255.255.255.192 U     0      0        0 *
    10.244.29.1     0.0.0.0         255.255.255.255 UH    0      0        0 cali2e486421e22
    10.244.29.4     0.0.0.0         255.255.255.255 UH    0      0        0 calibe3388252a1
    10.244.41.0     10.10.16.47     255.255.255.192 UG    0      0        0 enahisic2i0
    10.244.243.192  10.10.16.82     255.255.255.192 UG    0      0        0 enahisic2i0
    14.14.18.0      0.0.0.0         255.255.255.0   U     0      0        0 enahisic2i3.310
    172.16.100.0    0.0.0.0         255.255.255.0   U     0      0        0 brqf1411bad-10
    172.17.0.0      0.0.0.0         255.255.0.0     U     0      0        0 docker0
    172.168.104.0   0.0.0.0         255.255.255.0   U     0      0        0 enah2i3.1022
    192.168.33.0    0.0.0.0         255.255.255.0   U     0      0        0 enahisic2i1
    192.168.122.0   0.0.0.0         255.255.255.0   U     0      0        0 virbr0
    You have mail in /var/spool/mail/root
    [root@bogon ~]# 

    cloud 路由

    root@cloud:~# route -n
    Kernel IP routing table
    Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
    0.0.0.0         10.10.16.254    0.0.0.0         UG    0      0        0 enahisic2i0
    9.251.0.0       172.17.0.1      255.255.0.0     UG    0      0        0 docker0
    10.10.16.0      0.0.0.0         255.255.255.0   U     0      0        0 enahisic2i0
    10.99.1.231     10.10.16.82     255.255.255.255 UGH   0      0        0 enahisic2i0
    10.110.79.116   10.10.16.82     255.255.255.255 UGH   0      0        0 enahisic2i0
    10.110.171.213  10.10.16.82     255.255.255.255 UGH   0      0        0 enahisic2i0
    10.244.2.0      0.0.0.0         255.255.255.0   U     0      0        0 cni0
    10.244.29.0     10.10.16.81     255.255.255.192 UG    0      0        0 enahisic2i0
    10.244.41.0     0.0.0.0         255.255.255.192 U     0      0        0 *
    10.244.41.1     0.0.0.0         255.255.255.255 UH    0      0        0 cali027a65c4a41
    10.244.41.3     0.0.0.0         255.255.255.255 UH    0      0        0 calib81bd12045e
    10.244.243.192  10.10.16.82     255.255.255.192 UG    0      0        0 enahisic2i0
    172.17.0.0      0.0.0.0         255.255.0.0     U     0      0        0 docker0
    root@cloud:~# 

     bogon 采用不同的as nu

    root@ubuntu:~# cat  bgp-peer-81.yaml 
    apiVersion: projectcalico.org/v3
    kind: BGPPeer
    metadata:
      name: bogon 
    spec:
      peerIP: 10.10.16.81
      asNumber: 64513
    root@ubuntu:~#
    root@ubuntu:~# ./calicoctl  apply -f bgp-peer-81.yaml 
    Successfully applied 1 'BGPPeer' resource(s)
    root@ubuntu:~# ./calicoctl  node status
    Calico process is running.
    
    IPv4 BGP status
    +--------------+---------------+-------+----------+--------------------------------+
    | PEER ADDRESS |   PEER TYPE   | STATE |  SINCE   |              INFO              |
    +--------------+---------------+-------+----------+--------------------------------+
    | 10.10.16.47  | node specific | up    | 08:45:50 | Established                    |
    | 10.10.16.81  | global        | start | 09:01:58 | Active Socket: Connection      |
    |              |               |       |          | closed                         |
    +--------------+---------------+-------+----------+--------------------------------+
    
    IPv6 BGP status
    No IPv6 peers found.
    
    root@ubuntu:~# ./calicoctl  node status
    Calico process is running.
    
    IPv4 BGP status
    +--------------+---------------+-------+----------+--------------------------------+
    | PEER ADDRESS |   PEER TYPE   | STATE |  SINCE   |              INFO              |
    +--------------+---------------+-------+----------+--------------------------------+
    | 10.10.16.47  | node specific | up    | 08:45:50 | Established                    |
    | 10.10.16.81  | global        | start | 09:01:58 | OpenSent Socket: Connection    |
    |              |               |       |          | closed                         |
    +--------------+---------------+-------+----------+--------------------------------+
    
    IPv6 BGP status
    No IPv6 peers found.
    
    root@ubuntu:~# ./calicoctl  node status
    Calico process is running.
    
    IPv4 BGP status
    +--------------+---------------+-------+----------+--------------------------------+
    | PEER ADDRESS |   PEER TYPE   | STATE |  SINCE   |              INFO              |
    +--------------+---------------+-------+----------+--------------------------------+
    | 10.10.16.47  | node specific | up    | 08:45:50 | Established                    |
    | 10.10.16.81  | global        | start | 09:01:58 | OpenSent Socket: Connection    |
    |              |               |       |          | closed                         |
    +--------------+---------------+-------+----------+--------------------------------+
    
    IPv6 BGP status
    No IPv6 peers found.
    
    root@ubuntu:~# ./calicoctl  node status
    Calico process is running.
    
    IPv4 BGP status
    +--------------+---------------+-------+----------+-----------------------------+
    | PEER ADDRESS |   PEER TYPE   | STATE |  SINCE   |            INFO             |
    +--------------+---------------+-------+----------+-----------------------------+
    | 10.10.16.47  | node specific | up    | 08:45:50 | Established                 |
    | 10.10.16.81  | global        | start | 09:03:15 | Idle BGP Error: Bad peer AS |
    +--------------+---------------+-------+----------+-----------------------------+
    
    IPv6 BGP status
    No IPv6 peers found.
    
    root@ubuntu:~# cat  bgp-peer-81.yaml 
    apiVersion: projectcalico.org/v3
    kind: BGPPeer
    metadata:
      name: bogon 
    spec:
      peerIP: 10.10.16.81
      asNumber: 64513
    root@ubuntu:~# 
    root@cloud:~# ./calicoctl  node status
    Calico process is running.
    
    IPv4 BGP status
    +--------------+---------------+-------+----------+--------------------------------+
    | PEER ADDRESS |   PEER TYPE   | STATE |  SINCE   |              INFO              |
    +--------------+---------------+-------+----------+--------------------------------+
    | 10.10.16.82  | node specific | up    | 08:45:50 | Established                    |
    | 10.10.16.81  | global        | start | 09:06:03 | Active BGP Error: Hold timer   |
    |              |               |       |          | expired                        |
    +--------------+---------------+-------+----------+--------------------------------+
    
    IPv6 BGP status
    No IPv6 peers found.
    
    root@cloud:~# 
    [root@bogon ~]# ./calicoctl  node status
    Calico process is running.
    
    IPv4 BGP status
    +--------------+---------------+-------+----------+----------------------------+
    | PEER ADDRESS |   PEER TYPE   | STATE |  SINCE   |            INFO            |
    +--------------+---------------+-------+----------+----------------------------+
    | 10.10.16.82  | node specific | start | 09:05:45 | Idle Received: Bad peer AS |
    +--------------+---------------+-------+----------+----------------------------+
    
    IPv6 BGP status
    No IPv6 peers found.
    
    You have mail in /var/spool/mail/root
    [root@bogon ~]# 

    bogon pod无法访问其他pod 

    [root@bogon ~]#  ip a
    1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
        link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
        inet 127.0.0.1/8 scope host lo
           valid_lft forever preferred_lft forever
    2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000
        link/ipip 0.0.0.0 brd 0.0.0.0
    4: eth0@if103: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1440 qdisc noqueue state UP group default 
        link/ether fa:34:76:a4:9d:f8 brd ff:ff:ff:ff:ff:ff link-netnsid 0
        inet 10.244.29.4/32 brd 10.244.29.4 scope global eth0
           valid_lft forever preferred_lft forever
    [root@bogon ~]# ping  10.244.243.197
    PING 10.244.243.197 (10.244.243.197) 56(84) bytes of data.
    ^C
    --- 10.244.243.197 ping statistics ---
    1 packets transmitted, 0 received, 100% packet loss, time 0ms
    
    [root@bogon ~]# ping 10.244.41.3
    PING 10.244.41.3 (10.244.41.3) 56(84) bytes of data.
    ^C
    --- 10.244.41.3 ping statistics ---
    2 packets transmitted, 0 received, 100% packet loss, time 1020ms
    
    [root@bogon ~]# 

    cloud pod 可以访问 ubuntu pod

    root@cloud:~# ping  10.244.243.197
    PING 10.244.243.197 (10.244.243.197) 56(84) bytes of data.
    64 bytes from 10.244.243.197: icmp_seq=1 ttl=62 time=0.271 ms
    64 bytes from 10.244.243.197: icmp_seq=2 ttl=62 time=0.145 ms
    ^C
    --- 10.244.243.197 ping statistics ---
    2 packets transmitted, 2 received, 0% packet loss, time 1017ms
    rtt min/avg/max/mdev = 0.145/0.208/0.271/0.063 ms
    root@cloud:~# ping 10.244.29.4
    PING 10.244.29.4 (10.244.29.4) 56(84) bytes of data.
    ^C
    --- 10.244.29.4 ping statistics ---
    57 packets transmitted, 0 received, 100% packet loss, time 57342ms
    
    root@cloud:~# ip a
    1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
        link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
        inet 127.0.0.1/8 scope host lo
           valid_lft forever preferred_lft forever
    2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000
        link/ipip 0.0.0.0 brd 0.0.0.0
    4: eth0@if44: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1440 qdisc noqueue state UP group default 
        link/ether f2:a1:42:ff:d4:86 brd ff:ff:ff:ff:ff:ff link-netnsid 0
        inet 10.244.41.3/32 brd 10.244.41.3 scope global eth0
           valid_lft forever preferred_lft forever
    root@cloud:~# 

    恢复bogon的as num

    root@ubuntu:~# ./calicoctl  apply -f bgp-peer-81.yaml 
    Successfully applied 1 'BGPPeer' resource(s)
    root@ubuntu:~# cat bgp-peer-81.yaml 
    apiVersion: projectcalico.org/v3
    kind: BGPPeer
    metadata:
      name: bogon 
    spec:
      peerIP: 10.10.16.81
      asNumber: 64512
    root@ubuntu:~# 
    [root@bogon ~]# ./calicoctl  node status
    Calico process is running.
    
    IPv4 BGP status
    +--------------+---------------+-------+----------+-------------+
    | PEER ADDRESS |   PEER TYPE   | STATE |  SINCE   |    INFO     |
    +--------------+---------------+-------+----------+-------------+
    | 10.10.16.82  | node specific | up    | 09:09:09 | Established |
    | 10.10.16.47  | node specific | up    | 09:09:06 | Established |
    +--------------+---------------+-------+----------+-------------+
    
    IPv6 BGP status
    No IPv6 peers found.
    
    You have mail in /var/spool/mail/root
    root@ubuntu:~# ./calicoctl  node status
    Calico process is running.
    
    IPv4 BGP status
    +--------------+---------------+-------+----------+-------------+
    | PEER ADDRESS |   PEER TYPE   | STATE |  SINCE   |    INFO     |
    +--------------+---------------+-------+----------+-------------+
    | 10.10.16.47  | node specific | up    | 08:45:50 | Established |
    | 10.10.16.81  | global        | up    | 09:09:08 | Established |
    +--------------+---------------+-------+----------+-------------+
    
    IPv6 BGP status
    No IPv6 peers found.
    
    root@ubuntu:~# 

    竟然都有两个peer

    root@cloud:~# ./calicoctl  node status
    Calico process is running.
    
    IPv4 BGP status
    +--------------+---------------+-------+----------+-------------+
    | PEER ADDRESS |   PEER TYPE   | STATE |  SINCE   |    INFO     |
    +--------------+---------------+-------+----------+-------------+
    | 10.10.16.82  | node specific | up    | 08:45:50 | Established |
    | 10.10.16.81  | global        | up    | 09:09:06 | Established |
    +--------------+---------------+-------+----------+-------------+
    
    IPv6 BGP status
    No IPv6 peers found.
    
    root@cloud:~# 
    root@ubuntu:~# ./calicoctl get bgppeer
    NAME      PEERIP        NODE       ASN     
    bogon     10.10.16.81   (global)   64512   
    unbuntu                 all()      0       
    
    root@ubuntu:~# ./calicoctl get bgppeer

     删除 bgpeer

    root@ubuntu:~# ./calicoctl delete  bgppeer bogon
    Successfully deleted 1 'BGPPeer' resource(s)
    root@ubuntu:~# ./calicoctl get bgppeer
    NAME      PEERIP   NODE    ASN   
    unbuntu            all()   0     
    
    root@ubuntu:~# 
    root@ubuntu:~# 

     bogon恢复了

    root@ubuntu:~# ./calicoctl get nodes --output=wide
    NAME     ASN       IPV4             IPV6   
    bogon    (64512)   10.10.16.81/24          
    cloud    (64512)   10.10.16.47/24          
    ubuntu   (64512)   10.10.16.82/24          
    
    root@ubuntu:~# ./calicoctl get nodes status
    resource does not exist: Node(status) with error: nodes "status" not found
    root@ubuntu:~# ./calicoctl get  status
    Failed to get resources: resource type 'status' is not supported
    root@ubuntu:~# ./calicoctl  node status
    Calico process is running.
    
    IPv4 BGP status
    +--------------+---------------+-------+----------+-------------+
    | PEER ADDRESS |   PEER TYPE   | STATE |  SINCE   |    INFO     |
    +--------------+---------------+-------+----------+-------------+
    | 10.10.16.47  | node specific | up    | 08:45:50 | Established |
    | 10.10.16.81  | node specific | up    | 09:35:24 | Established |
    +--------------+---------------+-------+----------+-------------+
    
    IPv6 BGP status
    No IPv6 peers found.
    [root@bogon ~]# ./calicoctl  node status
    Calico process is running.
    
    IPv4 BGP status
    +--------------+---------------+-------+----------+-------------+
    | PEER ADDRESS |   PEER TYPE   | STATE |  SINCE   |    INFO     |
    +--------------+---------------+-------+----------+-------------+
    | 10.10.16.82  | node specific | up    | 09:35:24 | Established |
    +--------------+---------------+-------+----------+-------------+
    
    IPv6 BGP status
    No IPv6 peers found.
    
    You have mail in /var/spool/mail/root
    [root@bogon ~]# 

    bgpeer 恢复正常了

    root@cloud:~# ./calicoctl  node status
    Calico process is running.
    
    IPv4 BGP status
    +--------------+---------------+-------+----------+-------------+
    | PEER ADDRESS |   PEER TYPE   | STATE |  SINCE   |    INFO     |
    +--------------+---------------+-------+----------+-------------+
    | 10.10.16.82  | node specific | up    | 08:45:50 | Established |
    +--------------+---------------+-------+----------+-------------+
    
    IPv6 BGP status
    No IPv6 peers found.
    
    root@cloud:~# 
    [root@bogon ~]#  ip a
    1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
        link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
        inet 127.0.0.1/8 scope host lo
           valid_lft forever preferred_lft forever
    2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000
        link/ipip 0.0.0.0 brd 0.0.0.0
    4: eth0@if103: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1440 qdisc noqueue state UP group default 
        link/ether fa:34:76:a4:9d:f8 brd ff:ff:ff:ff:ff:ff link-netnsid 0
        inet 10.244.29.4/32 brd 10.244.29.4 scope global eth0
           valid_lft forever preferred_lft forever
    [root@bogon ~]# ping  10.244.243.197
    PING 10.244.243.197 (10.244.243.197) 56(84) bytes of data.
    ^C
    --- 10.244.243.197 ping statistics ---
    1 packets transmitted, 0 received, 100% packet loss, time 0ms
    
    [root@bogon ~]# ping 10.244.41.3
    PING 10.244.41.3 (10.244.41.3) 56(84) bytes of data.
    ^C
    --- 10.244.41.3 ping statistics ---
    2 packets transmitted, 0 received, 100% packet loss, time 1020ms
    
    [root@bogon ~]# ping 10.244.41.3
    PING 10.244.41.3 (10.244.41.3) 56(84) bytes of data.
    64 bytes from 10.244.41.3: icmp_seq=1 ttl=62 time=0.241 ms
    ^C
    --- 10.244.41.3 ping statistics ---
    1 packets transmitted, 1 received, 0% packet loss, time 0ms
    rtt min/avg/max/mdev = 0.241/0.241/0.241/0.000 ms
    You have mail in /var/spool/mail/root
    [root@bogon ~]# ping  10.244.243.197
    PING 10.244.243.197 (10.244.243.197) 56(84) bytes of data.
    64 bytes from 10.244.243.197: icmp_seq=1 ttl=62 time=0.271 ms
    64 bytes from 10.244.243.197: icmp_seq=2 ttl=62 time=0.183 ms
    ^C
    --- 10.244.243.197 ping statistics ---
    2 packets transmitted, 2 received, 0% packet loss, time 1034ms
    rtt min/avg/max/mdev = 0.183/0.227/0.271/0.044 ms
    [root@bogon ~]# 

    为 node 节点进行分组(添加 label)

    root@ubuntu:~# kubectl get node --show-labels
    NAME     STATUS   ROLES    AGE     VERSION   LABELS
    bogon    Ready    worker   3d22h   v1.18.1   beta.kubernetes.io/arch=arm64,beta.kubernetes.io/os=linux,kubernetes.io/arch=arm64,kubernetes.io/hostname=bogon,kubernetes.io/os=linux,node-role.kubernetes.io/worker=worker
    cloud    Ready    worker   3d22h   v1.21.1   beta.kubernetes.io/arch=arm64,beta.kubernetes.io/os=linux,kubernetes.io/arch=arm64,kubernetes.io/hostname=cloud,kubernetes.io/os=linux,node-role.kubernetes.io/worker=worker
    ubuntu   Ready    master   3d22h   v1.18.1   beta.kubernetes.io/arch=arm64,beta.kubernetes.io/os=linux,kubernetes.io/arch=arm64,kubernetes.io/hostname=ubuntu,kubernetes.io/os=linux,node-role.kubernetes.io/master=,route-reflector=true
    root@ubuntu:~# 
    root@ubuntu:~# kubectl label nodes ubuntu rr-id=rr1
    node/ubuntu labeled
    root@ubuntu:~# kubectl label nodes ubuntu rr-group=rr1
    node/ubuntu labeled
    root@ubuntu:~# vi  bgp-rr.yaml
    apiVersion: projectcalico.org/v3
    kind: BGPPeer
    metadata:
      name: unbuntu
    spec:
      nodeSelector: rr-group == 'rr1'
      peerSelector: rr-id  == 'rr1'
    ~
    root@ubuntu:~# ./calicoctl apply -f rr1-to-node-peer.yaml
    Successfully applied 1 'BGPPeer' resource(s)
    root@ubuntu:~# ./calicoctl  node status
    Calico process is running.
    
    IPv4 BGP status
    +--------------+---------------+-------+----------+-------------+
    | PEER ADDRESS |   PEER TYPE   | STATE |  SINCE   |    INFO     |
    +--------------+---------------+-------+----------+-------------+
    | 10.10.16.47  | node specific | up    | 08:45:51 | Established |
    | 10.10.16.81  | node specific | up    | 09:35:25 | Established |
    +--------------+---------------+-------+----------+-------------+
    root@ubuntu:~# cat  rr1-to-node-peer.yaml 
    apiVersion: projectcalico.org/v3
    kind: BGPPeer
    metadata:
      name: rr1-to-node-peer              ## 给BGPPeer取一个名称,方便识别
    
    spec:
      nodeSelector: rr-group == 'rr1'     ## 通过节点选择器添加有rr-group == ‘rr1’标签的节点
    
      peerSelector: rr-id  == 'rr1'       ## 通过peer选择器添加有rr-id == ‘rr1’标签的路由反射器
    root@ubuntu:~# kubectl get node --show-labels
    NAME     STATUS   ROLES    AGE     VERSION   LABELS
    bogon    Ready    worker   3d22h   v1.18.1   beta.kubernetes.io/arch=arm64,beta.kubernetes.io/os=linux,kubernetes.io/arch=arm64,kubernetes.io/hostname=bogon,kubernetes.io/os=linux,node-role.kubernetes.io/worker=worker
    cloud    Ready    worker   3d22h   v1.21.1   beta.kubernetes.io/arch=arm64,beta.kubernetes.io/os=linux,kubernetes.io/arch=arm64,kubernetes.io/hostname=cloud,kubernetes.io/os=linux,node-role.kubernetes.io/worker=worker
    ubuntu   Ready    master   3d22h   v1.18.1   beta.kubernetes.io/arch=arm64,beta.kubernetes.io/os=linux,kubernetes.io/arch=arm64,kubernetes.io/hostname=ubuntu,kubernetes.io/os=linux,node-role.kubernetes.io/master=,route-reflector=true,rr-group=rr1,rr-id=rr1
    root@ubuntu:~# ./calicoctl  node status
    Calico process is running.
    
    IPv4 BGP status
    +--------------+---------------+-------+----------+-------------+
    | PEER ADDRESS |   PEER TYPE   | STATE |  SINCE   |    INFO     |
    +--------------+---------------+-------+----------+-------------+
    | 10.10.16.47  | node specific | up    | 08:45:51 | Established |
    | 10.10.16.81  | node specific | up    | 09:35:25 | Established |
    +--------------+---------------+-------+----------+-------------+
    
    IPv6 BGP status
    No IPv6 peers found.
    
    root@ubuntu:~# ./calicoctl get bgppeer
    NAME      PEERIP   NODE    ASN   
    unbuntu            all()   0     
    
    root@ubuntu:~# ./calicoctl  node status
    Calico process is running.
    
    IPv4 BGP status
    +--------------+---------------+-------+----------+-------------+
    | PEER ADDRESS |   PEER TYPE   | STATE |  SINCE   |    INFO     |
    +--------------+---------------+-------+----------+-------------+
    | 10.10.16.47  | node specific | up    | 08:45:51 | Established |
    | 10.10.16.81  | node specific | up    | 09:35:25 | Established |
    +--------------+---------------+-------+----------+-------------+
    
    IPv6 BGP status
    No IPv6 peers found.
    
    root@ubuntu:~# 

    peer仍然存在

    [root@bogon ~]# ping  10.244.243.197
    PING 10.244.243.197 (10.244.243.197) 56(84) bytes of data.
    64 bytes from 10.244.243.197: icmp_seq=1 ttl=62 time=0.287 ms
    64 bytes from 10.244.243.197: icmp_seq=2 ttl=62 time=0.189 ms
    ^C
    --- 10.244.243.197 ping statistics ---
    2 packets transmitted, 2 received, 0% packet loss, time 1021ms
    rtt min/avg/max/mdev = 0.189/0.238/0.287/0.049 ms
    You have mail in /var/spool/mail/root
    [root@bogon ~]# ping 10.244.41.3
    PING 10.244.41.3 (10.244.41.3) 56(84) bytes of data.
    64 bytes from 10.244.41.3: icmp_seq=1 ttl=62 time=0.240 ms
    ^C
    --- 10.244.41.3 ping statistics ---
    1 packets transmitted, 1 received, 0% packet loss, time 0ms
    rtt min/avg/max/mdev = 0.240/0.240/0.240/0.000 ms
    [root@bogon ~]# 

    原来是有两个bgp peer删掉一个就可以了

    root@ubuntu:~# ./calicoctl get bgppeer
    NAME               PEERIP   NODE                ASN   
    rr1-to-node-peer            rr-group == 'rr1'   0     
    unbuntu                     all()               0     
    
    root@ubuntu:~# ./calicoctl delete  bgppeer  unbuntu 
    Successfully deleted 1 'BGPPeer' resource(s)
    root@ubuntu:~# ./calicoctl get bgppeer
    NAME               PEERIP   NODE                ASN   
    rr1-to-node-peer            rr-group == 'rr1'   0     
    
    root@ubuntu:~# ./calicoctl  node status
    Calico process is running.
    
    IPv4 BGP status
    No IPv4 peers found.
    
    IPv6 BGP status
    No IPv6 peers found.

    给bogon 添加label

    root@ubuntu:~# ./calicoctl  node status
    Calico process is running.
    
    IPv4 BGP status
    No IPv4 peers found.
    
    IPv6 BGP status
    No IPv6 peers found.
    
    root@ubuntu:~# ./calicoctl get bgppeer
    NAME               PEERIP   NODE                ASN   
    rr1-to-node-peer            rr-group == 'rr1'   0     
    
    root@ubuntu:~# kubectl get nodes --show-labels
    NAME     STATUS   ROLES    AGE     VERSION   LABELS
    bogon    Ready    worker   4d14h   v1.18.1   beta.kubernetes.io/arch=arm64,beta.kubernetes.io/os=linux,kubernetes.io/arch=arm64,kubernetes.io/hostname=bogon,kubernetes.io/os=linux,node-role.kubernetes.io/worker=worker
    cloud    Ready    worker   4d14h   v1.21.1   beta.kubernetes.io/arch=arm64,beta.kubernetes.io/os=linux,kubernetes.io/arch=arm64,kubernetes.io/hostname=cloud,kubernetes.io/os=linux,node-role.kubernetes.io/worker=worker
    ubuntu   Ready    master   4d14h   v1.18.1   beta.kubernetes.io/arch=arm64,beta.kubernetes.io/os=linux,kubernetes.io/arch=arm64,kubernetes.io/hostname=ubuntu,kubernetes.io/os=linux,node-role.kubernetes.io/master=,route-reflector=true,rr-group=rr1,rr-id=rr1
    root@ubuntu:~# kubectl label nodes bogon rr-group=rr1
    node/bogon labeled
    root@ubuntu:~# ./calicoctl get bgppeer
    NAME               PEERIP   NODE                ASN   
    rr1-to-node-peer            rr-group == 'rr1'   0     
    
    root@ubuntu:~# ./calicoctl  node status
    Calico process is running.
    
    IPv4 BGP status
    +--------------+---------------+-------+----------+-------------+
    | PEER ADDRESS |   PEER TYPE   | STATE |  SINCE   |    INFO     |
    +--------------+---------------+-------+----------+-------------+
    | 10.10.16.81  | node specific | up    | 01:32:08 | Established |
    +--------------+---------------+-------+----------+-------------+
    
    IPv6 BGP status
    No IPv6 peers found.
    
    root@ubuntu:~# 
    root@ubuntu:~# ./calicoctl  node status
    Calico process is running.
    
    IPv4 BGP status
    +--------------+---------------+-------+----------+-------------+
    | PEER ADDRESS |   PEER TYPE   | STATE |  SINCE   |    INFO     |
    +--------------+---------------+-------+----------+-------------+
    | 10.10.16.81  | node specific | up    | 01:32:08 | Established |
    +--------------+---------------+-------+----------+-------------+
    
    IPv6 BGP status
    No IPv6 peers found.
    
    root@ubuntu:~# ./calicoctl get nodes --output=wide
    NAME     ASN       IPV4             IPV6   
    bogon    (64512)   10.10.16.81/24          
    cloud    (64512)   10.10.16.47/24          
    ubuntu   (64512)   10.10.16.82/24          
    
    root@ubuntu:~# 
    [root@bogon ~]# ./calicoctl  node status
    Calico process is running.
    
    IPv4 BGP status
    +--------------+---------------+-------+----------+-------------+
    | PEER ADDRESS |   PEER TYPE   | STATE |  SINCE   |    INFO     |
    +--------------+---------------+-------+----------+-------------+
    | 10.10.16.82  | node specific | up    | 01:32:07 | Established |
    +--------------+---------------+-------+----------+-------------+
    
    IPv6 BGP status
    No IPv6 peers found.
    
    You have new mail in /var/spool/mail/root
    [root@bogon ~]# 
    root@cloud:~# ./calicoctl  node status
    Calico process is running.
    
    IPv4 BGP status
    No IPv4 peers found.
    
    IPv6 BGP status
    No IPv6 peers found.
    
    root@cloud:~# 
    [root@bogon ~]# ./calicoctl  node status
    Calico process is running.
    
    IPv4 BGP status
    +--------------+---------------+-------+----------+-------------+
    | PEER ADDRESS |   PEER TYPE   | STATE |  SINCE   |    INFO     |
    +--------------+---------------+-------+----------+-------------+
    | 10.10.16.82  | node specific | up    | 01:32:07 | Established |
    +--------------+---------------+-------+----------+-------------+
    
    IPv6 BGP status
    No IPv6 peers found.
    
    [root@bogon ~]#

    bogon pod ping cloud pod 和ubuntu pod

    [root@bogon ~]# ip a
    1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
        link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
        inet 127.0.0.1/8 scope host lo
           valid_lft forever preferred_lft forever
    2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000
        link/ipip 0.0.0.0 brd 0.0.0.0
    4: eth0@if103: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1440 qdisc noqueue state UP group default 
        link/ether fa:34:76:a4:9d:f8 brd ff:ff:ff:ff:ff:ff link-netnsid 0
        inet 10.244.29.4/32 brd 10.244.29.4 scope global eth0
           valid_lft forever preferred_lft forever
    [root@bogon ~]# ping  10.244.243.197
    PING 10.244.243.197 (10.244.243.197) 56(84) bytes of data.
    64 bytes from 10.244.243.197: icmp_seq=1 ttl=62 time=0.286 ms
    ^C
    --- 10.244.243.197 ping statistics ---
    1 packets transmitted, 1 received, 0% packet loss, time 0ms
    rtt min/avg/max/mdev = 0.286/0.286/0.286/0.000 ms
    [root@bogon ~]# ping 10.244.41.3
    PING 10.244.41.3 (10.244.41.3) 56(84) bytes of data.
    ^C
    --- 10.244.41.3 ping statistics ---
    3 packets transmitted, 0 received, 100% packet loss, time 2073ms
    
    [root@bogon ~]# 

     leaf节点

    用一条服务器模拟

    [root@localhost ~]# vtysh
    
    Hello, this is Quagga (version 0.99.22.4).
    Copyright 1996-2005 Kunihiro Ishiguro, et al.
    
    localhost.localdomain# show running-config
    Building configuration...
    
    Current configuration:
    !
    hostname localhost.localdomain
    hostname bgpd
    log stdout
    !
    password zebra
    !
    interface enp6s0
     ipv6 nd suppress-ra
    !
    interface enp125s0f0
     ipv6 nd suppress-ra
    !
    interface enp125s0f1
     ipv6 nd suppress-ra
    !
    interface enp125s0f2
     ipv6 nd suppress-ra
    !
    interface enp125s0f3
     ipv6 nd suppress-ra
    !
    interface lo
    !
    router bgp 7675
     bgp router-id 10.10.16.251
     neighbor 10.10.16.82 remote-as 64512
    !
    line vty
    !
    end
    localhost.localdomain# show  ip  bgp  summary
    BGP router identifier 10.10.16.251, local AS number 7675
    RIB entries 0, using 0 bytes of memory
    Peers 1, using 4560 bytes of memory
    
    Neighbor        V    AS MsgRcvd MsgSent   TblVer  InQ OutQ Up/Down  State/PfxRcd
    10.10.16.82     4 64512       0       9        0    0    0 never    Active     
    
    Total number of neighbors 1
    localhost.localdomain# 

    ubuntu配置leaf

    root@ubuntu:~# cat rr1-to-leaf1-peer.yaml
    apiVersion: projectcalico.org/v3
    kind: BGPPeer
    metadata:
      name: rr1-to-leaf1-peer        ## 给BGPPeer取一个名称,方便识别
    
    spec:
      nodeSelector: rr-id == 'rr1'   ## 通过节点选择器添加有rr-id == 'rr1'标签的节点
    
      peerIP: 10.10.16.251           ##  leaf01交换机的地址
      asNumber: 7675                ##  leaf01交换机的AS号
    root@ubuntu:~# ./calicoctl apply -f rr1-to-leaf1-peer.yaml
    Successfully applied 1 'BGPPeer' resource(s)
    root@ubuntu:~# ./calicoctl apply -f rr1-to-leaf1-peer.yaml
    Successfully applied 1 'BGPPeer' resource(s)

    配置完后,查看leaf

    [root@localhost ~]# route -n
    Kernel IP routing table
    Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
    0.0.0.0         10.10.16.254    0.0.0.0         UG    0      0        0 enp125s0f0
    10.10.16.0      0.0.0.0         255.255.255.0   U     0      0        0 enp125s0f0
    10.244.29.0     10.10.16.82     255.255.255.192 UG    0      0        0 enp125s0f0
    10.244.243.192  10.10.16.82     255.255.255.192 UG    0      0        0 enp125s0f0
    [root@localhost ~]# ping  10.244.243.197
    PING 10.244.243.197 (10.244.243.197) 56(84) bytes of data.
    64 bytes from 10.244.243.197: icmp_seq=1 ttl=63 time=0.188 ms
    64 bytes from 10.244.243.197: icmp_seq=2 ttl=63 time=0.137 ms
    64 bytes from 10.244.243.197: icmp_seq=3 ttl=63 time=0.109 ms
    ^C
    --- 10.244.243.197 ping statistics ---
    3 packets transmitted, 3 received, 0% packet loss, time 2054ms
    rtt min/avg/max/mdev = 0.109/0.144/0.188/0.035 ms
    [root@localhost ~]# 

    leaf节点ping bog   pod

    [root@localhost ~]# route -n
    Kernel IP routing table
    Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
    0.0.0.0         10.10.16.254    0.0.0.0         UG    0      0        0 enp125s0f0
    10.10.16.0      0.0.0.0         255.255.255.0   U     0      0        0 enp125s0f0
    10.244.29.0     10.10.16.82     255.255.255.192 UG    0      0        0 enp125s0f0
    10.244.243.192  10.10.16.82     255.255.255.192 UG    0      0        0 enp125s0f0
    [root@localhost ~]# ping  10.244.243.197
    PING 10.244.243.197 (10.244.243.197) 56(84) bytes of data.
    64 bytes from 10.244.243.197: icmp_seq=1 ttl=63 time=0.188 ms
    64 bytes from 10.244.243.197: icmp_seq=2 ttl=63 time=0.137 ms
    64 bytes from 10.244.243.197: icmp_seq=3 ttl=63 time=0.109 ms
    ^C
    --- 10.244.243.197 ping statistics ---
    3 packets transmitted, 3 received, 0% packet loss, time 2054ms
    rtt min/avg/max/mdev = 0.109/0.144/0.188/0.035 ms
    [root@localhost ~]# 

    leaf节点ping ubuntu  pod

    [root@localhost ~]# route -n
    Kernel IP routing table
    Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
    0.0.0.0         10.10.16.254    0.0.0.0         UG    0      0        0 enp125s0f0
    10.10.16.0      0.0.0.0         255.255.255.0   U     0      0        0 enp125s0f0
    10.244.29.0     10.10.16.82     255.255.255.192 UG    0      0        0 enp125s0f0  ----要经过10.10.16.82
    10.244.243.192  10.10.16.82     255.255.255.192 UG    0      0        0 enp125s0f0
    [root@localhost ~]# ping  10.244.243.197
    PING 10.244.243.197 (10.244.243.197) 56(84) bytes of data.
    64 bytes from 10.244.243.197: icmp_seq=1 ttl=63 time=0.188 ms
    64 bytes from 10.244.243.197: icmp_seq=2 ttl=63 time=0.137 ms
    64 bytes from 10.244.243.197: icmp_seq=3 ttl=63 time=0.109 ms
    ^C
    --- 10.244.243.197 ping statistics ---
    3 packets transmitted, 3 received, 0% packet loss, time 2054ms
    rtt min/avg/max/mdev = 0.109/0.144/0.188/0.035 ms
    [root@localhost ~]# ping  10.244.29.4
    PING 10.244.29.4 (10.244.29.4) 56(84) bytes of data.
    64 bytes from 10.244.29.4: icmp_seq=1 ttl=63 time=0.277 ms
    From 10.10.16.82: icmp_seq=2 Redirect Host(New nexthop: 10.10.16.81)
    64 bytes from 10.244.29.4: icmp_seq=2 ttl=63 time=0.226 ms
    ^C
    --- 10.244.29.4 ping statistics ---
    2 packets transmitted, 2 received, 0% packet loss, time 1019ms
    rtt min/avg/max/mdev = 0.226/0.251/0.277/0.030 ms
    [root@localhost ~]# 

    对于leaf节点ping bog   pod 进行tcpdump

    [root@localhost ~]# ping  10.244.29.4
    PING 10.244.29.4 (10.244.29.4) 56(84) bytes of data.
    64 bytes from 10.244.29.4: icmp_seq=1 ttl=63 time=0.287 ms
    From 10.10.16.82: icmp_seq=2 Redirect Host(New nexthop: 10.10.16.81)
    64 bytes from 10.244.29.4: icmp_seq=2 ttl=63 time=0.244 ms
    64 bytes from 10.244.29.4: icmp_seq=3 ttl=63 time=0.134 ms
    ^C
    --- 10.244.29.4 ping statistics ---
    3 packets transmitted, 3 received, 0% packet loss, time 2075ms
    rtt min/avg/max/mdev = 0.134/0.221/0.287/0.066 ms
    [root@localhost ~]# 

    ubunbtu

    root@ubuntu:~# tcpdump -i  enahisic2i0 icmp and host   10.244.29.4 -eenv
    tcpdump: listening on enahisic2i0, link-type EN10MB (Ethernet), capture size 262144 bytes

      从enahisic2i0进,
    10:49:30.999566 b0:08:75:5f:b8:5b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 19734, offset 0, flags [DF], proto ICMP (1), length 84) 10.10.16.251 > 10.244.29.4: ICMP echo request, id 4929, seq 1, length 64
    enahisic2i0出,
     10:49:30.999658 48:57:02:64:e7:ab > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 19734, offset 0, flags [DF], proto ICMP (1), length 84) 
    10.10.16.251 > 10.244.29.4: ICMP echo request, id 4929, seq 1, length 64

    10:49:32.034781 b0:08:75:5f:b8:5b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 19763, offset 0, flags [DF], proto ICMP (1), length 84) 10.10.16.251 > 10.244.29.4: ICMP echo request, id 4929, seq 2, length 64

    10:49:32.034863 48:57:02:64:e7:ab > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 19763, offset 0, flags [DF], proto ICMP (1), length 84) 10.10.16.251 > 10.244.29.4: ICMP echo request, id 4929, seq 2, length 64

    251 mac

    [root@localhost ~]# ip a sh  enp125s0f0
    2: enp125s0f0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
        link/ether b0:08:75:5f:b8:5b brd ff:ff:ff:ff:ff:ff
        inet 10.10.16.251/24 scope global enp125s0f0
           valid_lft forever preferred_lft forever
    [root@localhost ~]# 

    ubuntu mac 

    enahisic2i0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
        link/ether 48:57:02:64:e7:ab brd ff:ff:ff:ff:ff:ff
        inet 10.10.16.82/24 brd 10.10.16.255 scope global enahisic2i0
           valid_lft forever preferred_lft forever
        inet 10.10.16.250/32 scope global enahisic2i0
           valid_lft forever preferred_lft forever
        inet6 fe80::4a57:2ff:fe64:e7ab/64 scope link 
           valid_lft forever preferred_lft forever

    81 mac

    [root@bogon ~]# ip a sh enahisic2i0
    2: enahisic2i0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
        link/ether 48:57:02:64:ea:1b brd ff:ff:ff:ff:ff:ff
        inet 10.10.16.81/24 scope global enahisic2i0
           valid_lft forever preferred_lft forever
        inet 10.10.16.250/24 scope global secondary enahisic2i0
           valid_lft forever preferred_lft forever
    You have mail in /var/spool/mail/root
    [root@bogon ~]# 

    bogon pod 

    [root@bogon ~]# ip a
    1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
        link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
        inet 127.0.0.1/8 scope host lo
           valid_lft forever preferred_lft forever
    2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000
        link/ipip 0.0.0.0 brd 0.0.0.0
    4: eth0@if103: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1440 qdisc noqueue state UP group default 
        link/ether fa:34:76:a4:9d:f8 brd ff:ff:ff:ff:ff:ff link-netnsid 0
        inet 10.244.29.4/32 brd 10.244.29.4 scope global eth0
           valid_lft forever preferred_lft forever
    [root@bogon ~]# ping  10.244.243.197
    PING 10.244.243.197 (10.244.243.197) 56(84) bytes of data.
    64 bytes from 10.244.243.197: icmp_seq=1 ttl=62 time=0.286 ms
    ^C
    --- 10.244.243.197 ping statistics ---
    1 packets transmitted, 1 received, 0% packet loss, time 0ms
    rtt min/avg/max/mdev = 0.286/0.286/0.286/0.000 ms
    [root@bogon ~]# ping 10.244.41.3
    PING 10.244.41.3 (10.244.41.3) 56(84) bytes of data.
    ^C
    --- 10.244.41.3 ping statistics ---
    3 packets transmitted, 0 received, 100% packet loss, time 2073ms
    
    [root@bogon ~]# tcpdump -i  eth0 icmp and host   10.10.16.251  -eenv  --没有snat
    tcpdump: listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes
    10:49:31.006872 ee:ee:ee:ee:ee:ee > fa:34:76:a4:9d:f8, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 62, id 19734, offset 0, flags [DF], proto ICMP (1), length 84)
        10.10.16.251 > 10.244.29.4: ICMP echo request, id 4929, seq 1, length 64
    10:49:31.006906 fa:34:76:a4:9d:f8 > ee:ee:ee:ee:ee:ee, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 13073, offset 0, flags [none], proto ICMP (1), length 84)
        10.244.29.4 > 10.10.16.251: ICMP echo reply, id 4929, seq 1, length 64
    10:49:32.042047 ee:ee:ee:ee:ee:ee > fa:34:76:a4:9d:f8, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 62, id 19763, offset 0, flags [DF], proto ICMP (1), length 84)
        10.10.16.251 > 10.244.29.4: ICMP echo request, id 4929, seq 2, length 64
    10:49:32.042073 fa:34:76:a4:9d:f8 > ee:ee:ee:ee:ee:ee, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 13123, offset 0, flags [none], proto ICMP (1), length 84)
        10.244.29.4 > 10.10.16.251: ICMP echo reply, id 4929, seq 2, length 64
    10:49:33.081916 ee:ee:ee:ee:ee:ee > fa:34:76:a4:9d:f8, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 19767, offset 0, flags [DF], proto ICMP (1), length 84)
        10.10.16.251 > 10.244.29.4: ICMP echo request, id 4929, seq 3, length 64
    10:49:33.081945 fa:34:76:a4:9d:f8 > ee:ee:ee:ee:ee:ee, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 13153, offset 0, flags [none], proto ICMP (1), length 84)
        10.244.29.4 > 10.10.16.251: ICMP echo reply, id 4929, seq 3, length 64

    251 tcpdump 

    [root@localhost ~]# tcpdump -i enp125s0f0 icmp and host 10.244.29.4 -eennvv
    tcpdump: listening on enp125s0f0, link-type EN10MB (Ethernet), capture size 262144 bytes
    22:57:20.268299 b0:08:75:5f:b8:5b > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 37937, offset 0, flags [DF], proto ICMP (1), length 84)
        10.10.16.251 > 10.244.29.4: ICMP echo request, id 4954, seq 1, length 64
    22:57:20.268462 48:57:02:64:ea:1b > b0:08:75:5f:b8:5b, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 31657, offset 0, flags [none], proto ICMP (1), length 84)
        10.244.29.4 > 10.10.16.251: ICMP echo reply, id 4954, seq 1, length 64
    22:57:21.322067 b0:08:75:5f:b8:5b > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 37948, offset 0, flags [DF], proto ICMP (1), length 84)
        10.10.16.251 > 10.244.29.4: ICMP echo request, id 4954, seq 2, length 64
    22:57:21.322198 48:57:02:64:ea:1b > b0:08:75:5f:b8:5b, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 31669, offset 0, flags [none], proto ICMP (1), length 84)
        10.244.29.4 > 10.10.16.251: ICMP echo reply, id 4954, seq 2, length 64

    ubuntu只有 icmp request ,reply 不经过ubuntu

    root@ubuntu:~# tcpdump -i  enahisic2i0 icmp and host   10.244.29.4 -eenv
    tcpdump: listening on enahisic2i0, link-type EN10MB (Ethernet), capture size 262144 bytes
    10:49:30.999566 b0:08:75:5f:b8:5b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 19734, offset 0, flags [DF], proto ICMP (1), length 84)
        10.10.16.251 > 10.244.29.4: ICMP echo request, id 4929, seq 1, length 64
    10:49:30.999658 48:57:02:64:e7:ab > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 19734, offset 0, flags [DF], proto ICMP (1), length 84)
        10.10.16.251 > 10.244.29.4: ICMP echo request, id 4929, seq 1, length 64
    10:49:32.034781 b0:08:75:5f:b8:5b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 19763, offset 0, flags [DF], proto ICMP (1), length 84)
        10.10.16.251 > 10.244.29.4: ICMP echo request, id 4929, seq 2, length 64
    10:49:32.034863 48:57:02:64:e7:ab > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 19763, offset 0, flags [DF], proto ICMP (1), length 84)
        10.10.16.251 > 10.244.29.4: ICMP echo request, id 4929, seq 2, length 64
    10:56:05.307169 b0:08:75:5f:b8:5b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 32478, offset 0, flags [DF], proto ICMP (1), length 84)
        10.10.16.251 > 10.244.29.4: ICMP echo request, id 4936, seq 1, length 64
    10:56:05.307308 48:57:02:64:e7:ab > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 32478, offset 0, flags [DF], proto ICMP (1), length 84)
        10.10.16.251 > 10.244.29.4: ICMP echo request, id 4936, seq 1, length 64
    10:56:06.354681 b0:08:75:5f:b8:5b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 32536, offset 0, flags [DF], proto ICMP (1), length 84)
        10.10.16.251 > 10.244.29.4: ICMP echo request, id 4936, seq 2, length 64
    10:56:06.354747 48:57:02:64:e7:ab > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 32536, offset 0, flags [DF], proto ICMP (1), length 84)
        10.10.16.251 > 10.244.29.4: ICMP echo request, id 4936, seq 2, length 64
    ^C
    [root@bogon ~]# ./calicoctl  node status
    Calico process is running.
    
    IPv4 BGP status
    +--------------+---------------+-------+----------+-------------+
    | PEER ADDRESS |   PEER TYPE   | STATE |  SINCE   |    INFO     |
    +--------------+---------------+-------+----------+-------------+
    | 10.10.16.82  | node specific | up    | 01:32:07 | Established |
    +--------------+---------------+-------+----------+-------------+
    
    IPv6 BGP status
    No IPv6 peers found.

    给251新增一个网段

    [root@localhost ~]# ip a add 172.168.19.251/24 dev  enp125s0f1
    [root@localhost ~]# ip a
    1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
        link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
        inet 127.0.0.1/8 scope host lo
           valid_lft forever preferred_lft forever
        inet6 ::1/128 scope host 
           valid_lft forever preferred_lft forever
    2: enp125s0f0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
        link/ether b0:08:75:5f:b8:5b brd ff:ff:ff:ff:ff:ff
        inet 10.10.16.251/24 scope global enp125s0f0
           valid_lft forever preferred_lft forever

    ubuntu还无法访问这个网段

    root@ubuntu:~# route -n
    Kernel IP routing table
    Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
    0.0.0.0         10.10.16.254    0.0.0.0         UG    0      0        0 enahisic2i0
    10.10.16.0      0.0.0.0         255.255.255.0   U     0      0        0 enahisic2i0
    10.244.29.0     10.10.16.81     255.255.255.192 UG    0      0        0 enahisic2i0
    10.244.243.192  0.0.0.0         255.255.255.192 U     0      0        0 *
    10.244.243.194  0.0.0.0         255.255.255.255 UH    0      0        0 cali0d27bc8b0f7
    10.244.243.197  0.0.0.0         255.255.255.255 UH    0      0        0 cali090a0c3c4a2
    172.17.0.0      0.0.0.0         255.255.0.0     U     0      0        0 docker0
    192.168.122.0   0.0.0.0         255.255.255.0   U     0      0        0 virbr0
    root@ubuntu:~# 

    251节点上把172.168.19.251/24这个网段发送出去

    localhost.localdomain# conf t
    localhost.localdomain(config)# router bgp  7675
    localhost.localdomain(config-router)#  network 172.168.19.251/24
    localhost.localdomain(config-router)# exit
    localhost.localdomain(config)# wr
    % Unknown command.
    localhost.localdomain(config)# exit
    localhost.localdomain# wr
    Building Configuration...
    Can't open configuration file /etc/quagga/zebra.conf.SbhiyG.
    Can't open configuration file /etc/quagga/bgpd.conf.omycEG.
    [OK]
    localhost.localdomain# 

    ubuntu路由信息

    root@ubuntu:~# route -n
    Kernel IP routing table
    Destination Gateway Genmask Flags Metric Ref Use Iface
    0.0.0.0 10.10.16.254 0.0.0.0 UG 0 0 0 enahisic2i0
    10.10.16.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i0
    10.244.29.0 10.10.16.81 255.255.255.192 UG 0 0 0 enahisic2i0
    10.244.243.192 0.0.0.0 255.255.255.192 U 0 0 0 *
    10.244.243.194 0.0.0.0 255.255.255.255 UH 0 0 0 cali0d27bc8b0f7
    10.244.243.197 0.0.0.0 255.255.255.255 UH 0 0 0 cali090a0c3c4a2
    172.17.0.0 0.0.0.0 255.255.0.0 U 0 0 0 docker0
    172.168.19.0 10.10.16.251 255.255.255.0 UG 0 0 0 enahisic2i0
    192.168.122.0 0.0.0.0 255.255.255.0 U 0 0 0 virbr0
    root@ubuntu:~#

    ubuntu  ping   172.168.19.251

    root@ubuntu:~#  ping 172.168.19.251
    PING 172.168.19.251 (172.168.19.251) 56(84) bytes of data.
    64 bytes from 172.168.19.251: icmp_seq=1 ttl=64 time=0.124 ms
    64 bytes from 172.168.19.251: icmp_seq=2 ttl=64 time=0.083 ms
    64 bytes from 172.168.19.251: icmp_seq=3 ttl=64 time=0.074 ms
    ^C
    --- 172.168.19.251 ping statistics ---
    3 packets transmitted, 3 received, 0% packet loss, time 2051ms
    rtt min/avg/max/mdev = 0.074/0.093/0.124/0.024 ms
    root@ubuntu:~# 
    [root@localhost ~]# tcpdump -i enp125s0f0 icmp and host 172.168.19.251 -eennvv
    tcpdump: listening on enp125s0f0, link-type EN10MB (Ethernet), capture size 262144 bytes
    23:31:13.314954 48:57:02:64:e7:ab > b0:08:75:5f:b8:5b, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 20566, offset 0, flags [DF], proto ICMP (1), length 84)
        10.10.16.82 > 172.168.19.251: ICMP echo request, id 45785, seq 1, length 64
    23:31:13.314964 b0:08:75:5f:b8:5b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 23367, offset 0, flags [none], proto ICMP (1), length 84)
        172.168.19.251 > 10.10.16.82: ICMP echo reply, id 45785, seq 1, length 64
    23:31:14.319749 48:57:02:64:e7:ab > b0:08:75:5f:b8:5b, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 20813, offset 0, flags [DF], proto ICMP (1), length 84)
        10.10.16.82 > 172.168.19.251: ICMP echo request, id 45785, seq 2, length 64
    23:31:14.319756 b0:08:75:5f:b8:5b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 23438, offset 0, flags [none], proto ICMP (1), length 84)
        172.168.19.251 > 10.10.16.82: ICMP echo reply, id 45785, seq 2, length 64

    bogon 访问172.168.19.251 仍然不通,因为ubuntu没有把路由同步给bogon

    [root@bogon ~]# ping 172.168.19.251
    PING 172.168.19.251 (172.168.19.251) 56(84) bytes of data.
    ^C
    --- 172.168.19.251 ping statistics ---
    2 packets transmitted, 0 received, 100% packet loss, time 1056ms
    
    [root@bogon ~]#
    [root@bogon ~]# route -n
    Kernel IP routing table
    Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
    0.0.0.0         10.10.16.254    0.0.0.0         UG    0      0        0 enahisic2i0
    10.2.0.1        0.0.0.0         255.255.255.255 UH    0      0        0 v-cali-peer
    10.10.16.0      0.0.0.0         255.255.255.0   U     0      0        0 enahisic2i0
    10.10.34.0      0.0.0.0         255.255.255.0   U     0      0        0 enahisic2i2
    10.10.102.0     0.0.0.0         255.255.255.0   U     0      0        0 enahisic2i1
    10.244.29.0     0.0.0.0         255.255.255.192 U     0      0        0 *
    10.244.29.1     0.0.0.0         255.255.255.255 UH    0      0        0 cali2e486421e22
    10.244.29.4     0.0.0.0         255.255.255.255 UH    0      0        0 calibe3388252a1
    10.244.243.192  10.10.16.82     255.255.255.192 UG    0      0        0 enahisic2i0
    14.14.18.0      0.0.0.0         255.255.255.0   U     0      0        0 enahisic2i3.310
    172.16.100.0    0.0.0.0         255.255.255.0   U     0      0        0 brqf1411bad-10
    172.17.0.0      0.0.0.0         255.255.0.0     U     0      0        0 docker0
    172.168.104.0   0.0.0.0         255.255.255.0   U     0      0        0 enah2i3.1022
    192.168.33.0    0.0.0.0         255.255.255.0   U     0      0        0 enahisic2i1
    192.168.122.0   0.0.0.0         255.255.255.0   U     0      0        0 virbr0
    You have mail in /var/spool/mail/root
    [root@bogon ~]# route -n | grep  172.168.19
    [root@bogon ~]# tcpdump -i enahisic2i0 icmp and  host 172.168.19.251 -eennvv
    tcpdump: listening on enahisic2i0, link-type EN10MB (Ethernet), capture size 262144 bytes
    11:31:08.028856 48:57:02:64:ea:1b > f4:1d:6b:87:53:2a, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 44135, offset 0, flags [DF], proto ICMP (1), length 84)
        10.10.16.81 > 172.168.19.251: ICMP echo request, id 0, seq 1, length 64
    11:31:09.092515 48:57:02:64:ea:1b > f4:1d:6b:87:53:2a, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 44165, offset 0, flags [DF], proto ICMP (1), length 84)
        10.10.16.81 > 172.168.19.251: ICMP echo request, id 0, seq 2, length 64
    ^C
    2 packets captured
    2 packets received by filter
    0 packets dropped by kernel
    You have mail in /var/spool/mail/root
    [root@bogon ~]# tcpdump -i enahisic2i0 icmp and  host 172.168.19.251 -eennvv
    tcpdump: listening on enahisic2i0, link-type EN10MB (Ethernet), capture size 262144 bytes
    11:33:01.475621 48:57:02:64:ea:1b > f4:1d:6b:87:53:2a, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 15976, offset 0, flags [DF], proto ICMP (1), length 84)
        10.10.16.81 > 172.168.19.251: ICMP echo request, id 45862, seq 1, length 64
    11:33:02.532507 48:57:02:64:ea:1b > f4:1d:6b:87:53:2a, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 16021, offset 0, flags [DF], proto ICMP (1), length 84)
        10.10.16.81 > 172.168.19.251: ICMP echo request, id 45862, seq 2, length 64
    ^C
    2 packets captured
    2 packets received by filter
    0 packets dropped by kernel
    [root@bogon ~]# 

    将251节点的as number 改成 64512

    localhost.localdomain# show running-config
    Building configuration...
    
    Current configuration:
    !
    hostname localhost.localdomain
    hostname bgpd
    log stdout
    !
    password zebra
    !
    interface enp6s0
     ipv6 nd suppress-ra
    !
    interface enp125s0f0
     ipv6 nd suppress-ra
    !
    interface enp125s0f1
     ipv6 nd suppress-ra
    !
    interface enp125s0f2
     ipv6 nd suppress-ra
    !
    interface enp125s0f3
     ipv6 nd suppress-ra
    !
    interface lo
    !
    router bgp 7675
     bgp router-id 10.10.16.251
     network 172.168.19.0/24
     neighbor 10.10.16.82 remote-as 64512
    !
    line vty
    !
    end
    localhost.localdomain# conf t
    localhost.localdomain(config)# no router bgp 7675 
    localhost.localdomain(config)# router bgp 64512
    localhost.localdomain(config-router)# exit
    localhost.localdomain(config)# wr
    % Unknown command.
    localhost.localdomain(config)# exit
    localhost.localdomain# wr
    Building Configuration...
    Can't open configuration file /etc/quagga/zebra.conf.ofjRYH.
    Can't open configuration file /etc/quagga/bgpd.conf.e0jI8H.
    [OK]
    localhost.localdomain#  show running-config
    Building configuration...
    
    Current configuration:
    !
    hostname localhost.localdomain
    hostname bgpd
    log stdout
    !
    password zebra
    !
    interface enp6s0
     ipv6 nd suppress-ra
    !
    interface enp125s0f0
     ipv6 nd suppress-ra
    !
    interface enp125s0f1
     ipv6 nd suppress-ra
    !
    interface enp125s0f2
     ipv6 nd suppress-ra
    !
    interface enp125s0f3
     ipv6 nd suppress-ra
    !
    interface lo
    !
    router bgp 64512
     bgp router-id 10.10.16.251
    !
    line vty
    !
    end
    localhost.localdomain#

     改完之后ubuntu没有172.168.19.251/24的路由了

    root@ubuntu:~# route -n
    Kernel IP routing table
    Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
    0.0.0.0         10.10.16.254    0.0.0.0         UG    0      0        0 enahisic2i0
    10.10.16.0      0.0.0.0         255.255.255.0   U     0      0        0 enahisic2i0
    10.244.29.0     10.10.16.81     255.255.255.192 UG    0      0        0 enahisic2i0
    10.244.243.192  0.0.0.0         255.255.255.192 U     0      0        0 *
    10.244.243.194  0.0.0.0         255.255.255.255 UH    0      0        0 cali0d27bc8b0f7
    10.244.243.197  0.0.0.0         255.255.255.255 UH    0      0        0 cali090a0c3c4a2
    172.17.0.0      0.0.0.0         255.255.0.0     U     0      0        0 docker0
    192.168.122.0   0.0.0.0         255.255.255.0   U     0      0        0 virbr0
    root@ubuntu:~# ./calicoctl get bgppeer
    NAME                PEERIP         NODE                ASN    
    rr1-to-leaf1-peer   10.10.16.251   rr-id == 'rr1'      7675   
    rr1-to-node-peer                   rr-group == 'rr1'   0      
    
    root@ubuntu:~# ./calicoctl get bgppeer
    NAME                PEERIP         NODE                ASN    
    rr1-to-leaf1-peer   10.10.16.251   rr-id == 'rr1'      7675   
    rr1-to-node-peer                   rr-group == 'rr1'   0      
    
    root@ubuntu:~# 

    原因是251没有邻居了

    localhost.localdomain# show  ip  bgp  summary
    No IPv4 neighbor is configured
    localhost.localdomain# 

    给251 配置bgp邻居

    localhost.localdomain# show  ip  bgp  summary
    No IPv4 neighbor is configured
    localhost.localdomain# conf t
    localhost.localdomain(config)# router bgp 64512
    localhost.localdomain(config-router)# neighbor 10.10.16.82 remote-as 64512
    localhost.localdomain(config-router)# network 172.168.19.251/24
    localhost.localdomain(config-router)# exit
    localhost.localdomain(config)# exit
    localhost.localdomain# wr
    Building Configuration...
    Can't open configuration file /etc/quagga/zebra.conf.9iI9n9.
    Can't open configuration file /etc/quagga/bgpd.conf.xLFCy9.
    [OK]
    localhost.localdomain# 

    更改交换机as

    root@ubuntu:~# cat  rr1-to-leaf1-peer.yaml 
    apiVersion: projectcalico.org/v3
    kind: BGPPeer
    metadata:
      name: rr1-to-leaf1-peer        ## 给BGPPeer取一个名称,方便识别
    
    spec:
      nodeSelector: rr-id == 'rr1'   ## 通过节点选择器添加有rr-id == 'rr1'标签的节点
    
      peerIP: 10.10.16.251           ##  leaf01交换机的地址
      asNumber:  64512                ##  leaf01交换机的AS号
    root@ubuntu:~# 
    root@ubuntu:~# ./calicoctl apply -f rr1-to-leaf1-peer.yaml 
    Successfully applied 1 'BGPPeer' resource(s)
    root@ubuntu:~# route -n
    Kernel IP routing table
    Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
    0.0.0.0         10.10.16.254    0.0.0.0         UG    0      0        0 enahisic2i0
    10.10.16.0      0.0.0.0         255.255.255.0   U     0      0        0 enahisic2i0
    10.244.29.0     10.10.16.81     255.255.255.192 UG    0      0        0 enahisic2i0
    10.244.243.192  0.0.0.0         255.255.255.192 U     0      0        0 *
    10.244.243.194  0.0.0.0         255.255.255.255 UH    0      0        0 cali0d27bc8b0f7
    10.244.243.197  0.0.0.0         255.255.255.255 UH    0      0        0 cali090a0c3c4a2
    172.17.0.0      0.0.0.0         255.255.0.0     U     0      0        0 docker0
    172.168.19.0    10.10.16.251    255.255.255.0   UG    0      0        0 enahisic2i0
    192.168.122.0   0.0.0.0         255.255.255.0   U     0      0        0 virbr0
    root@ubuntu:~#  ping 172.168.19.251
    PING 172.168.19.251 (172.168.19.251) 56(84) bytes of data.
    64 bytes from 172.168.19.251: icmp_seq=1 ttl=64 time=0.292 ms
    64 bytes from 172.168.19.251: icmp_seq=2 ttl=64 time=0.079 ms
    ^C
    --- 172.168.19.251 ping statistics ---
    2 packets transmitted, 2 received, 0% packet loss, time 1011ms
    rtt min/avg/max/mdev = 0.079/0.185/0.292/0.107 ms
    root@ubuntu:~# ./calicoctl get bgppeer
    NAME                PEERIP         NODE                ASN     
    rr1-to-leaf1-peer   10.10.16.251   rr-id == 'rr1'      64512   
    rr1-to-node-peer                   rr-group == 'rr1'   0       

    bogon仍然没有172.168.19的路由

    [root@bogon ~]# route -n | grep 172.168.19
    [root@bogon ~]# route -n | grep 172.168.19
    [root@bogon ~]# 

    251改造成k8s node

    root@ubuntu:~# kubectl get nodes -o wide
    NAME      STATUS   ROLES    AGE     VERSION   INTERNAL-IP    EXTERNAL-IP   OS-IMAGE                   KERNEL-VERSION                CONTAINER-RUNTIME
    bogon     Ready    worker   4d19h   v1.18.1   10.10.16.81    <none>        CentOS Linux 7 (AltArch)   4.14.0-115.8.1.el7a.aarch64   docker://20.10.7
    centos7   Ready    <none>   20m     v1.18.1   10.10.16.251   <none>        CentOS Linux 7 (AltArch)   4.14.0-115.el7a.0.1.aarch64   docker://1.13.1
    cloud     Ready    worker   4d20h   v1.21.1   10.10.16.47    <none>        Ubuntu 18.04.3 LTS         5.5.19-050519-generic         docker://19.3.13
    ubuntu    Ready    master   4d20h   v1.18.1   10.10.16.82    <none>        Ubuntu 18.04.3 LTS         5.0.0-23-generic              containerd://1.3.7
    root@ubuntu:~# 
    root@ubuntu:~# ./calicoctl node status
    Calico process is running.
    
    IPv4 BGP status
    +--------------+---------------+-------+----------+--------------------------------+
    | PEER ADDRESS |   PEER TYPE   | STATE |  SINCE   |              INFO              |
    +--------------+---------------+-------+----------+--------------------------------+
    | 10.10.16.251 | node specific | start | 07:14:44 | Idle Socket: Connection        |
    |              |               |       |          | refused                        |
    | 14.14.18.89  | node specific | start | 07:14:44 | Connect                        |
    +--------------+---------------+-------+----------+--------------------------------+
    root@ubuntu:~# ./calicoctl get  bgppeer
    NAME                PEERIP         NODE                ASN     
    rr1-to-leaf1-peer   10.10.16.251   rr-id == 'rr1'      64512   
    rr1-to-node-peer                   rr-group == 'rr1'   0    

    删掉bgp pper

    root@ubuntu:~# ./calicoctl delete bgppeer rr1-to-leaf1-peer
    Successfully deleted 1 'BGPPeer' resource(s)
    root@ubuntu:~# kubectl get nodes -o wide
    NAME      STATUS   ROLES    AGE     VERSION   INTERNAL-IP    EXTERNAL-IP   OS-IMAGE                   KERNEL-VERSION                CONTAINER-RUNTIME
    bogon     Ready    worker   4d20h   v1.18.1   10.10.16.81    <none>        CentOS Linux 7 (AltArch)   4.14.0-115.8.1.el7a.aarch64   docker://20.10.7
    centos7   Ready    <none>   21m     v1.18.1   10.10.16.251   <none>        CentOS Linux 7 (AltArch)   4.14.0-115.el7a.0.1.aarch64   docker://1.13.1
    cloud     Ready    worker   4d20h   v1.21.1   10.10.16.47    <none>        Ubuntu 18.04.3 LTS         5.5.19-050519-generic         docker://19.3.13
    ubuntu    Ready    master   4d20h   v1.18.1   10.10.16.82    <none>        Ubuntu 18.04.3 LTS         5.0.0-23-generic              containerd://1.3.7
    root@ubuntu:~# ./calicoctl get  bgppeer
    NAME               PEERIP   NODE                ASN   
    rr1-to-node-peer            rr-group == 'rr1'   0     
    
    root@ubuntu:~# 
    root@ubuntu:~# kubectl get pods -o wide   -n kube-system
    NAME                                       READY   STATUS    RESTARTS   AGE     IP               NODE      NOMINATED NODE   READINESS GATES
    calico-kube-controllers-5978c5f6b5-tk6pg   1/1     Running   0          4d20h   10.244.243.194   ubuntu    <none>           <none>
    calico-node-gnp9x                          0/1     Running   0          59m     10.10.16.82      ubuntu    <none>           <none>
    calico-node-jcvsf                          1/1     Running   0          59m     10.10.16.251     centos7   <none>           <none>
    calico-node-pq756                          0/1     Running   0          59m     10.10.16.81      bogon     <none>           <none>
    calico-node-pxr58                          1/1     Running   0          52m     10.10.16.47      cloud     <none>           <none>
    bird: Reconfiguration requested by SIGHUP
    bird: Reconfiguring
    bird: device1: Reconfigured
    bird: direct1: Reconfigured
    bird: Adding protocol Node_14_14_18_89
    bird: Node_14_14_18_89: Initializing
    bird: Node_14_14_18_89: Starting
    bird: Node_14_14_18_89: State changed to start
    bird: Reconfigured
    2021-06-23 08:03:04.084 [INFO][90] felix/summary.go 100: Summarising 9 dataplane reconciliation loops over 1m2.3s: avg=5ms longest=11ms (resync-nat-v4,resync-raw-v4)
    2021-06-23 08:03:12.698 [WARNING][90] felix/int_dataplane.go 878: Failed to auto-detect host MTU - no interfaces matched the MTU interface pattern. To use auto-MTU, set mtuIfacePattern to match your host's interfaces
    2021-06-23 08:03:42.548 [INFO][91] monitor-addresses/startup.go 788: Using autodetected IPv4 address 10.10.16.82/24 on matching interface enahisic2i0
    2021-06-23 08:03:42.699 [WARNING][90] felix/int_dataplane.go 878: Failed to auto-detect host MTU - no interfaces matched the MTU interface pattern. To use auto-MTU, set mtuIfacePattern to match your host's interfaces
    2021-06-23 08:04:07.433 [INFO][90] felix/summary.go 100: Summarising 11 dataplane reconciliation loops over 1m3.3s: avg=5ms longest=16ms ()
    2021-06-23 08:04:12.702 [WARNING][90] felix/int_dataplane.go 878: Failed to auto-detect host MTU - no interfaces matched the MTU interface pattern. To use auto-MTU, set mtuIfacePattern to match your host's interfaces
    2021-06-23 08:04:42.549 [INFO][91] monitor-addresses/startup.go 788: Using autodetected IPv4 address 10.10.16.82/24 on matching interface enahisic2i0
    2021-06-23 08:04:42.703 [WARNING][90] felix/int_dataplane.go 878: Failed to auto-detect host MTU - no interfaces matched the MTU interface pattern. To use auto-MTU, set mtuIfacePattern to match your host's interfaces
    2021-06-23 08:05:09.720 [INFO][90] felix/summary.go 100: Summarising 8 dataplane reconciliation loops over 1m2.3s: avg=4ms longest=19ms ()
    2021-06-23 08:05:12.705 [WARNING][90] felix/int_dataplane.go 878: Failed to auto-detect host MTU - no interfaces matched the MTU interface pattern. To use auto-MTU, set mtuIfacePattern to match your host's interfaces
    2021-06-23 08:05:42.552 [INFO][91] monitor-addresses/startup.go 788: Using autodetected IPv4 address 10.10.16.82/24 on matching interface enahisic2i0
    2021-06-23 08:05:42.706 [WARNING][90] felix/int_dataplane.go 878: Failed to auto-detect host MTU - no interfaces matched the MTU interface pattern. To use auto-MTU, set mtuIfacePattern to match your host's interfaces
    2021-06-23 08:06:11.983 [INFO][90] felix/summary.go 100: Summarising 9 dataplane reconciliation loops over 1m2.3s: avg=4ms longest=12ms (resync-nat-v4,resync-raw-v4)
    2021-06-23 08:06:12.708 [WARNING][90] felix/int_dataplane.go 878: Failed to auto-detect host MTU - no interfaces matched the MTU interface pattern. To use auto-MTU, set mtuIfacePattern to match your host's interfaces
    2021-06-23 08:06:36.324 [INFO][93] confd/watchercache.go 96: Watch channel closed by remote - recreate watcher ListRoot="/calico/ipam/v2/host/ubuntu"
    2021-06-23 08:06:42.554 [INFO][91] monitor-addresses/startup.go 788: Using autodetected IPv4 address 10.10.16.82/24 on matching interface enahisic2i0
    2021-06-23 08:06:42.709 [WARNING][90] felix/int_dataplane.go 878: Failed to auto-detect host MTU - no interfaces matched the MTU interface pattern. To use auto-MTU, set mtuIfacePattern to match your host's interfaces
    2021-06-23 08:07:09.406 [INFO][90] felix/watchercache.go 96: Watch channel closed by remote - recreate watcher ListRoot="/calico/resources/v3/projectcalico.org/profiles"
    2021-06-23 08:07:12.710 [WARNING][90] felix/int_dataplane.go 878: Failed to auto-detect host MTU - no interfaces matched the MTU interface pattern. To use auto-MTU, set mtuIfacePattern to match your host's interfaces
    2021-06-23 08:07:15.071 [INFO][90] felix/summary.go 100: Summarising 11 dataplane reconciliation loops over 1m3.1s: avg=5ms longest=20ms ()
    2021-06-23 08:07:42.557 [INFO][91] monitor-addresses/startup.go 788: Using autodetected IPv4 address 10.10.16.82/24 on matching interface enahisic2i0
    2021-06-23 08:07:42.711 [WARNING][90] felix/int_dataplane.go 878: Failed to auto-detect host MTU - no interfaces matched the MTU interface pattern. To use auto-MTU, set mtuIfacePattern to match your host's interfaces
    2021-06-23 08:08:06.739 [INFO][93] confd/watchercache.go 96: Watch channel closed by remote - recreate watcher ListRoot="/calico/resources/v3/projectcalico.org/bgppeers"
    2021-06-23 08:08:12.712 [WARNING][90] felix/int_dataplane.go 878: Failed to auto-detect host MTU - no interfaces matched the MTU interface pattern. To use auto-MTU, set mtuIfacePattern to match your host's interfaces
    2021-06-23 08:08:18.442 [INFO][90] felix/summary.go 100: Summarising 8 dataplane reconciliation loops over 1m3.4s: avg=4ms longest=16ms ()
    2021-06-23 08:08:42.558 [INFO][91] monitor-addresses/startup.go 788: Using autodetected IPv4 address 10.10.16.82/24 on matching interface enahisic2i0
    2021-06-23 08:08:42.713 [WARNING][90] felix/int_dataplane.go 878: Failed to auto-detect host MTU - no interfaces matched the MTU interface pattern. To use auto-MTU, set mtuIfacePattern to match your host's interfaces
    ^C
    root@ubuntu:~# kubectl logs  -f  calico-node-gnp9x  -n kube-system

    编辑kubectl edit ds calico-node -n kube-system

            - name: IP_AUTODETECTION_METHOD
              value: "can-reach=10.10.16.254"
              #value: interface=en.*

    成功了

    root@ubuntu:~# ./calicoctl node status
    Calico process is running.
    
    IPv4 BGP status
    +--------------+---------------+-------+----------+-------------+
    | PEER ADDRESS |   PEER TYPE   | STATE |  SINCE   |    INFO     |
    +--------------+---------------+-------+----------+-------------+
    | 10.10.16.81  | node specific | up    | 08:25:14 | Established |
    +--------------+---------------+-------+----------+-------------+
    
    IPv6 BGP status
    No IPv6 peers found.
    
    root@ubuntu:~# 

    Calico配置及原理

     

    Kubernetes-Calico百度云实践:集群外RR模式(微信)




  • 相关阅读:
    今天开始用 VSU 2010
    Visual Studio 2010 模型设计工具 基本应用
    Asp.Net访问Oracle 数据库 执行SQL语句和调用存储过程
    Enterprise Library 4.1 Security Block 快速使用图文笔记
    解决“System.Data.OracleClient 需要 Oracle 客户端软件 8.1.7 或更高版本。”(图)
    一个Oracle存储过程示例
    Enterprise Library 4.1 Application Settings 快速使用图文笔记
    Oracle 10g for Windows 简体中文版的安装过程
    Oracle 11g for Windows 简体中文版的安装过程
    Oracle 9i 数据库 创建数据库 Net 配置 创建表 SQL查询 创建存储过程 (图)
  • 原文地址:https://www.cnblogs.com/dream397/p/14919315.html
Copyright © 2011-2022 走看看