ovs流表机制（四）用vxlan实现多个节点的vm通信（二）

 新增一个节点10.10.18.227

 

 

10.10.16.82通过vxlan vni=48和vxlan vni=32分别连接10.10.18.227和10.10.18.216

10.10.28.227网络配置如下

#/bin/bash
set -e
#添加ovs网桥: br-tun/br-int
#ovs-vsctl add-br br-tun
#ovs-vsctl add-br br-int
#ovs-vsctl set-fail-mode br-tun secure
#ovs-vsctl set-fail-mode br-int standalone
#添加patch口，连接br-tun,br-int
ovs-vsctl add-port br-tun patch-int -- set interface patch-int type=patch -- set interface patch-int options:peer=patch-tun
ovs-vsctl add-port br-int patch-tun -- set interface patch-tun type=patch -- set interface patch-tun options:peer=patch-int
#添加linux bridge
brctl addbr qbr1
ip link set qbr1 up
#添加veth口，连接br-int,qbr1
ip link add name qvo1 type veth peer name qvb1
ip link set qvo1 up
ip link set qvb1 up
ovs-vsctl add-port br-int qvo1
brctl addif qbr1 qvb1
#设置qvo1的vlan tag
ovs-vsctl set port qvo1 tag=100
#添加namespace：
ip netns add ns1
ip link add name veth1 type veth peer name veth1_br
ip link set dev veth1 netns ns1
brctl addif qbr1 veth1_br
ip link set veth1_br up
ip netns exec ns1 ip link set veth1 up
ip netns exec ns1 ip addr add 192.168.10.32/24 dev veth1


ovs-vsctl add-port br-tun vxlan-01 -- set interface vxlan-01 type=vxlan options:remote_ip=10.10.16.82 options:key=0x30  ofport_request=2

10.10.28.227流表配置如下

 ## patch-int port =1 
 ## vxlan-01 port =2
 ##================== table 0
 #处理patch-int包，虚机出流量
 ovs-ofctl add-flow br-tun 'cookie=0x79, table=0,  priority=1,in_port=1 actions=resubmit(,2)'
 #处理vxlan包，从vxlan tunnel进入的数据包
 ovs-ofctl add-flow br-tun 'cookie=0x79, table=0,  priority=1,in_port=2 actions=resubmit(,4)'
 #默认规则
 ovs-ofctl add-flow br-tun 'cookie=0x79, table=0, priority=0 actions=drop'
#处理patch-int包，虚机出流量 table 0 --> resubmit 2

#===========table 2
 #单播包
 ovs-ofctl add-flow br-tun 'cookie=0x79,  table=2, priority=0,dl_dst=00:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,20)'

 #广播包
 ovs-ofctl add-flow br-tun 'cookie=0x79,  table=2, priority=0,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,22)'
#=================== table 3
ovs-ofctl add-flow br-tun 'cookie=0x79,  table=3, priority=0 actions=drop'
#处理vxlan包，从vxlan tunnel进入的数据包会resubmit(4)
#vxlan id =0x30, 10.10.18.227节点 vlan id =10
#=================== table 4
#######vxlan_id=0X30
 ovs-ofctl add-flow br-tun 'cookie=0x79,  table=4,  priority=1,tun_id=0x30 actions=mod_vlan_vid:10,resubmit(,10)'
 ovs-ofctl add-flow br-tun 'cookie=0x79,  table=4,  priority=0 actions=drop'

#=================== table 6
ovs-ofctl add-flow br-tun 'cookie=0x79,  table=6, priority=0 actions=drop'

#处理vxlan包，从vxlan tunnel进入的数据包会resubmit(4)
# table 4 mod_vlan_id --> resubmit(10)
#==================== table 10
ovs-ofctl add-flow br-tun 'cookie=0x79, table=10, priority=1 actions=learn(table=20,hard_timeout=300,priority=1,cookie=0xa9eb8f9011f7e038,NXM_OF_VLAN_TCI[0..11],NXM_OF_ETH_DST[]=NXM_OF_ETH_SRC[],load:0->NXM_OF_VLAN_TCI[],load:NXM_NX_TUN_ID[]->NXM_NX_TUN_ID[],output:NXM_OF_IN_PORT[]),output:1'

## table 10 learn --> table 20 saves learning  resuslt 
#处理patch-int包，虚机出流量 table 0 --> resubmit 2
# table 2 unicast --> table 20
##===================== table 20 没有match则跳转到 table 22
# table 20
ovs-ofctl add-flow br-tun 'cookie=0x79, table=20, priority=0 actions=resubmit(,22)'

#处理patch-int包，虚机出流量 table 0 --> resubmit 2
# table 2 broadcast --> table 22
##========================  table 22  flood
 ovs-ofctl add-flow br-tun 'cookie=0x79, table=22, dl_vlan=100 actions=strip_vlan,set_tunnel:0x30,output:2'
 ovs-ofctl add-flow br-tun 'cookie=0x79, table=22,priority=0 actions=drop'

10.10.16.82新增的流表配置如下

ovs-vsctl add-port br-tun vxlan-02 -- set interface vxlan-02 type=vxlan options:remote_ip=10.10.18.227 options:key=0x30
table 0处理vxlan vni=48收到的包
ovs-ofctl add-flow br-tun 'cookie=0x79, table=0, priority=1,in_port=4 actions=resubmit(,4)'

table 4

ovs-ofctl add-flow br-tun 'cookie=0x79, table=4, priority=1,tun_id=0x30 actions=mod_vlan_vid:22,resubmit(,10)'

table 22

ovs-ofctl add-flow br-tun 'cookie=0x79, table=22, dl_vlan=22 actions=strip_vlan,set_tunnel:0x30,output:4'

10.10.16.82上执行：

[root@kunpeng82 devuser]# ip netns exec ns1 ping 192.168.10.32
PING 192.168.10.32 (192.168.10.32) 56(84) bytes of data.
64 bytes from 192.168.10.32: icmp_seq=1 ttl=64 time=1.20 ms
64 bytes from 192.168.10.32: icmp_seq=2 ttl=64 time=0.340 ms
64 bytes from 192.168.10.32: icmp_seq=3 ttl=64 time=0.766 ms
64 bytes from 192.168.10.32: icmp_seq=4 ttl=64 time=0.343 ms
^C
--- 192.168.10.32 ping statistics ---
4 packets transmitted, 4 received, 0% packet loss, time 3047ms
rtt min/avg/max/mdev = 0.340/0.663/1.204/0.357 ms
[root@kunpeng82 devuser]# 

[root@kunpeng82 devuser]# ovs-ofctl dump-flows br-tun
 cookie=0x79, duration=97548.746s, table=0, n_packets=3563, n_bytes=161602, priority=1,in_port="patch-int" actions=resubmit(,2)
 cookie=0x79, duration=97548.740s, table=0, n_packets=171, n_bytes=13734, priority=1,in_port="vxlan-01" actions=resubmit(,4)
 cookie=0x79, duration=65887.029s, table=0, n_packets=91, n_bytes=7966, priority=1,in_port="vxlan-02" actions=resubmit(,4)
 cookie=0x79, duration=97548.734s, table=0, n_packets=1850, n_bytes=79492, priority=0 actions=drop
 cookie=0x79, duration=97548.728s, table=2, n_packets=199, n_bytes=17430, priority=0,dl_dst=00:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,20)
 cookie=0x79, duration=97548.723s, table=2, n_packets=3364, n_bytes=144172, priority=0,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,22)
 cookie=0x79, duration=97548.717s, table=3, n_packets=0, n_bytes=0, priority=0 actions=drop
 cookie=0x79, duration=97548.711s, table=4, n_packets=171, n_bytes=13734, priority=1,tun_id=0x20 actions=mod_vlan_vid:22,resubmit(,10)
 cookie=0x79, duration=70260.958s, table=4, n_packets=91, n_bytes=7966, priority=1,tun_id=0x30 actions=mod_vlan_vid:22,resubmit(,10)
 cookie=0x79, duration=97548.706s, table=4, n_packets=0, n_bytes=0, priority=0 actions=drop
 cookie=0x79, duration=97548.700s, table=6, n_packets=0, n_bytes=0, priority=0 actions=drop
 cookie=0x79, duration=97548.695s, table=10, n_packets=262, n_bytes=21700, priority=1 actions=learn(table=20,hard_timeout=300,priority=1,cookie=0xa9eb8f9011f7e038,NXM_OF_VLAN_TCI[0..11],NXM_OF_ETH_DST[]=NXM_OF_ETH_SRC[],load:0->NXM_OF_VLAN_TCI[],load:NXM_NX_TUN_ID[]->NXM_NX_TUN_ID[],output:NXM_OF_IN_PORT[]),output:"patch-int"
 cookie=0xa9eb8f9011f7e038, duration=271.640s, table=20, n_packets=32, n_bytes=2744, hard_timeout=300, priority=1,vlan_tci=0x0016/0x0fff,dl_dst=d2:68:22:61:55:7c actions=load:0->NXM_OF_VLAN_TCI[],load:0x30->NXM_NX_TUN_ID[],output:"vxlan-02"
 cookie=0x79, duration=97548.689s, table=20, n_packets=25, n_bytes=2058, priority=0 actions=resubmit(,22)
 cookie=0x79, duration=70260.964s, table=22, n_packets=3273, n_bytes=140042, dl_vlan=22 actions=strip_vlan,set_tunnel:0x30,output:"vxlan-02"
 cookie=0x79, duration=97548.678s, table=22, n_packets=26, n_bytes=1820, priority=0 actions=drop
[root@kunpeng82 devuser]# 

 

 发现flow表中缺少一条

cookie=0x79, duration=2.930s, table=22, n_packets=0, n_bytes=0, dl_vlan=22 actions=strip_vlan,set_tunnel:0x20,output:"vxlan-01"

[root@kunpeng82 devuser]# ovs-ofctl add-flow br-tun 'cookie=0x79, table=22, dl_vlan=22 actions=strip_vlan,set_tunnel:0x20,output:2'
[root@kunpeng82 devuser]# ovs-ofctl dump-flows br-tun
 cookie=0x79, duration=98229.607s, table=0, n_packets=3632, n_bytes=164500, priority=1,in_port="patch-int" actions=resubmit(,2)
 cookie=0x79, duration=98229.601s, table=0, n_packets=171, n_bytes=13734, priority=1,in_port="vxlan-01" actions=resubmit(,4)
 cookie=0x79, duration=66567.890s, table=0, n_packets=91, n_bytes=7966, priority=1,in_port="vxlan-02" actions=resubmit(,4)
 cookie=0x79, duration=98229.595s, table=0, n_packets=1850, n_bytes=79492, priority=0 actions=drop
 cookie=0x79, duration=98229.589s, table=2, n_packets=199, n_bytes=17430, priority=0,dl_dst=00:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,20)
 cookie=0x79, duration=98229.584s, table=2, n_packets=3433, n_bytes=147070, priority=0,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,22)
 cookie=0x79, duration=98229.578s, table=3, n_packets=0, n_bytes=0, priority=0 actions=drop
 cookie=0x79, duration=98229.572s, table=4, n_packets=171, n_bytes=13734, priority=1,tun_id=0x20 actions=mod_vlan_vid:22,resubmit(,10)
 cookie=0x79, duration=70941.819s, table=4, n_packets=91, n_bytes=7966, priority=1,tun_id=0x30 actions=mod_vlan_vid:22,resubmit(,10)
 cookie=0x79, duration=98229.567s, table=4, n_packets=0, n_bytes=0, priority=0 actions=drop
 cookie=0x79, duration=98229.561s, table=6, n_packets=0, n_bytes=0, priority=0 actions=drop
 cookie=0x79, duration=98229.556s, table=10, n_packets=262, n_bytes=21700, priority=1 actions=learn(table=20,hard_timeout=300,priority=1,cookie=0xa9eb8f9011f7e038,NXM_OF_VLAN_TCI[0..11],NXM_OF_ETH_DST[]=NXM_OF_ETH_SRC[],load:0->NXM_OF_VLAN_TCI[],load:NXM_NX_TUN_ID[]->NXM_NX_TUN_ID[],output:NXM_OF_IN_PORT[]),output:"patch-int"
 cookie=0x79, duration=98229.550s, table=20, n_packets=25, n_bytes=2058, priority=0 actions=resubmit(,22)
 cookie=0x79, duration=2.930s, table=22, n_packets=0, n_bytes=0, dl_vlan=22 actions=strip_vlan,set_tunnel:0x20,output:"vxlan-01"
 cookie=0x79, duration=98229.539s, table=22, n_packets=26, n_bytes=1820, priority=0 actions=drop


又把覆盖了

 cookie=0xa9eb8f9011f7e038, duration=271.640s, table=20, n_packets=32, n_bytes=2744, hard_timeout=300, priority=1,vlan_tci=0x0016/0x0fff,dl_dst=d2:68:22:61:55:7c actions=load:0->NXM_OF_VLAN_TCI[],load:0x30->NXM_NX_TUN_ID[],output:"vxlan-02"

如何实现源端复制

[root@kunpeng82 devuser]# ovs-ofctl show br-tun
OFPT_FEATURES_REPLY (xid=0x2): dpid:0000beb2cf13ff44
n_tables:254, n_buffers:0
capabilities: FLOW_STATS TABLE_STATS PORT_STATS QUEUE_STATS ARP_MATCH_IP
actions: output enqueue set_vlan_vid set_vlan_pcp strip_vlan mod_dl_src mod_dl_dst mod_nw_src mod_nw_dst mod_nw_tos mod_tp_src mod_tp_dst
 1(patch-int): addr:7a:5d:ed:67:3b:10
     config:     0
     state:      0
     speed: 0 Mbps now, 0 Mbps max
 2(vxlan-01): addr:f2:f5:da:6d:02:64
     config:     0
     state:      0
     speed: 0 Mbps now, 0 Mbps max
 4(vxlan-02): addr:a6:3a:79:61:b0:ef
     config:     0
     state:      0
     speed: 0 Mbps now, 0 Mbps max
 LOCAL(br-tun): addr:be:b2:cf:13:ff:44
     config:     0
     state:      0
     speed: 0 Mbps now, 0 Mbps max
OFPT_GET_CONFIG_REPLY (xid=0x4): frags=normal miss_send_len=0


[root@kunpeng82 devuser]# ovs-ofctl dump-flows br-tun
 cookie=0x79, duration=104378.426s, table=0, n_packets=3647, n_bytes=165662, priority=1,in_port="patch-int" actions=resubmit(,2)
 cookie=0x79, duration=104378.420s, table=0, n_packets=185, n_bytes=14826, priority=1,in_port="vxlan-01" actions=resubmit(,4)
 cookie=0x79, duration=72716.709s, table=0, n_packets=91, n_bytes=7966, priority=1,in_port="vxlan-02" actions=resubmit(,4)
 cookie=0x79, duration=104378.414s, table=0, n_packets=1852, n_bytes=79632, priority=0 actions=drop
 cookie=0x79, duration=104378.408s, table=2, n_packets=206, n_bytes=18060, priority=0,dl_dst=00:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,20)
 cookie=0x79, duration=104378.403s, table=2, n_packets=3441, n_bytes=147602, priority=0,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,22)
 cookie=0x79, duration=104378.397s, table=3, n_packets=0, n_bytes=0, priority=0 actions=drop
 cookie=0x79, duration=104378.391s, table=4, n_packets=185, n_bytes=14826, priority=1,tun_id=0x20 actions=mod_vlan_vid:22,resubmit(,10)
 cookie=0x79, duration=4101.148s, table=4, n_packets=0, n_bytes=0, priority=1,tun_id=0x30 actions=mod_vlan_vid:22,resubmit(,10)
 cookie=0x79, duration=104378.386s, table=4, n_packets=0, n_bytes=0, priority=0 actions=drop
 cookie=0x79, duration=104378.380s, table=6, n_packets=0, n_bytes=0, priority=0 actions=drop
 cookie=0x79, duration=104378.375s, table=10, n_packets=276, n_bytes=22792, priority=1 actions=learn(table=20,hard_timeout=300,priority=1,cookie=0xa9eb8f9011f7e038,NXM_OF_VLAN_TCI[0..11],NXM_OF_ETH_DST[]=NXM_OF_ETH_SRC[],load:0->NXM_OF_VLAN_TCI[],load:NXM_NX_TUN_ID[]->NXM_NX_TUN_ID[],output:NXM_OF_IN_PORT[]),output:"patch-int"
 cookie=0x79, duration=104378.369s, table=20, n_packets=25, n_bytes=2058, priority=0 actions=resubmit(,22)
 cookie=0x79, duration=8.518s, table=22, n_packets=0, n_bytes=0, dl_vlan=22 actions=strip_vlan,set_tunnel:0x30,output:"vxlan-02",output:"vxlan-01"
 cookie=0x79, duration=104378.358s, table=22, n_packets=28, n_bytes=1960, priority=0 actions=drop
[root@kunpeng82 devuser]# 
[root@kunpeng82 devuser]# ovs-ofctl add-flow br-tun 'cookie=0x79, table=22, dl_vlan=22 actions=strip_vlan,set_tunnel:0x30,output:4,2'

[root@kunpeng82 devuser]# ip netns exec ns1 ping 192.168.10.22
PING 192.168.10.22 (192.168.10.22) 56(84) bytes of data.
64 bytes from 192.168.10.22: icmp_seq=1 ttl=64 time=1.20 ms
64 bytes from 192.168.10.22: icmp_seq=2 ttl=64 time=0.264 ms
^C
--- 192.168.10.22 ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 1001ms
rtt min/avg/max/mdev = 0.264/0.736/1.208/0.472 ms
[root@kunpeng82 devuser]# ip netns exec ns1 ping 192.168.10.32
PING 192.168.10.32 (192.168.10.32) 56(84) bytes of data.
64 bytes from 192.168.10.32: icmp_seq=1 ttl=64 time=1.05 ms
64 bytes from 192.168.10.32: icmp_seq=2 ttl=64 time=0.337 ms
64 bytes from 192.168.10.32: icmp_seq=3 ttl=64 time=0.366 ms
^C
--- 192.168.10.32 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 2058ms
rtt min/avg/max/mdev = 0.337/0.584/1.050/0.330 ms
[root@kunpeng82 devuser]#

清空neigh
[root@kunpeng82 devuser]# ip netns exec ns1 ip n
192.168.10.22 dev veth1 lladdr 7e:53:2a:70:ac:48 STALE
192.168.10.32 dev veth1 lladdr d2:68:22:61:55:7c STALE
[root@kunpeng82 devuser]# ip netns exec ns1 ip n del 192.168.10.22
Device and destination are required arguments.
[root@kunpeng82 devuser]# ip netns exec ns1 ip n del 192.168.10.22 dev veth1 lladdr 7e:53:2a:70:ac:48
[root@kunpeng82 devuser]# ip netns exec ns1 ip n del 192.168.10.32 dev veth1 lladdr d2:68:22:61:55:7c
[root@kunpeng82 devuser]# ip netns exec ns1 ip n
开始ping
[root@kunpeng82 devuser]# ip netns exec ns1 ping 192.168.10.32
PING 192.168.10.32 (192.168.10.32) 56(84) bytes of data.
64 bytes from 192.168.10.32: icmp_seq=1 ttl=64 time=1.67 ms
64 bytes from 192.168.10.32: icmp_seq=2 ttl=64 time=0.497 ms
^C
--- 192.168.10.32 ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 1001ms
rtt min/avg/max/mdev = 0.497/1.084/1.672/0.588 ms
[root@kunpeng82 devuser]# 


//10.10.18.227节点上收到广播报文

[root@host-10-10-18-227 ~]# tcpdump -i vxlan_sys_4789 -eennvv
tcpdump: listening on vxlan_sys_4789, link-type EN10MB (Ethernet), capture size 262144 bytes
18:58:26.003274 2e:6b:9b:c1:f9:2d > ff:ff:ff:ff:ff:ff, ethertype ARP (0x0806), length 42: Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.32 tell 192.168.10.12, length 28
18:58:26.003665 d2:68:22:61:55:7c > 2e:6b:9b:c1:f9:2d, ethertype ARP (0x0806), length 42: Ethernet (len 6), IPv4 (len 4), Reply 192.168.10.32 is-at d2:68:22:61:55:7c, length 28
18:58:26.004129 2e:6b:9b:c1:f9:2d > d2:68:22:61:55:7c, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 21266, offset 0, flags [DF], proto ICMP (1), length 84)
192.168.10.12 > 192.168.10.32: ICMP echo request, id 12831, seq 1, length 64

//10.10.18.216节点上收到广播报文

[root@host-10-10-18-216 ~]# tcpdump -i vxlan_sys_4789 -nnvv
tcpdump: listening on vxlan_sys_4789, link-type EN10MB (Ethernet), capture size 262144 bytes
18:58:15.286320 ARP, Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.32 tell 192.168.10.12, length 28
18:59:02.223977 IP6 (hlim 255, next-header ICMPv6 (58) payload length: 16) fe80::2c6b:9bff:fec1:f92d > ff02::2: [icmp6 sum ok] ICMP6, router solicitation, length 16
source link-address option (1), length 8 (1): 2e:6b:9b:c1:f9:2d
0x0000: 2e6b 9bc1 f92d