ip link add link eth0 name macvtap0 type macvtap mode bridge qemu-kvm -nographic -kernel /boot/vmlinuz-guest -append "console=ttyS0 root=/dev/vda" -drive file=/tmp/testroot.img,if=virtio,cache=none -net nic,model=virtio,macaddr=$(< /sys/class/net/macvtap0/address) -net tap,fd=3 3<>/dev/tap$(< /sys/class/net/macvtap0/ifindex)
[root@localhost ~]# ip link add peerh type veth peer name peerv [root@localhost ~]# ip link add link peerv name macvtap0 type macvtap [root@localhost ~]# ip link set macvtap0 address 1a:46:0b:ca:bc:7b up [root@localhost ~]# ip link show macvtap0 97: macvtap0@peerv: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc pfifo_fast state LOWERLAYERDOWN mode DEFAULT group default qlen 500 link/ether 1a:46:0b:ca:bc:7b brd ff:ff:ff:ff:ff:ff [root@localhost ~]#
-netdev tap,id=network-0,vhost=on,vhostfds=3,fds=4 -device driver=virtio-net-pci,netdev=network-0,mac=1a:46:0b:ca:bc:7b,disable-modern=false,mq=on,vectors=4,romfile=
qemu-system-aarch64: -netdev tap,id=network-0,vhost=on,vhostfds=3,fds=4: TUNGETIFF ioctl() failed: Inappropriate ioctl for device TUNSETOFFLOAD ioctl() failed: Inappropriate ioctl for device qemu-system-aarch64: -netdev tap,id=network-0,vhost=on,vhostfds=3,fds=4: vhost-net requested but could not be initialized
原理是macvtap0的ifndex有误
root@ubuntu:/home/ubuntu/qemu_learn# cat /sys/class/net/macvtap0/ifindex 471
虚拟机内
CentOS Linux 7 (AltArch) Kernel 4.18.0-147.8.1.el7.aarch64 on an aarch64 localhost login: root Password: Last login: Thu Nov 5 03:26:45 from gateway [root@localhost ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000 link/ether 1a:46:0b:ca:bc:7b brd ff:ff:ff:ff:ff:ff inet6 fe80::1846:bff:feca:bc7b/64 scope link valid_lft forever preferred_lft forever [root@localhost ~]# ip a | grep '1a:46:0b:ca:bc:7b' link/ether 1a:46:0b:ca:bc:7b brd ff:ff:ff:ff:ff:ff [root@localhost ~]#
[root@localhost ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000 link/ether 1a:46:0b:ca:bc:7b brd ff:ff:ff:ff:ff:ff ---mac没改变 inet 10.10.100.82/24 scope global eth0 valid_lft forever preferred_lft forever inet6 fe80::1846:bff:feca:bc7b/64 scope link valid_lft forever preferred_lft forever
主机侧
root@ubuntu:/home/ubuntu# ip a show peerv 469: peerv@peerh: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000 link/ether ea:78:75:7d:3d:6a brd ff:ff:ff:ff:ff:ff inet6 fe80::e878:75ff:fe7d:3d6a/64 scope link valid_lft forever preferred_lft forever root@ubuntu:/home/ubuntu# ip a show peerh 470: peerh@peerv: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000 link/ether f6:56:ad:40:45:9f brd ff:ff:ff:ff:ff:ff inet6 fe80::f456:adff:fe40:459f/64 scope link valid_lft forever preferred_lft forever root@ubuntu:/home/ubuntu# ip a add 10.10.100.83/24 dev peerh root@ubuntu:/home/ubuntu# ping 10.10.100.82 PING 10.10.100.82 (10.10.100.82) 56(84) bytes of data. 64 bytes from 10.10.100.82: icmp_seq=1 ttl=64 time=0.448 ms 64 bytes from 10.10.100.82: icmp_seq=2 ttl=64 time=0.292 ms ^C --- 10.10.100.82 ping statistics --- 2 packets transmitted, 2 received, 0% packet loss, time 1005ms rtt min/avg/max/mdev = 0.292/0.370/0.448/0.078 ms root@ubuntu:/home/ubuntu# ip a show peerh 470: peerh@peerv: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000 link/ether f6:56:ad:40:45:9f brd ff:ff:ff:ff:ff:ff inet 10.10.100.83/24 scope global peerh valid_lft forever preferred_lft forever inet6 fe80::f456:adff:fe40:459f/64 scope link valid_lft forever preferred_lft forever root@ubuntu:/home/ubuntu#
虚拟机启动后，host侧
root@ubuntu:/home/ubuntu# cat /sys/class/net/macvtap0/ifindex
471
root@ubuntu:/home/ubuntu# ip a | grep macvtap0
471: macvtap0@peerv: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 500
root@ubuntu:/home/ubuntu#
root@ubuntu:/home/ubuntu# ip netns exec net1 ip a 1: lo: <LOOPBACK> mtu 65536 qdisc noop state DOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 root@ubuntu:/home/ubuntu# ip link set macvtap0 netns net1 root@ubuntu:/home/ubuntu# ip link set peerv netns net1 root@ubuntu:/home/ubuntu# ip netns exec net1 ip a 1: lo: <LOOPBACK> mtu 65536 qdisc noop state DOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 469: peerv@if470: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000 link/ether ea:78:75:7d:3d:6a brd ff:ff:ff:ff:ff:ff link-netnsid 0 471: macvtap0@peerv: <BROADCAST,MULTICAST,M-DOWN> mtu 1500 qdisc noop state DOWN group default qlen 500 link/ether 1a:46:0b:ca:bc:7b brd ff:ff:ff:ff:ff:ff root@ubuntu:/home/ubuntu# ip netns exec net1 ip link set macvtap0 up root@ubuntu:/home/ubuntu# ip netns exec net1 ip link set peerv up root@ubuntu:/home/ubuntu# ping 10.10.100.82 PING 10.10.100.82 (10.10.100.82) 56(84) bytes of data. 64 bytes from 10.10.100.82: icmp_seq=1 ttl=64 time=0.659 ms 64 bytes from 10.10.100.82: icmp_seq=2 ttl=64 time=0.188 ms 64 bytes from 10.10.100.82: icmp_seq=3 ttl=64 time=0.233 ms ^C --- 10.10.100.82 ping statistics --- 3 packets transmitted, 3 received, 0% packet loss, time 2055ms rtt min/avg/max/mdev = 0.188/0.360/0.659/0.212 ms root@ubuntu:/home/ubuntu#
虚拟机
[root@localhost ~]# ping 10.10.100.83 PING 10.10.100.83 (10.10.100.83) 56(84) bytes of data. 64 bytes from 10.10.100.83: icmp_seq=1 ttl=64 time=0.244 ms 64 bytes from 10.10.100.83: icmp_seq=2 ttl=64 time=0.203 ms --- 10.10.100.83 ping statistics --- 2 packets transmitted, 2 received, 0% packet loss, time 1001ms rtt min/avg/max/mdev = 0.203/0.223/0.244/0.025 ms [root@localhost ~]#
给host侧
root@ubuntu:/home/ubuntu# ip netns exec net1 ip a 1: lo: <LOOPBACK> mtu 65536 qdisc noop state DOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 469: peerv@if470: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000 link/ether ea:78:75:7d:3d:6a brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet6 fe80::e878:75ff:fe7d:3d6a/64 scope link valid_lft forever preferred_lft forever 471: macvtap0@peerv: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 500 link/ether 1a:46:0b:ca:bc:7b brd ff:ff:ff:ff:ff:ff inet6 fe80::1846:bff:feca:bc7b/64 scope link valid_lft forever preferred_lft forever . root@ubuntu:/home/ubuntu# ip netns exec net1 ip a add 10.10.100.82/24 dev macvtap0 ---和虚拟机ip一样 root@ubuntu:/home/ubuntu# ip netns exec net1 ip a 1: lo: <LOOPBACK> mtu 65536 qdisc noop state DOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 469: peerv@if470: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000 link/ether ea:78:75:7d:3d:6a brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet6 fe80::e878:75ff:fe7d:3d6a/64 scope link valid_lft forever preferred_lft forever 471: macvtap0@peerv: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 500 link/ether 1a:46:0b:ca:bc:7b brd ff:ff:ff:ff:ff:ff inet 10.10.100.82/24 scope global macvtap0 valid_lft forever preferred_lft forever inet6 fe80::1846:bff:feca:bc7b/64 scope link valid_lft forever preferred_lft forever root@ubuntu:/home/ubuntu#
虚拟机ping
[root@localhost ~]# ping 10.10.100.83 PING 10.10.100.83 (10.10.100.83) 56(84) bytes of data. 64 bytes from 10.10.100.83: icmp_seq=1 ttl=64 time=0.265 ms 64 bytes from 10.10.100.83: icmp_seq=2 ttl=64 time=0.269 ms --- 10.10.100.83 ping statistics --- 2 packets transmitted, 2 received, 0% packet loss, time 1001ms rtt min/avg/max/mdev = 0.265/0.267/0.269/0.002 ms [root@localhost ~]#
macvtap + kata
创建容器
root@ubuntu:/home/ubuntu# docker run -it --runtime=kata-runtime --rm debian /bin/bash root@abfb1728e810:/# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000 link/ether 02:42:ac:11:00:04 brd ff:ff:ff:ff:ff:ff inet 172.17.0.4/16 brd 172.17.255.255 scope global eth0 valid_lft forever preferred_lft forever inet6 fe80::42:acff:fe11:4/64 scope link valid_lft forever preferred_lft forever
root@abfb1728e810:/# ping 172.17.0.1 PING 172.17.0.1 (172.17.0.1) 56(84) bytes of data. 64 bytes from 172.17.0.1: icmp_seq=1 ttl=64 time=0.387 ms 64 bytes from 172.17.0.1: icmp_seq=2 ttl=64 time=0.214 ms ^C --- 172.17.0.1 ping statistics --- 2 packets transmitted, 2 received, 0% packet loss, time 31ms rtt min/avg/max/mdev = 0.214/0.300/0.387/0.088 ms
root@ubuntu: runtime# ip netns exec cnitest-d6e30f25-c26b-f8c8-7d93-c6f3462bef80 ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever 2: tap0_kata: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UNKNOWN group default qlen 1000 link/ether d2:f5:21:84:ac:99 brd ff:ff:ff:ff:ff:ff 472: eth0@if473: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000 link/ether 02:42:ac:11:00:04 brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 172.17.0.4/16 brd 172.17.255.255 scope global eth0 valid_lft forever preferred_lft forever root@ubuntu: runtime# ip netns exec cnitest-d6e30f25-c26b-f8c8-7d93-c6f3462bef80 ip a flush eth0@ Device "eth0@" does not exist. root@ubuntu: runtime# ip netns exec cnitest-d6e30f25-c26b-f8c8-7d93-c6f3462bef80 ip a flush eth0 ---删掉 root@ubuntu: runtime# ip netns exec cnitest-d6e30f25-c26b-f8c8-7d93-c6f3462bef80 ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever 2: tap0_kata: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UNKNOWN group default qlen 1000 link/ether d2:f5:21:84:ac:99 brd ff:ff:ff:ff:ff:ff 472: eth0@if473: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000 link/ether 02:42:ac:11:00:04 brd ff:ff:ff:ff:ff:ff link-netnsid 0
容器还可以ping
root@abfb1728e810:/# ping 172.17.0.1 PING 172.17.0.1 (172.17.0.1) 56(84) bytes of data. 64 bytes from 172.17.0.1: icmp_seq=1 ttl=64 time=0.521 ms 64 bytes from 172.17.0.1: icmp_seq=2 ttl=64 time=0.210 ms 64 bytes from 172.17.0.1: icmp_seq=3 ttl=64 time=0.229 ms ^C --- 172.17.0.1 ping statistics --- 3 packets transmitted, 3 received, 0% packet loss, time 32ms rtt min/avg/max/mdev = 0.210/0.320/0.521/0.142 ms root@abfb1728e810:/#