https://doc.dpdk.org/guides-18.11/howto/lm_bond_virtio_sriov.html
https://doc.dpdk.org/guides-18.05/nics/intel_vf.html
step
不需要echo xx > /sys/bus/pci/devices/0000:05:00.0/sriov_numvfs
[root@localhost cloud_images]# ls /sys/bus/pci/devices/0000:05:00.0 broken_parity_status consistent_dma_mask_bits d3cold_allowed dma_mask_bits enable irq max_link_speed msi_bus remove resource resource4 sriov_drivers_autoprobe subsystem uevent class current_link_speed device driver iommu local_cpulist max_link_width numa_node rescan resource0 revision sriov_numvfs subsystem_device vendor config current_link_width devspec driver_override iommu_group local_cpus modalias power reset resource2 rom sriov_totalvfs subsystem_vendor vpd [root@localhost cloud_images]# ls /sys/bus/pci/devices/0000:05:00.0/max_vfs ls: cannot access /sys/bus/pci/devices/0000:05:00.0/max_vfs: No such file or directory [root@localhost cloud_images]# ls /sys/bus/pci/devices/0000:05:00.0/sriov_numvf ls: cannot access /sys/bus/pci/devices/0000:05:00.0/sriov_numvf: No such file or directory [root@localhost cloud_images]# ls /sys/bus/pci/devices/0000:05:00.0/sriov_numvfs /sys/bus/pci/devices/0000:05:00.0/sriov_numvfs [root@localhost cloud_images]# cat /sys/bus/pci/devices/0000:05:00.0/sriov_numvfs 0 [root@localhost cloud_images]# cat /sys/bus/pci/devices/0000:05:00.0/sriov_totalvfs 120 [root@localhost cloud_images]# lspci | grep "Virtual Function" [root@localhost cloud_images]# cat /sys/bus/pci/devices/0000:05:00.0/sriov_numvfs 0 [root@localhost cloud_images]#
step
网卡绑定vfio
-device pci-assign,host=0000:05.0要改成 -device vfio-pci,host=0000:05:00.0
qemu-system-aarch64 -name vm2 -nographic -enable-kvm -M virt -cpu host -smp 2 -m 4096 -object memory-backend-file,id=mem,size=4096M,mem-path=/mnt/huge,share=on -numa node,memdev=mem -mem-prealloc -global virtio-blk-device.scsi=off -device virtio-scsi-device,id=scsi -kernel vmlinuz-4.18 --append "console=ttyAMA0 root=UUID=6a09973e-e8fd-4a6d-a8c0-1deb9556f477" -initrd initramfs-4.18 -drive file=vhuser-test1.qcow2 -device vfio-pci,host=0000:05:00.0 -net nic,macaddr=00:00:00:99:99:01 -vnc :10
CentOS Linux 7 (AltArch) Kernel 4.18.0-147.8.1.el7.aarch64 on an aarch64 localhost login: root Password: Last login: Thu Nov 5 07:09:33 on ttyAMA0 [root@localhost ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000 link/ether 00:00:00:99:99:01 brd ff:ff:ff:ff:ff:ff inet6 fe80::200:ff:fe99:9901/64 scope link valid_lft forever preferred_lft forever 3: enp0s2: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000 link/ether 44:a1:91:a4:9b:eb brd ff:ff:ff:ff:ff:ff [root@localhost ~]# lscpi | grep -i ether -bash: lscpi: command not found [root@localhost ~]# lspci | grep -i ether 00:01.0 Ethernet controller: Red Hat, Inc. Virtio network device 00:02.0 Ethernet controller: Huawei Technologies Co., Ltd. Hi1822 Family (2*100GE) (rev 45) [root@localhost ~]#
[root@localhost ~]# lspci | grep -i ether 00:01.0 Ethernet controller: Red Hat, Inc. Virtio network device 00:02.0 Ethernet controller: Huawei Technologies Co., Ltd. Hi1822 Family (2*100GE) (rev 45) [root@localhost ~]# ip link set enp0s2 up [ 445.929369] hinic 0000:00:02.0: MGMT timeout, MSG id = 10 [ 445.933251] hinic 0000:00:02.0: Failed to set HW IO ctxt [ 445.956403] hinic 0000:00:02.0 enp0s2: Failed - HW interface up RTNETLINK answers: Connection timed out [root@localhost ~]# ethtool -i enp0s2 driver: hinic version: firmware-version: expansion-rom-version: bus-info: 0000:00:02.0 supports-statistics: no supports-test: no supports-eeprom-access: no supports-register-dump: no supports-priv-flags: no [root@localhost ~]# whereis dpdk dpdk: /usr/share/dpdk [root@localhost ~]# ls /usr/share/dpdk usertools [root@localhost ~]#
加上 iommu=pt intel_iommu=on iommu.passthrough=1 解决 MGMT timeout, MSG id = 10
qemu-system-aarch64 -name vm2 -nographic -enable-kvm -M virt -cpu host -smp 2 -m 4096 -object memory-backend-file,id=mem,size=4096M,mem-path=/mnt/huge,share=on -numa node,memdev=mem -mem-prealloc -global virtio-blk-device.scsi=off -device virtio-scsi-device,id=scsi -kernel vmlinuz-4.18 --append "console=ttyAMA0 root=UUID=6a09973e-e8fd-4a6d-a8c0-1deb9556f477 iommu=pt intel_iommu=on iommu.passthrough=1" -initrd initramfs-4.18 -drive file=vhuser-test1.qcow2 -device vfio-pci,host=0000:05:00.0 -net nic,macaddr=00:00:00:99:99:01 -vnc :10
[root@localhost ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000 link/ether 00:00:00:99:99:01 brd ff:ff:ff:ff:ff:ff inet6 fe80::200:ff:fe99:9901/64 scope link valid_lft forever preferred_lft forever 3: eth1: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000 link/ether 52:54:00:12:34:56 brd ff:ff:ff:ff:ff:ff [root@localhost ~]# ip link set eth1 up [root@localhost ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000 link/ether 00:00:00:99:99:01 brd ff:ff:ff:ff:ff:ff inet6 fe80::200:ff:fe99:9901/64 scope link valid_lft forever preferred_lft forever 3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000 link/ether 52:54:00:12:34:56 brd ff:ff:ff:ff:ff:ff inet6 fe80::5054:ff:fe12:3456/64 scope link tentative valid_lft forever preferred_lft forever [root@localhost ~]# cat /proc/cmdline console=ttyAMA0 root=UUID=6a09973e-e8fd-4a6d-a8c0-1deb9556f477 iommu=pt intel_iommu=on iommu.passthrough=1 [root@localhost ~]# netstat -lpn Active Internet connections (only servers) Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name tcp 0 0 0.0.0.0:111 0.0.0.0:* LISTEN 705/rpcbind tcp6 0 0 :::111 :::* LISTEN 705/rpcbind udp 0 0 0.0.0.0:68 0.0.0.0:* 852/dhclient udp 0 0 0.0.0.0:111 0.0.0.0:* 705/rpcbind udp 0 0 127.0.0.1:323 0.0.0.0:* 737/chronyd udp 0 0 0.0.0.0:879 0.0.0.0:* 705/rpcbind udp6 0 0 :::111 :::* 705/rpcbind udp6 0 0 ::1:323 :::* 737/chronyd udp6 0 0 :::879 :::* 705/rpcbind Active UNIX domain sockets (only servers) Proto RefCnt Flags Type State I-Node PID/Program name Path unix 2 [ ACC ] STREAM LISTENING 278 1/systemd /run/systemd/journal/stdout unix 2 [ ACC ] STREAM LISTENING 18021 865/gssproxy /var/lib/gssproxy/default.sock unix 2 [ ACC ] STREAM LISTENING 18022 865/gssproxy /run/gssproxy.sock unix 2 [ ACC ] STREAM LISTENING 16005 1/systemd /run/systemd/private unix 2 [ ACC ] SEQPACKET LISTENING 16022 1/systemd /run/udev/control unix 2 [ ACC ] STREAM LISTENING 17648 1/systemd /var/run/rpcbind.sock unix 2 [ ACC ] STREAM LISTENING 17652 1/systemd /run/dbus/system_bus_socket [root@localhost ~]#
[root@localhost test-pmd]# ./testpmd -l 0-3 -n 4 -- -i EAL: Detected 128 lcore(s) EAL: Detected 4 NUMA nodes EAL: Multi-process socket /var/run/dpdk/rte/mp_socket EAL: Selected IOVA mode 'VA' EAL: No available hugepages reported in hugepages-2048kB EAL: Probing VFIO support... EAL: VFIO support initialized EAL: PCI device 0000:05:00.0 on NUMA socket 0 EAL: probe driver: 19e5:200 net_hinic EAL: Cannot open /dev/vfio/25: Device or resource busy EAL: Failed to open group 25 EAL: Requested device 0000:05:00.0 cannot be used ----------已经透传了 EAL: PCI device 0000:06:00.0 on NUMA socket 0 EAL: probe driver: 19e5:200 net_hinic EAL: using IOMMU type 1 (Type 1) net_hinic: Initializing pf hinic-0000:06:00.0 in primary process net_hinic: Hwif is not ready net_hinic: Initialize hwif failed, dev_name: 0000:06:00.0 net_hinic: Initialize hwif failed, dev_name: 0000:06:00.0 net_hinic: Create nic device failed, dev_name: 0000:06:00.0 net_hinic: Initialize 0000:06:00.0 in primary failed EAL: Releasing pci mapped resource for 0000:06:00.0 EAL: Calling pci_unmap_resource for 0000:06:00.0 at 0x8300000000 EAL: Calling pci_unmap_resource for 0000:06:00.0 at 0x8300030000 EAL: Requested device 0000:06:00.0 cannot be used EAL: PCI device 0000:7d:00.0 on NUMA socket 0 EAL: probe driver: 19e5:a222 net_hns3 EAL: PCI device 0000:7d:00.1 on NUMA socket 0 EAL: probe driver: 19e5:a221 net_hns3 EAL: PCI device 0000:7d:00.2 on NUMA socket 0 EAL: probe driver: 19e5:a222 net_hns3 EAL: PCI device 0000:7d:00.3 on NUMA socket 0 EAL: probe driver: 19e5:a221 net_hns3 testpmd: No probed ethernet devices Interactive-mode selected testpmd: create a new mbuf pool <mbuf_pool_socket_0>: n=171456, size=2176, socket=0 testpmd: preferred mempool ops selected: ring_mp_mc Done testpmd>
qemu-system-aarch64 -name vm2 -daemonize -enable-kvm -M virt -cpu host -smp 2 -m 4096 -object memory-backend-file,id=mem,size=4096M,mem-path=/mnt/huge,share=on -numa node,memdev=mem -mem-prealloc -global virtio-blk-device.scsi=off -device virtio-scsi-device,id=scsi -kernel vmlinuz-4.18 --append "console=ttyAMA0 root=UUID=6a09973e-e8fd-4a6d-a8c0-1deb9556f477 iommu=pt intel_iommu=on iommu.passthrough=1" -initrd initramfs-4.18 -drive file=vhuser-test1.qcow2 -device vfio-pci,host=0000:05:00.0 -net nic,macaddr=00:00:00:99:99:01 -serial telnet:localhost:4321,server,nowait -vnc :10
[root@localhost ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000 link/ether 00:00:00:99:99:01 brd ff:ff:ff:ff:ff:ff inet6 fe80::200:ff:fe99:9901/64 scope link valid_lft for00:02.0 hinic -bash: 00:02.0: command not found [root@localhost ~]# ed modules(DPDK driver) are loaded 0:02.0 hinic [root@localhost ~]# c. Please specify device in "bus:slot.func" format 0:02.0 hinic [root@localhost ~]# /usr/share/dpdk/usertools/dpdk-devbind.py -b 000:02.0 hinic [root@localhost ~]# [root@localhost ~]# [root@localhost ~]# croot@localhost ~]# 0:02.0 hinic Warning - no supported modules(DPDK driver) are loadedvbind.py -b 000:02.0 hinic Unknown device: hinic. Please specify device in "bus:slot.func" format [root@localhost ~]# cd /usr/share/dpdk/usertools/ [root@localhost usertools]# ls cpu_layout.py dpdk-devbind.py dpdk-pmdinfo.py dpdk-telemetry-client.py cpu_layout.pyc dpdk-devbind.pyc dpdk-pmdinfo.pyc dpdk-telemetry-client.pyc cpu_layout.pyo dpdk-devbind.pyo dpdk-pmdinfo.pyo dpdk-telemetry-client.pyo [root@localhost usertools]# ./dpdk-devbind.py -b hinic 00:02.0 Warning - no supported modules(DPDK driver) are loaded [ 622.108332] hinic 0000:00:02.0: hwif status is not ready [ 622.112328] hinic 0000:00:02.0: HW interface is not ready [ 622.116131] hinic 0000:00:02.0: Failed to init HW interface [ 622.120113] hinic 0000:00:02.0: Failed to initialize HW device [ 622.124215] hinic 0000:00:02.0: Failed to initialize NIC device [ 622.128811] hinic: probe of 0000:00:02.0 failed with error -14 Error: bind failed for 0000:00:02.0 - Cannot bind to driver hinic [root@localhost usertools]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000 link/ether 00:00:00:99:99:01 brd ff:ff:ff:ff:ff:ff inet6 fe80::200:ff:fe99:9901/64 scope link valid_lft forever preferred_lft forever [root@localhost usertools]#