dpdk uio

root@gobgp:~# lspci | grep -i ether
01:00.0 Ethernet controller: Red Hat, Inc. Virtio network device (rev 01)
07:00.0 Ethernet controller: Red Hat, Inc. Virtio network device (rev 01)
root@gobgp:~# lspci -s 07:00.0
07:00.0 Ethernet controller: Red Hat, Inc. Virtio network device (rev 01)
root@gobgp:~# lspci -s 07:00.0 -v
07:00.0 Ethernet controller: Red Hat, Inc. Virtio network device (rev 01)
        Subsystem: Red Hat, Inc. Virtio network device
        Physical Slot: 0-6
        Flags: bus master, fast devsel, latency 0, IRQ 38
        Memory at 10c40000 (32-bit, non-prefetchable) [size=4K]
        Memory at 8000c00000 (64-bit, prefetchable) [size=16K]
        Expansion ROM at 10c00000 [disabled] [size=256K]
        Capabilities: [dc] MSI-X: Enable- Count=3 Masked-
        Capabilities: [c8] Vendor Specific Information: VirtIO: <unknown>
        Capabilities: [b4] Vendor Specific Information: VirtIO: Notify
        Capabilities: [a4] Vendor Specific Information: VirtIO: DeviceCfg
        Capabilities: [94] Vendor Specific Information: VirtIO: ISR
        Capabilities: [84] Vendor Specific Information: VirtIO: CommonCfg
        Capabilities: [7c] Power Management version 3
        Capabilities: [40] Express Endpoint, MSI 00
        Kernel driver in use: uio_pci_generic

root@gobgp:~# 

root@gobgp:~# cat /sys/bus/pci/drivers/uio_pci_generic/bind
cat: /sys/bus/pci/drivers/uio_pci_generic/bind: Permission denied
 
root@gobgp:~# ls -l /sys/bus/pci/devices/0000:07:00.0
lrwxrwxrwx 1 root root 0 Aug 24 14:20 /sys/bus/pci/devices/0000:07:00.0 -> ../../../devices/pci0000:00/0000:00:01.6/0000:07:00.0
root@gobgp:~# ls -l /sys/bus/pci/devices/0000:07:00.0/driver 
lrwxrwxrwx 1 root root 0 Aug 24 14:20 /sys/bus/pci/devices/0000:07:00.0/driver -> ../../../../bus/pci/drivers/uio_pci_generic
root@gobgp:~#

root@gobgp:~# cat /dev/uio0
cat: /dev/uio0: Invalid argument
root@gobgp:~# 

Example code using uio_pci_generic
Here is some sample userspace driver code using uio_pci_generic:

#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h>

int main()
{
    int uiofd;
    int configfd;
    int err;
    int i;
    unsigned icount;
    unsigned char command_high;

    uiofd = open("/dev/uio0", O_RDONLY);
    if (uiofd < 0) {
        perror("uio open:");
        return errno;
    }
    configfd = open("/sys/class/uio/uio0/device/config", O_RDWR);
    if (configfd < 0) {
        perror("config open:");
        return errno;
    }

    /* Read and cache command value */
    err = pread(configfd, &command_high, 1, 5);
    if (err != 1) {
        perror("command config read:");
        return errno;
    }
    command_high &= ~0x4;

    for(i = 0;; ++i) {
        /* Print out a message, for debugging. */
        if (i == 0)
            fprintf(stderr, "Started uio test driver.
");
        else
            fprintf(stderr, "Interrupts: %d
", icount);

        /****************************************/
        /* Here we got an interrupt from the
           device. Do something to it. */
        /****************************************/

        /* Re-enable interrupts. */
        err = pwrite(configfd, &command_high, 1, 5);
        if (err != 1) {
            perror("config write:");
            break;
        }

        /* Wait for next interrupt. */
        err = read(uiofd, &icount, 4);
        if (err != 4) {
            perror("uio read:");
            break;
        }

    }
    return errno;
}

 

#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h>
#include <stdint.h>
#include <sys/mman.h>


#define EDU_IO_SIZE 0x100
#define EDU_CARD_VERSION_ADDR  0x0
#define EDU_CARD_LIVENESS_ADDR 0x1
#define EDU_RAISE_INT_ADDR 0x18
#define EDU_CLEAR_INT_ADDR 0x19

int main()
{
    int uiofd;
    int configfd;
    int bar0fd;
    int resetfd;
    int err;
    int i;
    unsigned icount;
    unsigned char command_high;
    volatile uint32_t *bar0;

    uiofd = open("/dev/uio0", O_RDWR);
    if (uiofd < 0) {
        perror("uio open:");
        return errno;
    }

    configfd = open("/sys/class/uio/uio0/device/config", O_RDWR);
    if (configfd < 0) {
        perror("config open:");
        return errno;
    }

    /* Read and cache command value */
    err = pread(configfd, &command_high, 1, 5);
    if (err != 1) {
        perror("command config read:");
        return errno;
    }
    command_high &= ~0x4;

    /* Map edu's MMIO */
    bar0fd = open("/sys/class/uio/uio0/device/resource0", O_RDWR);
    if (bar0fd < 0) {
        perror("bar0fd open:");
        return errno;
    }

    /* mmap the device's BAR */
    bar0 = (volatile uint32_t *)mmap(NULL, EDU_IO_SIZE, PROT_READ|PROT_WRITE, MAP_SHARED, bar0fd, 0);
    if (bar0 == MAP_FAILED) {
        perror("Error mapping bar0!");
        return errno;
    }
    fprintf(stdout, "Version = %08X
", bar0[EDU_CARD_VERSION_ADDR]);

    /* Test the invertor function */
    i = 0x12345678;
    bar0[EDU_CARD_LIVENESS_ADDR] = i;
    fprintf(stdout, "Inversion: %08X --> %08X
", i, bar0[EDU_CARD_LIVENESS_ADDR]);

    /* Clear previous interrupt */
    bar0[EDU_CLEAR_INT_ADDR] = 0xABCDABCD;

    /* Raise an interrupt */
    bar0[EDU_RAISE_INT_ADDR] = 0xABCDABCD;

    for(i = 0;; ++i) {
        /* Print out a message, for debugging. */
        if (i == 0)
            fprintf(stderr, "Started uio test driver.
");
        else
            fprintf(stderr, "Interrupts: %d
", icount);

        /****************************************/
        /* Here we got an interrupt from the
           device. Do something to it. */
        /****************************************/

        /* Re-enable interrupts. */
        err = pwrite(configfd, &command_high, 1, 5);
        if (err != 1) {
            perror("config write:");
            break;
        }

        /* Clear previous interrupt */
        bar0[EDU_CLEAR_INT_ADDR] = 0xABCDABCD;

        /* Raise an interrupt */
        bar0[EDU_RAISE_INT_ADDR] = 0xABCDABCD;

        /* Wait for next interrupt. */
        err = read(uiofd, &icount, 4);
        if (err != 4) {
            perror("uio read:");
            break;
        }

    }
    return errno;
}
The result looks something like this:

Version = 010000ED
Inversion: 12345678 --> EDCBA987
Started uio test driver.
Interrupts: 3793548
Interrupts: 3793549
Interrupts: 3793550
Interrupts: 3793551
Interrupts: 3793552
Interrupts: 3793553
Interrupts: 3793554
Interrupts: 3793555
Interrupts: 3793556
...

dpdk uio

./usertools/dpdk-devbind.py  --bind=igb_uio  0000:06:00.0

[root@localhost dpdk-19.11]# ls /sys/bus/pci/devices/0000:06:00.0/u
uevent  uio/    
[root@localhost dpdk-19.11]# ls /sys/bus/pci/devices/0000:06:00.0/uio/
uio0
[root@localhost dpdk-19.11]# ls /sys/bus/pci/devices/0000:06:00.0/uio/uio0/maps/
map0  map1  map2
[root@localhost dpdk-19.11]# 

 

[root@localhost dpdk-19.11]# ls /sys/bus/pci/devices/0000:06:00.0/uio/uio0/maps/map0
addr  name  offset  size
[root@localhost dpdk-19.11]# cat  /sys/bus/pci/devices/0000:06:00.0/uio/uio0/maps/map0/addr 
0x0000080010400000
[root@localhost dpdk-19.11]# cat  /sys/bus/pci/devices/0000:06:00.0/uio/uio0/maps/map0/name
BAR0
[root@localhost dpdk-19.11]# cat  /sys/bus/pci/devices/0000:06:00.0/uio/uio0/maps/map0/offset 
0x0
[root@localhost dpdk-19.11]# cat  /sys/bus/pci/devices/0000:06:00.0/uio/uio0/maps/map0/size
0x0000000000020000
[root@localhost dpdk-19.11]# cat  /sys/bus/pci/devices/0000:06:00.0/uio/uio0/maps/map1/addr 
0x0000080011320000
[root@localhost dpdk-19.11]# cat  /sys/bus/pci/devices/0000:06:00.0/uio/uio0/maps/map2/addr 
0x0000080008b00000
[root@localhost dpdk-19.11]# cat  /sys/bus/pci/devices/0000:06:00.0/uio/uio0/maps/map1/name
BAR2
[root@localhost dpdk-19.11]# cat  /sys/bus/pci/devices/0000:06:00.0/uio/uio0/maps/map2/name
BAR4
[root@localhost dpdk-19.11]# 

[root@localhost dpdk-19.11]# lspci -s 0000:06:00.0 -v
06:00.0 Ethernet controller: Huawei Technologies Co., Ltd. Hi1822 Family (2*25GE) (rev 45)
        Subsystem: Huawei Technologies Co., Ltd. Device d139
        Flags: bus master, fast devsel, latency 0, NUMA node 0
        Memory at 80010400000 (64-bit, prefetchable) [size=128K]
        Memory at 80011320000 (64-bit, prefetchable) [size=32K]
        Memory at 80008b00000 (64-bit, prefetchable) [size=1M]
        Expansion ROM at e9300000 [disabled] [size=1M]
        Capabilities: [40] Express Endpoint, MSI 00
        Capabilities: [80] MSI: Enable- Count=1/32 Maskable+ 64bit+
        Capabilities: [a0] MSI-X: Enable- Count=32 Masked-
        Capabilities: [b0] Power Management version 3
        Capabilities: [c0] Vital Product Data
        Capabilities: [100] Advanced Error Reporting
        Capabilities: [150] Alternative Routing-ID Interpretation (ARI)
        Capabilities: [200] Single Root I/O Virtualization (SR-IOV)
        Capabilities: [310] #19
        Capabilities: [4e0] Device Serial Number 44-a1-91-ff-ff-a4-9b-ec
        Capabilities: [4f0] Transaction Processing Hints
        Capabilities: [600] Vendor Specific Information: ID=0000 Rev=0 Len=028 <?>
        Capabilities: [630] Access Control Services
        Kernel driver in use: igb_uio
        Kernel modules: hinic

[root@localhost dpdk-19.11]# 

[root@localhost dpdk-19.11]# cat /proc/iomem  | grep 0000:06:00.0
        e9300000-e93fffff : 0000:06:00.0
        80008b00000-80008bfffff : 0000:06:00.0
        80008c00000-800103fffff : 0000:06:00.0
        80010400000-8001041ffff : 0000:06:00.0
        80010420000-80010b9ffff : 0000:06:00.0
        80010ba0000-8001131ffff : 0000:06:00.0
        80011320000-80011327fff : 0000:06:00.0
[root@localhost dpdk-19.11]# 

[root@localhost dpdk-19.11]# ls  /sys/bus/pci/devices/0000:06:00.0/resource*
/sys/bus/pci/devices/0000:06:00.0/resource   /sys/bus/pci/devices/0000:06:00.0/resource2
/sys/bus/pci/devices/0000:06:00.0/resource0  /sys/bus/pci/devices/0000:06:00.0/resource4
[root@localhost dpdk-19.11]# cat  /sys/bus/pci/devices/0000:06:00.0/resource*
0x0000080010400000 0x000008001041ffff 0x000000000014220c
0x0000000000000000 0x0000000000000000 0x0000000000000000
0x0000080011320000 0x0000080011327fff 0x000000000014220c
0x0000000000000000 0x0000000000000000 0x0000000000000000
0x0000080008b00000 0x0000080008bfffff 0x000000000014220c
0x0000000000000000 0x0000000000000000 0x0000000000000000
0x00000000e9300000 0x00000000e93fffff 0x0000000000046200
0x0000080010420000 0x0000080010b9ffff 0x000000000014220c
0x0000000000000000 0x0000000000000000 0x0000000000000000
0x0000080010ba0000 0x000008001131ffff 0x000000000014220c
0x0000000000000000 0x0000000000000000 0x0000000000000000
0x0000080008c00000 0x00000800103fffff 0x000000000014220c
0x0000000000000000 0x0000000000000000 0x0000000000000000
cat: /sys/bus/pci/devices/0000:06:00.0/resource0: Input/output error
cat: /sys/bus/pci/devices/0000:06:00.0/resource2: Input/output error
cat: /sys/bus/pci/devices/0000:06:00.0/resource4: Input/output error
[root@localhost dpdk-19.11]#

 

[root@localhost data1]# cat uio.c 
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <stdint.h>

#define PAGE_SIZE 0x1000

int main()
{
    int fd;
    void *base;
    uint32_t *pmem;

    fd = open("/dev/uio0", O_RDWR);
    if (fd < 0) {
        fprintf(stderr, "Failed to open device
");
        exit(1);
    }
    base = mmap(NULL, PAGE_SIZE, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
    if (base == MAP_FAILED) {
        fprintf(stderr, "Failed to mmap
");
        exit(1);
    }

    pmem = (uint32_t*)base;

    printf("pmem[0] = 0x%08x
", pmem[0]);
    printf("pmem[1] = 0x%08x
", pmem[1]);

[root@localhost data1]# ./uio 
pmem[0] = 0x01040080
pmem[1] = 0x002250c1

 portio

Since these ioport regions can not be mapped, they will not appear under /sys/class/uio/uioX/maps/ like the normal memory described above. Without information about the port regions a hardware has to offer, it becomes difficult for the userspace part of the driver to find out which ports belong to which UIO device.

To address this situation, the new directory /sys/class/uio/uioX/portio/ was added. It only exists if the driver wants to pass information about one or more port regions to userspace. If that is the case, subdirectories named port0, port1, and so on, will appear underneath /sys/class/uio/uioX/portio/.

Each portX/ directory contains four read-only files that show name, start, size, and type of the port region:

• name: A string identifier for this port region. The string is optional and can be empty. Drivers can set it to make it easier for userspace to find a certain port region.

• start: The first port of this region.

• size: The number of ports in this region.

• porttype: A string describing the type of port.

igb_uio & igb & ioremap & mmap

igb_uio 驱动会遍历该 PCI 设备的 BAR 空间，对于类型为存储器空间 IORESOURCE_MEM 的 BAR（Memory BAR），将其物理地址、大小等信息保存到 uio_info 结构的 mem 数组中；将类型为寄存器空间 IORESOURCE_IO 的 BAR（IO BAR），将其物理地址、大小等信息保存到 uio_info 结构的 port 数组中。

[root@localhost dpdk-19.11]# lspci -s 0000:06:00.0 -v
06:00.0 Ethernet controller: Huawei Technologies Co., Ltd. Hi1822 Family (2*25GE) (rev 45)
        Subsystem: Huawei Technologies Co., Ltd. Device d139
        Flags: fast devsel, NUMA node 0
        Memory at 80010400000 (64-bit, prefetchable) [size=128K]
        Memory at 80011320000 (64-bit, prefetchable) [size=32K]
        Memory at 80008b00000 (64-bit, prefetchable) [size=1M]
        Expansion ROM at e9300000 [disabled] [size=1M]
        Capabilities: [40] Express Endpoint, MSI 00
        Capabilities: [80] MSI: Enable- Count=1/32 Maskable+ 64bit+
        Capabilities: [a0] MSI-X: Enable- Count=32 Masked-
        Capabilities: [b0] Power Management version 3
        Capabilities: [c0] Vital Product Data
        Capabilities: [100] Advanced Error Reporting
        Capabilities: [150] Alternative Routing-ID Interpretation (ARI)
        Capabilities: [200] Single Root I/O Virtualization (SR-IOV)
        Capabilities: [310] #19
        Capabilities: [4e0] Device Serial Number 44-a1-91-ff-ff-a4-9b-ec
        Capabilities: [4f0] Transaction Processing Hints
        Capabilities: [600] Vendor Specific Information: ID=0000 Rev=0 Len=028 <?>
        Capabilities: [630] Access Control Services
        Kernel driver in use: igb_uio
        Kernel modules: hinic

[root@localhost dpdk-19.11]#

而 igb 驱动同样也会遍历 BAR 空间，但是它不会记录空间的物理地址，而是调用 ioremap() 将物理地址映射为虚拟地址，然后驱动就可以在内核态中读写映射出来的虚拟地址，而不是像 igb_uio 驱动似的在用户态中进行读写。