rte_pktmbuf_append rte_pktmbuf_chain rte_pktmbuf_clone

DPDK: rte_mbuf

https://www.yuque.com/zzqcn/opensource/oirzxh

 

Mbuf

https://promisechen.github.io/dpdk/zp/mbuf.html

 https://promisechen.github.io/dpdk/zp/mbuf.html

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <errno.h>
#include <sys/queue.h>

#include <rte_launch.h>
#include <rte_eal.h>
#include <rte_per_lcore.h>
#include <rte_lcore.h>
#include <rte_debug.h>
#include <rte_mempool.h>
#include <rte_mbuf.h>
#include <rte_ether.h>

#define RAW_MEMPOOL

#define MBUF_COUNT  (1024-1)
#define PRIV_SIZE   16
// ETH_MAX_len = 1518
// ETH_MTU = ETH_MAX_LEN - ETH_HDR_LEN - ETHER_CRC_LEN = 1518 - 14 - 4 = 1500
#define MBUF_DATAROOM_SIZE (RTE_PKTMBUF_HEADROOM + RTE_ETHER_MAX_LEN)
#define MBUF_SIZE   (sizeof(struct rte_mbuf) + PRIV_SIZE + MBUF_DATAROOM_SIZE)
#define CACHE_SIZE  32


struct rte_mempool* mpool;
struct rte_mbuf *m = NULL;
static void mbuf_dump(struct rte_mbuf* m)
{
    printf("RTE_PKTMBUF_HEADROOM: %u
", RTE_PKTMBUF_HEADROOM);
    printf("sizeof(mbuf): %lu
", sizeof(struct rte_mbuf));
    printf("m: %p
", m);
    printf("m->refcnt: %u
", m->refcnt);
    printf("m->buf_addr: %p
", m->buf_addr);
    printf("m->data_off: %u
", m->data_off);
    printf("m->buf_len: %u
", m->buf_len);
    printf("m->pkt_len: %u
", m->pkt_len);
    printf("m->data_len: %u
", m->data_len);
    printf("m->nb_segs: %u
", m->nb_segs);
    printf("m->next: %p
", m->next);
    printf("m->buf_addr+m->data_off: %p
", (char*)m->buf_addr+m->data_off);
    printf("rte_pktmbuf_mtod(m): %p
", rte_pktmbuf_mtod(m, char*));
    printf("rte_pktmbuf_data_len(m): %u
", rte_pktmbuf_data_len(m));
    printf("rte_pktmbuf_pkt_len(m): %u
", rte_pktmbuf_pkt_len(m)); 
    printf("rte_pktmbuf_headroom(m): %u
", rte_pktmbuf_headroom(m));
    printf("rte_pktmbuf_tailroom(m): %u
", rte_pktmbuf_tailroom(m));
    printf("rte_pktmbuf_data_room_size(mpool): %u
", 
                rte_pktmbuf_data_room_size(m->pool));
    printf("rte_pktmbuf_priv_size(mpool): %u

",
                rte_pktmbuf_priv_size(m->pool));
}

static int mpool_init(void)
{
    struct rte_mempool_objsz objsz;
    uint32_t mbuf_size;

#ifdef RAW_MEMPOOL
    struct rte_pktmbuf_pool_private priv;
    priv.mbuf_data_room_size = MBUF_DATAROOM_SIZE;
    priv.mbuf_priv_size = PRIV_SIZE;

    mpool = rte_mempool_create("test_pool",
                               MBUF_COUNT,
                               MBUF_SIZE,
                               CACHE_SIZE,
                               sizeof(struct rte_pktmbuf_pool_private),
                               rte_pktmbuf_pool_init,
                               &priv,
                               rte_pktmbuf_init,
                               NULL,
                               SOCKET_ID_ANY,
                               MEMPOOL_F_SC_GET);
#else
    mpool = rte_pktmbuf_pool_create("test_pool",
                                    MBUF_COUNT,
                                    CACHE_SIZE,
                                    PRIV_SIZE,
                                    MBUF_DATAROOM_SIZE,
                                    SOCKET_ID_ANY);
#endif
    if(NULL == mpool)
        return -1;
    
    mbuf_size = rte_mempool_calc_obj_size(MBUF_SIZE, 0, &objsz);
    printf("mbuf_size: %u
", mbuf_size);
    printf("elt_size: %u, header_size: %u, trailer_size: %u, total_size: %u
",
        objsz.elt_size, objsz.header_size, objsz.trailer_size, objsz.total_size);

    return 0;
}
int lcore_hello()
{
    struct rte_mbuf *m2;
    printf("mempool count  : %u
", rte_mempool_avail_count(mpool));
    unsigned lcore_id = rte_lcore_id();
    printf("Starting core %u
", lcore_id);
    if(1 == lcore_id)
    {
    m = rte_pktmbuf_alloc(mpool);
    mbuf_dump(m);
    rte_pktmbuf_append(m, 1000);
    mbuf_dump(m);
    m2 = rte_pktmbuf_alloc(mpool);
    rte_pktmbuf_append(m2, 500);
    mbuf_dump(m2);
    rte_pktmbuf_chain(m, m2);
    mbuf_dump(m);
    printf("mempool count before free  : %u
", rte_mempool_avail_count(mpool));
    rte_pktmbuf_free(m);
    printf("mempool count after free  : %u
", rte_mempool_avail_count(mpool));
    }
    else 
    {
    }
    return 0;
}



int
main(int argc, char **argv)
{
    int ret;
    unsigned lcore_id;

    ret = rte_eal_init(argc, argv);
    if (ret < 0)
        rte_panic("Cannot init EAL
");
    mpool_init();

    /* call on every slave lcore */
    RTE_LCORE_FOREACH_SLAVE(lcore_id) {
        rte_eal_remote_launch(lcore_hello, NULL, lcore_id);
    }


    rte_eal_mp_wait_lcore();
    return 0;
}

  INSTALL-MAP helloworld.map
[root@localhost mbuf]# build/app/helloworld -c 0x3 -n 4
EAL: Detected 128 lcore(s)
EAL: Detected 4 NUMA nodes
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
EAL: Selected IOVA mode 'PA'
EAL: No available hugepages reported in hugepages-2048kB
EAL: Probing VFIO support...
EAL: VFIO support initialized
EAL: PCI device 0000:05:00.0 on NUMA socket 0
EAL:   probe driver: 19e5:200 net_hinic
EAL:   using IOMMU type 1 (Type 1)
net_hinic: Initializing pf hinic-0000:05:00.0 in primary process
net_hinic: Device 0000:05:00.0 hwif attribute:
net_hinic: func_idx:0, p2p_idx:0, pciintf_idx:0, vf_in_pf:0, ppf_idx:0, global_vf_id:15, func_type:2
net_hinic: num_aeqs:4, num_ceqs:4, num_irqs:32, dma_attr:2
net_hinic: Get public resource capability:
net_hinic: host_id: 0x0, ep_id: 0x0, intr_type: 0x0, max_cos_id: 0x7, er_id: 0x0, port_id: 0x0
net_hinic: host_total_function: 0xf2, host_oq_id_mask_val: 0x8, max_vf: 0x78
net_hinic: pf_num: 0x2, pf_id_start: 0x0, vf_num: 0xf0, vf_id_start: 0x10
net_hinic: Get share resource capability:
net_hinic: host_pctxs: 0x0, host_cctxs: 0x0, host_scqs: 0x0, host_srqs: 0x0, host_mpts: 0x0
net_hinic: Get l2nic resource capability:
net_hinic: max_sqs: 0x10, max_rqs: 0x10, vf_max_sqs: 0x4, vf_max_rqs: 0x4
net_hinic: Initialize 0000:05:00.0 in primary successfully
EAL: PCI device 0000:06:00.0 on NUMA socket 0
EAL:   probe driver: 19e5:200 net_hinic
EAL: PCI device 0000:7d:00.0 on NUMA socket 0
EAL:   probe driver: 19e5:a222 net_hns3
EAL: PCI device 0000:7d:00.1 on NUMA socket 0
EAL:   probe driver: 19e5:a221 net_hns3
EAL: PCI device 0000:7d:00.2 on NUMA socket 0
EAL:   probe driver: 19e5:a222 net_hns3
EAL: PCI device 0000:7d:00.3 on NUMA socket 0
EAL:   probe driver: 19e5:a221 net_hns3
mbuf_size: 2176
elt_size: 1792, header_size: 128, trailer_size: 256, total_size: 2176
mempool count  : 1023
Starting core 1
RTE_PKTMBUF_HEADROOM: 128
sizeof(mbuf): 128
m: 0x13f8d3c00
m->refcnt: 1
m->buf_addr: 0x13f8d3c90
m->data_off: 128
m->buf_len: 1646
m->pkt_len: 0
m->data_len: 0
m->nb_segs: 1
m->next: (nil)
m->buf_addr+m->data_off: 0x13f8d3d10
rte_pktmbuf_mtod(m): 0x13f8d3d10
rte_pktmbuf_data_len(m): 0
rte_pktmbuf_pkt_len(m): 0
rte_pktmbuf_headroom(m): 128
rte_pktmbuf_tailroom(m): 1518
rte_pktmbuf_data_room_size(mpool): 1646
rte_pktmbuf_priv_size(mpool): 16

RTE_PKTMBUF_HEADROOM: 128
sizeof(mbuf): 128
m: 0x13f8d3c00
m->refcnt: 1
m->buf_addr: 0x13f8d3c90
m->data_off: 128
m->buf_len: 1646
m->pkt_len: 1000
m->data_len: 1000
m->nb_segs: 1
m->next: (nil)
m->buf_addr+m->data_off: 0x13f8d3d10
rte_pktmbuf_mtod(m): 0x13f8d3d10
rte_pktmbuf_data_len(m): 1000
rte_pktmbuf_pkt_len(m): 1000
rte_pktmbuf_headroom(m): 128
rte_pktmbuf_tailroom(m): 518
rte_pktmbuf_data_room_size(mpool): 1646
rte_pktmbuf_priv_size(mpool): 16

RTE_PKTMBUF_HEADROOM: 128
sizeof(mbuf): 128
m: 0x13f8d3380
m->refcnt: 1
m->buf_addr: 0x13f8d3410
m->data_off: 128
m->buf_len: 1646
m->pkt_len: 500
m->data_len: 500
m->nb_segs: 1
m->next: (nil)
m->buf_addr+m->data_off: 0x13f8d3490
rte_pktmbuf_mtod(m): 0x13f8d3490
rte_pktmbuf_data_len(m): 500
rte_pktmbuf_pkt_len(m): 500
rte_pktmbuf_headroom(m): 128
rte_pktmbuf_tailroom(m): 1018
rte_pktmbuf_data_room_size(mpool): 1646
rte_pktmbuf_priv_size(mpool): 16

RTE_PKTMBUF_HEADROOM: 128
sizeof(mbuf): 128
m: 0x13f8d3c00
m->refcnt: 1
m->buf_addr: 0x13f8d3c90
m->data_off: 128
m->buf_len: 1646
m->pkt_len: 1500
m->data_len: 1000
m->nb_segs: 2
m->next: 0x13f8d3380
m->buf_addr+m->data_off: 0x13f8d3d10
rte_pktmbuf_mtod(m): 0x13f8d3d10
rte_pktmbuf_data_len(m): 1000
rte_pktmbuf_pkt_len(m): 1500
rte_pktmbuf_headroom(m): 128
rte_pktmbuf_tailroom(m): 518
rte_pktmbuf_data_room_size(mpool): 1646
rte_pktmbuf_priv_size(mpool): 16

mempool count before free  : 1021
mempool count after free  : 1023

rte_pktmbuf_append 

rte_pktmbuf_chain

rte_pktmbuf_clone

int lcore_hello()
{
    struct rte_mbuf *m2,*m3;
    printf("mempool count  : %u
", rte_mempool_avail_count(mpool));
    unsigned lcore_id = rte_lcore_id();
    printf("Starting core %u
", lcore_id);
    if(1 == lcore_id)
    {
        m = rte_pktmbuf_alloc(mpool);
        rte_pktmbuf_append(m, 1000);
        mbuf_dump(m);
        m2 = rte_pktmbuf_clone(m, mpool);
        mbuf_dump(m2);
        m3 = rte_pktmbuf_clone(m, mpool);
        mbuf_dump(m3);
        
        printf("mempool count before free: %u
", rte_mempool_avail_count(mpool));
        printf("m->refcnt: %u
", m->refcnt);
        rte_pktmbuf_free(m);
        printf("mempool count after free: %u
", rte_mempool_avail_count(mpool));
        printf("m->refcnt: %u
", m->refcnt);
        rte_pktmbuf_free(m3);
        printf("mempool count after free: %u
", rte_mempool_avail_count(mpool));
        printf("m->refcnt: %u
", m->refcnt);
        rte_pktmbuf_free(m2);
        printf("mempool count after free: %u
", rte_mempool_avail_count(mpool));
        printf("m->refcnt: %u
", m->refcnt);
        rte_pktmbuf_free(m);
        printf("mempool count after free: %u
", rte_mempool_avail_count(mpool));
        printf("m->refcnt: %u
", m->refcnt);
    }
    else 
    {
    }
    return 0;
}

mempool count  : 1023
Starting core 1
RTE_PKTMBUF_HEADROOM: 128
sizeof(mbuf): 128
m: 0x13f8d3c00
m->refcnt: 1
m->buf_addr: 0x13f8d3c90
m->data_off: 128
m->buf_len: 1646
m->pkt_len: 1000
m->data_len: 1000
m->nb_segs: 1
m->next: (nil)
m->buf_addr+m->data_off: 0x13f8d3d10
rte_pktmbuf_mtod(m): 0x13f8d3d10
rte_pktmbuf_data_len(m): 1000
rte_pktmbuf_pkt_len(m): 1000
rte_pktmbuf_headroom(m): 128
rte_pktmbuf_tailroom(m): 518
rte_pktmbuf_data_room_size(mpool): 1646
rte_pktmbuf_priv_size(mpool): 16

RTE_PKTMBUF_HEADROOM: 128
sizeof(mbuf): 128
m: 0x13f8d3380
m->refcnt: 1     ---不是2
m->buf_addr: 0x13f8d3c90
m->data_off: 128
m->buf_len: 1646
m->pkt_len: 1000
m->data_len: 1000
m->nb_segs: 1
m->next: (nil)
m->buf_addr+m->data_off: 0x13f8d3d10
rte_pktmbuf_mtod(m): 0x13f8d3d10
rte_pktmbuf_data_len(m): 1000
rte_pktmbuf_pkt_len(m): 1000
rte_pktmbuf_headroom(m): 128
rte_pktmbuf_tailroom(m): 518
rte_pktmbuf_data_room_size(mpool): 1646
rte_pktmbuf_priv_size(mpool): 16

RTE_PKTMBUF_HEADROOM: 128
sizeof(mbuf): 128
m: 0x13f8d2b00
m->refcnt: 1   --不是3
m->buf_addr: 0x13f8d3c90
m->data_off: 128
m->buf_len: 1646
m->pkt_len: 1000
m->data_len: 1000
m->nb_segs: 1
m->next: (nil)
m->buf_addr+m->data_off: 0x13f8d3d10
rte_pktmbuf_mtod(m): 0x13f8d3d10
rte_pktmbuf_data_len(m): 1000
rte_pktmbuf_pkt_len(m): 1000
rte_pktmbuf_headroom(m): 128
rte_pktmbuf_tailroom(m): 518
rte_pktmbuf_data_room_size(mpool): 1646
rte_pktmbuf_priv_size(mpool): 16

mempool count before free: 1020
m->refcnt: 3
mempool count after free: 1020  --没有增加
m->refcnt: 2
mempool count after free: 1021  -----竟然增加了
m->refcnt: 1
mempool count after free: 1023
m->refcnt: 1   --不是0
MEMPOOL: obj=0x13f8d3c00, mempool=0x13fae2c00, cookie=f2eef2eedadd2e55
PANIC in rte_mempool_check_cookies():
MEMPOOL: bad header cookie (put)
7: [/lib64/libc.so.6(+0xdf690) [0xffff9762f690]]
6: [/lib64/libpthread.so.0(+0x7d38) [0xffff976e7d38]]
5: [build/app/helloworld(eal_thread_loop+0x324) [0x598fc8]]
4: [build/app/helloworld(lcore_hello+0xab8) [0x465dd8]]
3: [build/app/helloworld(rte_mempool_check_cookies+0xf4) [0x58b678]]
2: [build/app/helloworld(__rte_panic+0xcc) [0x5a0e1c]]
1: [build/app/helloworld(rte_dump_stack+0x1c) [0x5a0cc8]]
Aborted

/* Creates a shallow copy of mbuf */
struct rte_mbuf *
rte_pktmbuf_clone(struct rte_mbuf *md, struct rte_mempool *mp)
{
        struct rte_mbuf *mc, *mi, **prev;
        uint32_t pktlen;
        uint16_t nseg;

        mc = rte_pktmbuf_alloc(mp);
        if (unlikely(mc == NULL))
                return NULL;

        mi = mc;
        prev = &mi->next;
        pktlen = md->pkt_len;
        nseg = 0;

        do {
                nseg++;
                rte_pktmbuf_attach(mi, md);
                *prev = mi;
                prev = &mi->next;
        } while ((md = md->next) != NULL &&
            (mi = rte_pktmbuf_alloc(mp)) != NULL);

        *prev = NULL;
        mc->nb_segs = nseg;
        mc->pkt_len = pktlen;

        /* Allocation of new indirect segment failed */
        if (unlikely(mi == NULL)) {
                rte_pktmbuf_free(mc);
                return NULL;
        }

        __rte_mbuf_sanity_check(mc, 1);
        return mc;
}

static inline void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m)
{
        RTE_ASSERT(RTE_MBUF_DIRECT(mi) &&
            rte_mbuf_refcnt_read(mi) == 1);

        if (RTE_MBUF_HAS_EXTBUF(m)) {
                rte_mbuf_ext_refcnt_update(m->shinfo, 1);
                mi->ol_flags = m->ol_flags;
                mi->shinfo = m->shinfo;
        } else {
                /* if m is not direct, get the mbuf that embeds the data */
                rte_mbuf_refcnt_update(rte_mbuf_from_indirect(m), 1);
                mi->priv_size = m->priv_size;
                mi->ol_flags = m->ol_flags | IND_ATTACHED_MBUF;
        }

        __rte_pktmbuf_copy_hdr(mi, m);

        mi->data_off = m->data_off;
        mi->data_len = m->data_len;
        mi->buf_iova = m->buf_iova;
        mi->buf_addr = m->buf_addr;
        mi->buf_len = m->buf_len;

        mi->next = NULL;  ------next等于null
        mi->pkt_len = mi->data_len;
        mi->nb_segs = 1;

        __rte_mbuf_sanity_check(mi, 1);
        __rte_mbuf_sanity_check(m, 0);
}

double free

MEMPOOL: obj=0x13f8d3c00, mempool=0x13fae2c00, cookie=f2eef2eedadd2e55
PANIC in rte_mempool_check_cookies():
MEMPOOL: bad header cookie (put)
8: [/lib64/libc.so.6(+0xdf690) [0xffffbe55f690]]
7: [/lib64/libpthread.so.0(+0x7d38) [0xffffbe617d38]]
6: [/root/dpdk-19.11/examples/mbuf/build/app/helloworld(eal_thread_loop+0x324) [0x598ec8]]
5: [/root/dpdk-19.11/examples/mbuf/build/app/helloworld(lcore_hello+0x210) [0x466434]]
4: [/root/dpdk-19.11/examples/mbuf/build/app/helloworld() [0x465c2c]]
3: [/root/dpdk-19.11/examples/mbuf/build/app/helloworld(rte_mempool_check_cookies+0xf4) [0x58b578]]
2: [/root/dpdk-19.11/examples/mbuf/build/app/helloworld(__rte_panic+0xcc) [0x5a0d1c]]
1: [/root/dpdk-19.11/examples/mbuf/build/app/helloworld(rte_dump_stack+0x1c) [0x5a0bc8]]

Program received signal SIGABRT, Aborted.
[Switching to Thread 0xffffbd40d910 (LWP 128933)]
0x0000ffffbe4b5238 in raise () from /lib64/libc.so.6
Missing separate debuginfos, use: debuginfo-install glibc-2.17-324.el7_9.aarch64 libgcc-4.8.5-44.el7.aarch64 numactl-libs-2.0.12-5.el7.aarch64
(gdb) bt
#0  0x0000ffffbe4b5238 in raise () from /lib64/libc.so.6
#1  0x0000ffffbe4b68b0 in abort () from /lib64/libc.so.6
#2  0x00000000005a0d24 in __rte_panic (funcname=0xbe5610 <__func__.7557> "rte_mempool_check_cookies", format=0xbe5290 "MEMPOOL: bad header cookie (put)
%.0s") at /root/dpdk-19.11/lib/librte_eal/linux/eal/eal_debug.c:62
#3  0x000000000058b578 in rte_mempool_check_cookies (mp=0x13fae2c00, obj_table_const=0xffffbd40cec0, n=0, free=0) at /root/dpdk-19.11/lib/librte_mempool/rte_mempool.c:1036
#4  0x0000000000465c2c in rte_mempool_generic_put (cache=0x13faedd80, n=1, obj_table=0xffffbd40cec0, mp=0x13fae2c00) at /root/dpdk-19.11/arm64-armv8a-linuxapp-gcc/include/rte_mempool.h:1364
#5  rte_mempool_put_bulk (n=1, obj_table=0xffffbd40cec0, mp=0x13fae2c00) at /root/dpdk-19.11/arm64-armv8a-linuxapp-gcc/include/rte_mempool.h:1388
#6  rte_mempool_put (obj=0x13f8d3c00, mp=0x13fae2c00) at /root/dpdk-19.11/arm64-armv8a-linuxapp-gcc/include/rte_mempool.h:1406
#7  rte_mbuf_raw_free (m=0x13f8d3c00) at /root/dpdk-19.11/arm64-armv8a-linuxapp-gcc/include/rte_mbuf.h:579
#8  rte_pktmbuf_free_seg (m=0x13f8d3c00) at /root/dpdk-19.11/arm64-armv8a-linuxapp-gcc/include/rte_mbuf.h:1223
#9  rte_pktmbuf_free (m=0x13f8d3c00) at /root/dpdk-19.11/arm64-armv8a-linuxapp-gcc/include/rte_mbuf.h:1244
#10 0x0000000000466434 in lcore_hello () at /root/dpdk-19.11/examples/mbuf/main.c:121
#11 0x0000000000598ec8 in eal_thread_loop (arg=0x0) at /root/dpdk-19.11/lib/librte_eal/linux/eal/eal_thread.c:153
#12 0x0000ffffbe617d38 in start_thread () from /lib64/libpthread.so.0
#13 0x0000ffffbe55f690 in thread_start () from /lib64/libc.so.6
(gdb)