1、 main线程没有调用pthread_join
一、初始化
1、rte_eal_cpu_init()函数中，通过读取/sys/devices/system/cpu/cpuX/下的相关信息，确定当前系统有哪些CPU核，已经每个核属于哪个CPU Socket。
2、eal_parse_args()函数，解析-c参数，确认哪些CPU核是可以使用的，以及设置第一个核为MASTER。
[root@localhost ring_test]# gdb ./build/app/dpdk_test GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-119.el7 Copyright (C) 2013 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "aarch64-redhat-linux-gnu". For bug reporting instructions, please see: <http://www.gnu.org/software/gdb/bugs/>... ./build/app/dpdk_test: No such file or directory. (gdb) set args -c 0x3f (gdb) b /data1/dpdk-19.11/lib/librte_eal/linux/eal/eal_thread.c:128 No symbol table is loaded. Use the "file" command. Make breakpoint pending on future shared library load? (y or [n]) n (gdb) quit [root@localhost ring_test]# gdb ./build/app/test_ring GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-119.el7 Copyright (C) 2013 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "aarch64-redhat-linux-gnu". For bug reporting instructions, please see: <http://www.gnu.org/software/gdb/bugs/>... Reading symbols from /data1/dpdk-19.11/demo/ring_test/build/app/test_ring...done. (gdb) set args -c 0x3f (gdb) b /data1/dpdk-19.11/lib/librte_eal/linux/eal/eal_thread.c:128 Breakpoint 1 at 0x592bf0: file /data1/dpdk-19.11/lib/librte_eal/linux/eal/eal_thread.c, line 128. (gdb) r Starting program: /data1/dpdk-19.11/demo/ring_test/./build/app/test_ring -c 0x3f [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib64/libthread_db.so.1". EAL: Detected 128 lcore(s) EAL: Detected 4 NUMA nodes [New Thread 0xffffbe43d910 (LWP 70068)] EAL: Multi-process socket /var/run/dpdk/rte/mp_socket [New Thread 0xffffbdc2d910 (LWP 70069)] EAL: Selected IOVA mode 'PA' EAL: No available hugepages reported in hugepages-2048kB EAL: Probing VFIO support... EAL: VFIO support initialized [New Thread 0xffffbd40d910 (LWP 70070)] [Switching to Thread 0xffffbd40d910 (LWP 70070)] Breakpoint 1, eal_thread_loop (arg=0x0) at /data1/dpdk-19.11/lib/librte_eal/linux/eal/eal_thread.c:133 133 n = read(m2s, &c, 1); (gdb) bt #0 eal_thread_loop (arg=0x0) at /data1/dpdk-19.11/lib/librte_eal/linux/eal/eal_thread.c:133 #1 0x0000ffffbe617d38 in start_thread (arg=0xffffbd40d910) at pthread_create.c:309 #2 0x0000ffffbe55f5f0 in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:91 (gdb) n [New Thread 0xffffbcbfd910 (LWP 70071)] [Switching to Thread 0xffffbcbfd910 (LWP 70071)] Breakpoint 1, eal_thread_loop (arg=0x0) at /data1/dpdk-19.11/lib/librte_eal/linux/eal/eal_thread.c:133 133 n = read(m2s, &c, 1); (gdb) n [New Thread 0xffffbc3ed910 (LWP 70079)] [Switching to Thread 0xffffbc3ed910 (LWP 70079)] Breakpoint 1, eal_thread_loop (arg=0x0) at /data1/dpdk-19.11/lib/librte_eal/linux/eal/eal_thread.c:133 133 n = read(m2s, &c, 1); (gdb) n [New Thread 0xffffbbbdd910 (LWP 70084)] [Switching to Thread 0xffffbbbdd910 (LWP 70084)] Breakpoint 1, eal_thread_loop (arg=0x0) at /data1/dpdk-19.11/lib/librte_eal/linux/eal/eal_thread.c:133 133 n = read(m2s, &c, 1); (gdb) n [New Thread 0xffffbb3cd910 (LWP 70087)] [Switching to Thread 0xffffbd40d910 (LWP 70070)] Breakpoint 1, eal_thread_loop (arg=0x0) at /data1/dpdk-19.11/lib/librte_eal/linux/eal/eal_thread.c:133 133 n = read(m2s, &c, 1); (gdb) n [Switching to Thread 0xffffbb3cd910 (LWP 70087)] Breakpoint 1, eal_thread_loop (arg=0x0) at /data1/dpdk-19.11/lib/librte_eal/linux/eal/eal_thread.c:133 133 n = read(m2s, &c, 1); (gdb) n [Switching to Thread 0xffffbc3ed910 (LWP 70079)] Breakpoint 1, eal_thread_loop (arg=0x0) at /data1/dpdk-19.11/lib/librte_eal/linux/eal/eal_thread.c:133 133 n = read(m2s, &c, 1); (gdb) n [Switching to Thread 0xffffbb3cd910 (LWP 70087)] Breakpoint 1, eal_thread_loop (arg=0x0) at /data1/dpdk-19.11/lib/librte_eal/linux/eal/eal_thread.c:133 133 n = read(m2s, &c, 1); (gdb) n [Switching to Thread 0xffffbcbfd910 (LWP 70071)] Breakpoint 1, eal_thread_loop (arg=0x0) at /data1/dpdk-19.11/lib/librte_eal/linux/eal/eal_thread.c:133 133 n = read(m2s, &c, 1); (gdb) n [Switching to Thread 0xffffbbbdd910 (LWP 70084)] Breakpoint 1, eal_thread_loop (arg=0x0) at /data1/dpdk-19.11/lib/librte_eal/linux/eal/eal_thread.c:133 133 n = read(m2s, &c, 1); (gdb) n EAL: PCI device 0000:05:00.0 on NUMA socket 0 EAL: probe driver: 19e5:200 net_hinic net_hinic: Initializing pf hinic-0000:05:00.0 in primary process net_hinic: Device 0000:05:00.0 hwif attribute: net_hinic: func_idx:0, p2p_idx:0, pciintf_idx:0, vf_in_pf:0, ppf_idx:0, global_vf_id:15, func_type:2 net_hinic: num_aeqs:4, num_ceqs:4, num_irqs:32, dma_attr:2 net_hinic: API CMD poll status timeout net_hinic: chain type: 0x7 net_hinic: chain hw cpld error: 0x1 net_hinic: chain hw check error: 0x0 net_hinic: chain hw current fsm: 0x0 net_hinic: chain hw current ci: 0x0 net_hinic: Chain hw current pi: 0x1 net_hinic: Send msg to mgmt failed net_hinic: Failed to get board info, err: -110, status: 0x0, out size: 0x0 net_hinic: Check card workmode failed, dev_name: 0000:05:00.0 net_hinic: Create nic device failed, dev_name: 0000:05:00.0 net_hinic: Initialize 0000:05:00.0 in primary failed EAL: Requested device 0000:05:00.0 cannot be used EAL: PCI device 0000:06:00.0 on NUMA socket 0 EAL: probe driver: 19e5:200 net_hinic EAL: using IOMMU type 1 (Type 1) net_hinic: Initializing pf hinic-0000:06:00.0 in primary process net_hinic: Device 0000:06:00.0 hwif attribute: net_hinic: func_idx:1, p2p_idx:1, pciintf_idx:0, vf_in_pf:0, ppf_idx:0, global_vf_id:135, func_type:0 net_hinic: num_aeqs:4, num_ceqs:4, num_irqs:32, dma_attr:2 net_hinic: Get public resource capability: net_hinic: host_id: 0x0, ep_id: 0x1, intr_type: 0x0, max_cos_id: 0x7, er_id: 0x1, port_id: 0x1 net_hinic: host_total_function: 0xf2, host_oq_id_mask_val: 0x8, max_vf: 0x78 net_hinic: pf_num: 0x2, pf_id_start: 0x0, vf_num: 0xf0, vf_id_start: 0x10 net_hinic: Get l2nic resource capability: net_hinic: max_sqs: 0x10, max_rqs: 0x10, vf_max_sqs: 0x4, vf_max_rqs: 0x4 net_hinic: Initialize 0000:06:00.0 in primary successfully EAL: PCI device 0000:7d:00.0 on NUMA socket 0 EAL: probe driver: 19e5:a222 net_hns3 EAL: PCI device 0000:7d:00.1 on NUMA socket 0 EAL: probe driver: 19e5:a221 net_hns3 EAL: PCI device 0000:7d:00.2 on NUMA socket 0 EAL: probe driver: 19e5:a222 net_hns3 EAL: PCI device 0000:7d:00.3 on NUMA socket 0 EAL: probe driver: 19e5:a221 net_hns3 Starting lcores. Starting core 1 core 1: Received 1 Starting core 2 core 2: Received 3 Starting core 3 core 3: Received 5 [Switching to Thread 0xffffbd40d910 (LWP 70070)] Breakpoint 1, eal_thread_loop (arg=0x0) at /data1/dpdk-19.11/lib/librte_eal/linux/eal/eal_thread.c:133 133 n = read(m2s, &c, 1); (gdb) n Starting core 4 core 4: Received 8 [Switching to Thread 0xffffbbbdd910 (LWP 70084)] Breakpoint 1, eal_thread_loop (arg=0x0) at /data1/dpdk-19.11/lib/librte_eal/linux/eal/eal_thread.c:133 133 n = read(m2s, &c, 1); (gdb) n [Switching to Thread 0xffffbcbfd910 (LWP 70071)] Breakpoint 1, eal_thread_loop (arg=0x0) at /data1/dpdk-19.11/lib/librte_eal/linux/eal/eal_thread.c:133 133 n = read(m2s, &c, 1); (gdb) n Starting core 5 Waiting for lcores to finish. [Switching to Thread 0xffffbc3ed910 (LWP 70079)] Breakpoint 1, eal_thread_loop (arg=0x0) at /data1/dpdk-19.11/lib/librte_eal/linux/eal/eal_thread.c:133 133 n = read(m2s, &c, 1); (gdb) info threads Id Target Id Frame 8 Thread 0xffffbb3cd910 (LWP 70087) "lcore-slave-5" 0x0000ffffbe52a9c4 in nanosleep () at ../sysdeps/unix/syscall-template.S:83 7 Thread 0xffffbbbdd910 (LWP 70084) "lcore-slave-4" 0x0000ffffbe61f3a8 in read () at ../sysdeps/unix/syscall-template.S:83 * 6 Thread 0xffffbc3ed910 (LWP 70079) "lcore-slave-3" eal_thread_loop (arg=0x0) at /data1/dpdk-19.11/lib/librte_eal/linux/eal/eal_thread.c:133 5 Thread 0xffffbcbfd910 (LWP 70071) "lcore-slave-2" 0x0000ffffbe61f3a8 in read () at ../sysdeps/unix/syscall-template.S:83 4 Thread 0xffffbd40d910 (LWP 70070) "lcore-slave-1" 0x0000ffffbe61f3a8 in read () at ../sysdeps/unix/syscall-template.S:83 3 Thread 0xffffbdc2d910 (LWP 70069) "rte_mp_handle" 0x0000ffffbe61f928 in recvmsg () at ../sysdeps/unix/syscall-template.S:83 2 Thread 0xffffbe43d910 (LWP 70068) "eal-intr-thread" 0x0000ffffbe55f7c4 in __GI_epoll_pwait (epfd=10, events=0xffffbe43cfd0, maxevents=<optimized out>, timeout=<optimized out>, set=0x0) at ../sysdeps/unix/sysv/linux/epoll_pwait.c:48 1 Thread 0xffffbe805510 (LWP 70065) "test_ring" 0x00000000005a196c in rte_eal_wait_lcore (slave_id=5) at /data1/dpdk-19.11/lib/librte_eal/common/eal_common_launch.c:30 (gdb) n
[root@localhost ring_test]# gdb ./build/app/test_ring GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-119.el7 Copyright (C) 2013 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "aarch64-redhat-linux-gnu". For bug reporting instructions, please see: <http://www.gnu.org/software/gdb/bugs/>... Reading symbols from /data1/dpdk-19.11/demo/ring_test/build/app/test_ring...done. (gdb) set args -c 0x3f (gdb) b main.c:20 Breakpoint 1 at 0x464cd8: file /data1/dpdk-19.11/demo/ring_test/main.c, line 20. (gdb) r Starting program: /data1/dpdk-19.11/demo/ring_test/./build/app/test_ring -c 0x3f [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib64/libthread_db.so.1". EAL: Detected 128 lcore(s) EAL: Detected 4 NUMA nodes [New Thread 0xffffbe43d910 (LWP 70505)] EAL: Multi-process socket /var/run/dpdk/rte/mp_socket [New Thread 0xffffbdc2d910 (LWP 70506)] EAL: Selected IOVA mode 'PA' EAL: No available hugepages reported in hugepages-2048kB EAL: Probing VFIO support... EAL: VFIO support initialized [New Thread 0xffffbd40d910 (LWP 70508)] [New Thread 0xffffbcbfd910 (LWP 70509)] [New Thread 0xffffbc3ed910 (LWP 70510)] [New Thread 0xffffbbbdd910 (LWP 70511)] [New Thread 0xffffbb3cd910 (LWP 70512)] EAL: PCI device 0000:05:00.0 on NUMA socket 0 EAL: probe driver: 19e5:200 net_hinic net_hinic: Initializing pf hinic-0000:05:00.0 in primary process net_hinic: Device 0000:05:00.0 hwif attribute: net_hinic: func_idx:0, p2p_idx:0, pciintf_idx:0, vf_in_pf:0, ppf_idx:0, global_vf_id:15, func_type:2 net_hinic: num_aeqs:4, num_ceqs:4, num_irqs:32, dma_attr:2 net_hinic: API CMD poll status timeout net_hinic: chain type: 0x7 net_hinic: chain hw cpld error: 0x1 net_hinic: chain hw check error: 0x0 net_hinic: chain hw current fsm: 0x0 net_hinic: chain hw current ci: 0x0 net_hinic: Chain hw current pi: 0x1 net_hinic: Send msg to mgmt failed net_hinic: Failed to get board info, err: -110, status: 0x0, out size: 0x0 net_hinic: Check card workmode failed, dev_name: 0000:05:00.0 net_hinic: Create nic device failed, dev_name: 0000:05:00.0 net_hinic: Initialize 0000:05:00.0 in primary failed EAL: Requested device 0000:05:00.0 cannot be used EAL: PCI device 0000:06:00.0 on NUMA socket 0 EAL: probe driver: 19e5:200 net_hinic EAL: using IOMMU type 1 (Type 1) net_hinic: Initializing pf hinic-0000:06:00.0 in primary process net_hinic: Device 0000:06:00.0 hwif attribute: net_hinic: func_idx:1, p2p_idx:1, pciintf_idx:0, vf_in_pf:0, ppf_idx:0, global_vf_id:135, func_type:0 net_hinic: num_aeqs:4, num_ceqs:4, num_irqs:32, dma_attr:2 net_hinic: Get public resource capability: net_hinic: host_id: 0x0, ep_id: 0x1, intr_type: 0x0, max_cos_id: 0x7, er_id: 0x1, port_id: 0x1 net_hinic: host_total_function: 0xf2, host_oq_id_mask_val: 0x8, max_vf: 0x78 net_hinic: pf_num: 0x2, pf_id_start: 0x0, vf_num: 0xf0, vf_id_start: 0x10 net_hinic: Get l2nic resource capability: net_hinic: max_sqs: 0x10, max_rqs: 0x10, vf_max_sqs: 0x4, vf_max_rqs: 0x4 net_hinic: Initialize 0000:06:00.0 in primary successfully EAL: PCI device 0000:7d:00.0 on NUMA socket 0 EAL: probe driver: 19e5:a222 net_hns3 EAL: PCI device 0000:7d:00.1 on NUMA socket 0 EAL: probe driver: 19e5:a221 net_hns3 EAL: PCI device 0000:7d:00.2 on NUMA socket 0 EAL: probe driver: 19e5:a222 net_hns3 EAL: PCI device 0000:7d:00.3 on NUMA socket 0 EAL: probe driver: 19e5:a221 net_hns3 Starting lcores. [Switching to Thread 0xffffbcbfd910 (LWP 70509)] Breakpoint 1, lcore_recv (p=0xfffffffff3f8) at /data1/dpdk-19.11/demo/ring_test/main.c:20 20 printf("Starting core %u ", lcore_id); (gdb) bt #0 lcore_recv (p=0xfffffffff3f8) at /data1/dpdk-19.11/demo/ring_test/main.c:20 #1 0x0000000000592d88 in eal_thread_loop (arg=0x0) at /data1/dpdk-19.11/lib/librte_eal/linux/eal/eal_thread.c:153 #2 0x0000ffffbe617d38 in start_thread (arg=0xffffbcbfd910) at pthread_create.c:309 #3 0x0000ffffbe55f5f0 in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:91 (gdb) b /data1/dpdk-19.11/lib/librte_eal/linux/eal/eal_thread.c:154 Breakpoint 2 at 0x592d8c: file /data1/dpdk-19.11/lib/librte_eal/linux/eal/eal_thread.c, line 154. (gdb) n Starting core 2 [Switching to Thread 0xffffbc3ed910 (LWP 70510)] Breakpoint 1, lcore_recv (p=0xfffffffff3f8) at /data1/dpdk-19.11/demo/ring_test/main.c:20 20 printf("Starting core %u ", lcore_id); (gdb) n Starting core 3 Waiting for lcores to finish. 22 while (rte_ring_dequeue(p->send_ring, &vp) != 0){ (gdb) p lcore_id $1 = 3 (gdb) n 25 struct data * d = (struct data *) vp; (gdb) n 26 printf("core %u: Received %d ", lcore_id, d->value); (gdb) n core 2: Received 3 core 3: Received 1 [Switching to Thread 0xffffbd40d910 (LWP 70508)] Breakpoint 1, lcore_recv (p=0xfffffffff3f8) at /data1/dpdk-19.11/demo/ring_test/main.c:20 20 printf("Starting core %u ", lcore_id); (gdb) n Starting core 1 [Switching to Thread 0xffffbc3ed910 (LWP 70510)] Breakpoint 2, eal_thread_loop (arg=0x0) at /data1/dpdk-19.11/lib/librte_eal/linux/eal/eal_thread.c:154 154 lcore_config[lcore_id].ret = ret; (gdb) bt #0 eal_thread_loop (arg=0x0) at /data1/dpdk-19.11/lib/librte_eal/linux/eal/eal_thread.c:154 #1 0x0000ffffbe617d38 in start_thread (arg=0xffffbc3ed910) at pthread_create.c:309 #2 0x0000ffffbe55f5f0 in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:91 (gdb) n 155 rte_wmb(); (gdb) n 160 if (lcore_config[lcore_id].core_role == ROLE_SERVICE) (gdb) n 163 lcore_config[lcore_id].state = FINISHED; (gdb) n 164 } (gdb) n 133 n = read(m2s, &c, 1); (gdb) list 128 while (1) { 129 void *fct_arg; 130 131 /* wait command */ 132 do { 133 n = read(m2s, &c, 1); 134 } while (n < 0 && errno == EINTR); 135 136 if (n <= 0) 137 rte_panic("cannot read on configuration pipe "); (gdb) p lcore_id $2 = 3 (gdb) n [Switching to Thread 0xffffbcbfd910 (LWP 70509)] Breakpoint 2, eal_thread_loop (arg=0x0) at /data1/dpdk-19.11/lib/librte_eal/linux/eal/eal_thread.c:154 154 lcore_config[lcore_id].ret = ret; (gdb) p lcore_id $3 = 2 (gdb) list 149 rte_panic("NULL function pointer "); 150 151 /* call the function and store the return value */ 152 fct_arg = lcore_config[lcore_id].arg; 153 ret = lcore_config[lcore_id].f(fct_arg); 154 lcore_config[lcore_id].ret = ret; 155 rte_wmb(); 156 157 /* when a service core returns, it should go directly to WAIT 158 * state, because the application will not lcore_wait() for it. (gdb) n 155 rte_wmb(); (gdb) n 160 if (lcore_config[lcore_id].core_role == ROLE_SERVICE) (gdb) list 155 rte_wmb(); 156 157 /* when a service core returns, it should go directly to WAIT 158 * state, because the application will not lcore_wait() for it. 159 */ 160 if (lcore_config[lcore_id].core_role == ROLE_SERVICE) 161 lcore_config[lcore_id].state = WAIT; 162 else 163 lcore_config[lcore_id].state = FINISHED; 164 } (gdb) n 163 lcore_config[lcore_id].state = FINISHED; (gdb) n 164 } (gdb) n 133 n = read(m2s, &c, 1); (gdb) list 128 while (1) { 129 void *fct_arg; 130 131 /* wait command */ 132 do { 133 n = read(m2s, &c, 1); 134 } while (n < 0 && errno == EINTR); 135 136 if (n <= 0) 137 rte_panic("cannot read on configuration pipe "); (gdb) n core 1: Received 5 [Switching to Thread 0xffffbb3cd910 (LWP 70512)] Breakpoint 1, lcore_recv (p=0xfffffffff3f8) at /data1/dpdk-19.11/demo/ring_test/main.c:20 20 printf("Starting core %u ", lcore_id); (gdb) n Starting core 5 22 while (rte_ring_dequeue(p->send_ring, &vp) != 0){ (gdb) n 25 struct data * d = (struct data *) vp; (gdb) n 26 printf("core %u: Received %d ", lcore_id, d->value); (gdb) n core 5: Received 8 27 d->value ++; (gdb) n 28 rte_ring_enqueue(p->recv_ring, (void *)d); (gdb) n 30 return 0; (gdb) n 31 } (gdb) n Breakpoint 2, eal_thread_loop (arg=0x0) at /data1/dpdk-19.11/lib/librte_eal/linux/eal/eal_thread.c:154 154 lcore_config[lcore_id].ret = ret; (gdb) c Continuing. [Switching to Thread 0xffffbd40d910 (LWP 70508)] Breakpoint 2, eal_thread_loop (arg=0x0) at /data1/dpdk-19.11/lib/librte_eal/linux/eal/eal_thread.c:154 154 lcore_config[lcore_id].ret = ret; (gdb) c Continuing. [Switching to Thread 0xffffbbbdd910 (LWP 70511)] Breakpoint 1, lcore_recv (p=0xfffffffff3f8) at /data1/dpdk-19.11/demo/ring_test/main.c:20 20 printf("Starting core %u ", lcore_id); (gdb) c Continuing. Starting core 4
