[root@localhost memzone]# build/app/helloworld -c 0xf -n 4
EAL: Detected 128 lcore(s)
EAL: Detected 4 NUMA nodes
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
EAL: Selected IOVA mode 'PA'
EAL: 256 hugepages of size 2097152 （2MB）reserved, but no mounted hugetlbfs found for that size
EAL: Probing VFIO support...
EAL: VFIO support initialized
EAL: PCI device 0000:05:00.0 on NUMA socket 0
EAL: probe driver: 19e5:200 net_hinic
EAL: using IOMMU type 1 (Type 1)
net_hinic: Initializing pf hinic-0000:05:00.0 in primary proces
[root@localhost dpdk-19.11]# ls /sys/kernel/mm/hugepages
hugepages-2048kB hugepages-524288kB
[root@localhost dpdk-19.11]#
[root@localhost dpdk-19.11]# mount | grep huge
cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,hugetlb)
hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime,seclabel,pagesize=512M)
none on /mnt/huge type hugetlbfs (rw,relatime,seclabel,pagesize=512M) ---挂了多个512M
[root@localhost dpdk-19.11]# mkdir -p /mnt/huge_2M
[root@localhost dpdk-19.11]# umount /mnt/huge
[root@localhost dpdk-19.11]# mount | grep huge
cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,hugetlb)
hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime,seclabel,pagesize=512M)
[root@localhost dpdk-19.11]# mount -t hugetlbfs none /mnt/huge_2M -o pagesize=2MB
[root@localhost dpdk-19.11]# mount | grep huge
cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,hugetlb)
hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime,seclabel,pagesize=512M)
none on /mnt/huge_2M type hugetlbfs (rw,relatime,seclabel,pagesize=2M)
[root@localhost dpdk-19.11]#
[root@localhost rte]# cat hugepage_info
/dev/hugepages@@@@
/* first, check if we have a mountpoint */
if (get_hugepage_dir(hpi->hugepage_sz,
hpi->hugedir, sizeof(hpi->hugedir)) < 0) {
uint32_t num_pages;
num_pages = get_num_hugepages(dirent->d_name);
if (num_pages > 0)
RTE_LOG(NOTICE, EAL,
"%" PRIu32 " hugepages of size "
"%" PRIu64 " reserved, but no mounted "
"hugetlbfs found for that size
",
num_pages, hpi->hugepage_sz)
nt
eal_hugepage_info_init(void)
{
const char dirent_start_text[] = "hugepages-";
const size_t dirent_start_len = sizeof(dirent_start_text) - 1;
unsigned i, num_sizes = 0;
DIR *dir;
struct dirent *dirent;
dir = opendir(sys_dir_path); //sys_dir_path[] = "/sys/kernel/mm/hugepages"
if (dir == NULL) {
RTE_LOG(ERR, EAL,
"Cannot open directory %s to read system hugepage info
",
sys_dir_path);
return -1;
}
/*遍历/sys/kernel/mm/hugepages目录下以“hugepages-”开头的目录*/
for (dirent = readdir(dir); dirent != NULL; dirent = readdir(dir)) {
struct hugepage_info *hpi;
if (strncmp(dirent->d_name, dirent_start_text,
dirent_start_len) != 0)
continue;
if (num_sizes >= MAX_HUGEPAGE_SIZES)
break;
/*internal_config为DPDK全局变量*/
hpi = &internal_config.hugepage_info[num_sizes];
/*保存hugepage的大小，最多保存三种大小，一般也只用到了1G，2M*/
hpi->hugepage_sz =
rte_str_to_size(&dirent->d_name[dirent_start_len]);
/*get_hugepage_dir函数会到/proc/mounts里去寻找对应大小hugepage页挂载的目录 */
hpi->hugedir = get_hugepage_dir(hpi->hugepage_sz);
/* first, check if we have a mountpoint */
if (hpi->hugedir == NULL) {
uint32_t num_pages;
num_pages = get_num_hugepages(dirent->d_name);
if (num_pages > 0)
RTE_LOG(NOTICE, EAL,
"%" PRIu32 " hugepages of size "
"%" PRIu64 " reserved, but no mounted "
"hugetlbfs found for that size
",
num_pages, hpi->hugepage_sz);
continue;
}
......
......
}
[root@localhost server]# echo 0 > /sys/kernel/mm/hugepages/hugepages-524288kB/nr_hugepages
[root@localhost server]# echo 0 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
[root@localhost memzone]# build/app/helloworld -c 0xf -n 4
EAL: Detected 128 lcore(s)
EAL: Detected 4 NUMA nodes
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
EAL: Selected IOVA mode 'PA'
EAL: No free hugepages reported in hugepages-524288kB
EAL: No free hugepages reported in hugepages-524288kB
EAL: No free hugepages reported in hugepages-524288kB
EAL: No free hugepages reported in hugepages-524288kB
EAL: No available hugepages reported in hugepages-524288kB
EAL: No free hugepages reported in hugepages-2048kB
EAL: No free hugepages reported in hugepages-2048kB
EAL: No free hugepages reported in hugepages-2048kB
EAL: No free hugepages reported in hugepages-2048kB
EAL: No available hugepages reported in hugepages-2048kB
EAL: FATAL: Cannot get hugepage information.
EAL: Cannot get hugepage information.
PANIC in main():
Cannot init EAL
5: [build/app/helloworld() [0x487574]]
4: [/lib64/libc.so.6(__libc_start_main+0xf0) [0xffffa5281724]]
3: [build/app/helloworld(main+0x9c) [0x47ebbc]]
2: [build/app/helloworld(__rte_panic+0xac) [0x46b08c]]
1: [build/app/helloworld(rte_dump_stack+0x28) [0x54aecc]]
Aborted (core dumped)
[root@localhost server]# echo 0 > /sys/kernel/mm/hugepages/hugepages-524288kB/nr_hugepages
[root@localhost server]# echo 0 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
[root@localhost server]# cat /proc/meminfo | grep -i huge
AnonHugePages: 0 kB
ShmemHugePages: 0 kB
HugePages_Total: 0
HugePages_Free: 0
HugePages_Rsvd: 0
HugePages_Surp: 0
Hugepagesize: 524288 kB
[root@localhost server]#
[root@localhost server]# echo 256 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
[root@localhost server]# cat /proc/meminfo | grep -i huge
AnonHugePages: 0 kB
ShmemHugePages: 0 kB
HugePages_Total: 0
HugePages_Free: 0
HugePages_Rsvd: 0
HugePages_Surp: 0
Hugepagesize: 524288 kB
[root@localhost server]# echo 256 > /sys/kernel/mm/hugepages/hugepages-524288kB/nr_hugepages
[root@localhost server]# cat /proc/meminfo | grep -i huge
AnonHugePages: 0 kB
ShmemHugePages: 0 kB
HugePages_Total: 256
HugePages_Free: 256
HugePages_Rsvd: 0
HugePages_Surp: 0
Hugepagesize: 524288 kB
[root@localhost server]#
[root@localhost server]# echo 0 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
[root@localhost server]# echo 0 > /sys/kernel/mm/hugepages/hugepages-524288kB/nr_hugepages
[root@localhost server]# free -g
total used free shared buff/cache available
Mem: 510 1 505 0 3 468
Swap: 0 0 0
[root@localhost server]# cat /proc/meminfo | grep -i huge
AnonHugePages: 0 kB
ShmemHugePages: 0 kB
HugePages_Total: 0
HugePages_Free: 0
HugePages_Rsvd: 0
HugePages_Surp: 0
Hugepagesize: 524288 kB
[root@localhost server]#
[root@localhost server]# echo 64 > /sys/kernel/mm/hugepages/hugepages-524288kB/nr_hugepages ---32G
[root@localhost server]# free -g
total used free shared buff/cache available
Mem: 510 33 473 0 3 436
Swap: 0 0 0
[root@localhost server]#
[root@localhost server]# numactl --hardware
available: 4 nodes (0-3)
node 0 cpus: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
node 0 size: 130055 MB
node 0 free: 119116 MB
node 1 cpus: 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
node 1 size: 130937 MB
node 1 free: 121737 MB
node 2 cpus: 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95
node 2 size: 130937 MB
node 2 free: 121960 MB
node 3 cpus: 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
node 3 size: 130935 MB
node 3 free: 121908 MB
node distances:
node 0 1 2 3
0: 10 16 32 33
1: 16 10 25 32
2: 32 25 10 16
3: 33 32 16 10
[root@localhost server]#