dpdk内存初始化（补充）

5070阅读　0评论2017-05-20　lvyilong316
分类：LINUX

 dpdk内存初始化补充

——lvyilong316

     关于dpdk内存初始化这里做一个补充，有些细节是上一篇没有讲到的。主要在函数eal_hugepage_info_init中。上一篇我们讲过这个函数是dpdk内存初始化最早执行的函数。这个函数由两处细节需要我们注意下。


点击(此处)折叠或打开

  1. int
  2. eal_hugepage_info_init(void)
  3. {
  4.          const char dirent_start_text[] = "hugepages-";
  5.          const size_t dirent_start_len = sizeof(dirent_start_text) - 1;
  6.          unsigned i, num_sizes = 0;
  7.          DIR *dir;
  8.          struct dirent *dirent;
  9.  
  10.          dir = opendir(sys_dir_path); /* /sys/kernel/mm/hugepages */
  11.          if (dir == NULL)
  12.                   rte_panic("Cannot open directory %s to read system hugepage "
  13.                              "info\n", sys_dir_path);
  14.  
  15.          for (dirent = readdir(dir); dirent != NULL; dirent = readdir(dir)) {
  16.                   struct hugepage_info *hpi;
  17.  
  18.                   if (strncmp(dirent->d_name, dirent_start_text,
  19.                                dirent_start_len) != 0)
  20.                            continue;
  21.  
  22.                   if (num_sizes >= MAX_HUGEPAGE_SIZES)
  23.                            break;
  24.  
  25.                   hpi = &internal_config.hugepage_info[num_sizes];
  26.                   hpi->hugepage_sz =
  27.                            rte_str_to_size(&dirent->d_name[dirent_start_len]);
  28.                   /*得到的是hugepage的挂载目录*/
  29.                   hpi->hugedir = get_hugepage_dir(hpi->hugepage_sz);
  30.  
  31.                   /* first, check if we have a mountpoint */
  32.                   if (hpi->hugedir == NULL) {
  33.                            uint32_t num_pages;
  34.  
  35.                            num_pages = get_num_hugepages(dirent->d_name);
  36.                            if (num_pages > 0)
  37.                                     RTE_LOG(NOTICE, EAL,
  38.                                              "%" PRIu32 " hugepages of size "
  39.                                              "%" PRIu64 " reserved, but no mounted "
  40.                                              "hugetlbfs found for that size\n",
  41.                                              num_pages, hpi->hugepage_sz);
  42.                            continue;
  43.                   }
  44.  
  45.                   /* try to obtain a writelock */
  46.                   hpi->lock_descriptor = open(hpi->hugedir, O_RDONLY);
  47.  
  48.                   /* if blocking lock failed */
  49.                   if (flock(hpi->lock_descriptor, LOCK_EX) == -1) {
  50.                            RTE_LOG(CRIT, EAL,
  51.                                     "Failed to lock hugepage directory!\n");
  52.                            break;
  53.                   }
  54.                   /* clear out the hugepages dir from unused pages */
  55.                   if (clear_hugedir(hpi->hugedir) == -1)
  56.                            break;
  57.  
  58.                   /* for now, put all pages into socket 0,
  59.                    * later they will be sorted */
  60.                   /* 这里还没有按socket统计页数，将内存页数直接记录到hupage_info的num_pages[0]里面了 */
  61.                   hpi->num_pages[0] = get_num_hugepages(dirent->d_name);
  62.  
  63.                   num_sizes++;
  64.          }
  65.          closedir(dir);
  66.  
  67.          /* something went wrong, and we broke from the for loop above */
  68.          if (dirent != NULL)
  69.                   return -1;
  70.  
  71.          internal_config.num_hugepage_sizes = num_sizes;
  72.  
  73.          /* sort the page directory entries by size, largest to smallest */
  74.          qsort(&internal_config.hugepage_info[0], num_sizes,
  75.                sizeof(internal_config.hugepage_info[0]), compare_hpi);
  76.  
  77.          /* now we have all info, check we have at least one valid size */
  78.          for (i = 0; i < num_sizes; i++)
  79.                   if (internal_config.hugepage_info[i].hugedir != NULL &&
  80.                       internal_config.hugepage_info[i].num_pages[0] > 0)
  81.                            return 0;
  82.  
  83.          /* no valid hugepage mounts available, return error */
  84.          return -1;
  85. }


首先是hpi->hugedir = get_hugepage_dir(hpi->hugepage_sz)，这里得到的是什么？如果不认真看代码很容易以为里得到的是/sys/kernel/mm/hugepages下面对应的目录。但如果查看get_hugepage_dir的代码就好发现这里是通过读取/proc/mounts文件，获取hugetlbfs的挂载目录。当然如果通过参数指定了 --huge-dir选项，那么就返回该选项指定的目录。

那么得到这个目录有什么用呢？我们看下面的clear_hugedir函数，这个函数内部实现就是将get_hugepage_dir获取到的目录下面的“*map_*”文件都删除（前提是没有其他进程引用）。为什么要删除这些文件呢？我们知道一个dpdk进程会创建多个map文件用来分配内存。而进程结束时只是unmmap，并没有删除这些文件，那对于hugetlbfs文件系统来说其实这些内存并没有释放（虽然进程也没有使用）。所以在启动dpdk进程时先对这类文件进行一次清理。我们看到进程调用get_num_hugepages获取当前系统可用页面数的逻辑是在clear_hugedir之后的，所以之前进程残留的map文件并不会影响下次dpdk进程启动的内存扫描。
上一篇：linux tun/tap设备的实现（kennel 3.10）
下一篇：dpdk内存管理——rte_malloc实现