浅析linux组调度配额、带宽和分享多余带宽发红包

6370阅读　0评论2014-11-28　gliethttp
分类：LINUX


  1. #ifdef CONFIG_CFS_BANDWIDTH
  2. /*
  3.  * Amount of runtime to allocate from global (tg) to local (per-cfs_rq) pool
  4.  * each time a cfs_rq requests quota.
  5.  *
  6.  * Note: in the case that the slice exceeds the runtime remaining (either due
  7.  * to consumption or the quota being specified to be smaller than the slice)
  8.  * we will always only issue the remaining available time.
  9.  *
  10.  * default: 5 msec, units: microseconds
  11.   */
  12. unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL;
  13. #endif

  15. /*
  16.  * Replenish runtime according to assigned quota and update expiration time.
  17.  * We use sched_clock_cpu directly instead of rq->clock to avoid adding
  18.  * additional synchronization around rq->lock.
  19.  *
  20.  * requires cfs_b->lock
  21.  */
  22. void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b)
  23. {
  24.     u64 now;

  26.     if (cfs_b->quota == RUNTIME_INF)
  27.         return;

  29.     now = sched_clock_cpu(smp_processor_id());
  30.     cfs_b->runtime = cfs_b->quota;
  31.     cfs_b->runtime_expires = now + ktime_to_ns(cfs_b->period);
  32.     // 组调度, 配额, quota
  33.     // 也就是说cfs_b->period这段时间分配给该cfs_b组的runtime配额为cfs_b->quota
  34.     // 不论该组消耗了多少配额, 都会在这里将配额重新填满, 触发接下来的消耗period[2014-11-26]
  35.     // start_bandwidth_timer(&cfs_b->period_timer, cfs_b->period);
  36. }


  39. static inline u64 sched_cfs_bandwidth_slice(void)
  40. {
  41.     return (u64)sysctl_sched_cfs_bandwidth_slice * NSEC_PER_USEC;
  42. }

  44. /* returns 0 on failure to allocate runtime */
  45. // 努力延长本cfs_rq的runtime_remaining时间寿命[gliethttp]
  46. static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq)
  47. {
  48.     struct task_group *tg = cfs_rq->tg;
  49.     struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg);
  50.     u64 amount = 0, min_amount, expires;

  52.     /* note: this is a positive sum as runtime_remaining <= 0 */
  53.     // 5ms + 欠的账(超出配额的部分), 如注释所说cfs_rq->runtime_remaining是一个负数, 欠费了
  54.     min_amount = sched_cfs_bandwidth_slice() - cfs_rq->runtime_remaining;

  56.     raw_spin_lock(&cfs_b->lock);
  57.     if (cfs_b->quota == RUNTIME_INF)
  58.         amount = min_amount; // 如果是infinite的,那么每次都获取5ms
  59.     else {
  60.         /*
  61.          * If the bandwidth pool has become inactive, then at least one
  62.          * period must have elapsed since the last consumption.
  63.          * Refresh the global state and ensure bandwidth timer becomes
  64.          * active.
  65.          */
  66.         if (!cfs_b->timer_active) {
  67.             // 如果带宽timer刚好stop了, 那么重新填充带宽timer的runtime和runtime_expires
  68.             // cfs_b->runtime = cfs_b->quota;
  69.             // cfs_b->runtime_expires = now + ktime_to_ns(cfs_b->period);
  70.             // 进入下一轮以cfs_b->period为间隔的, 有效时长(单位ns纳秒)cfs_b->runtime的带宽period[gliethttp]
  71.             __refill_cfs_bandwidth_runtime(cfs_b);
  72.             __start_cfs_bandwidth(cfs_b);
  73.         }

  75.         if (cfs_b->runtime > 0) {
  76.             amount = min(cfs_b->runtime, min_amount); // 如果剩余的cfs_b->runtime不足5ms(sysctl_sched_cfs_bandwidth_slice)
  77.             // 更甚至与不足-cfs_rq->runtime_remaining, 那么amount将等于2者最小者.
  78.             cfs_b->runtime -= amount; // 被瓜分出去amount
  79.             cfs_b->idle = 0;
  80.         }
  81.     }
  82.     expires = cfs_b->runtime_expires;
  83.     raw_spin_unlock(&cfs_b->lock);

  85.     cfs_rq->runtime_remaining += amount; // 重新为自己充电amount纳秒
  86.     /*
  87.      * we may have advanced our local expiration to account for allowed
  88.      * spread between our sched_clock and the one on which runtime was
  89.      * issued.
  90.      */
  91.     if ((s64)(expires - cfs_rq->runtime_expires) > 0)
  92.         cfs_rq->runtime_expires = expires;

  94.     return cfs_rq->runtime_remaining > 0;
  95. }

  97. // 发红包啦[20141128]! 每个处于trhottled的cfs_rq获得
  98. // 我有些多余的remaining纳秒带宽, 将在expires时到期
  99. // cfs_b你把这些带宽发给你手里饥渴的人吧[20141128]
  100. static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b,
  101.         u64 remaining, u64 expires)
  102. {
  103.     struct cfs_rq *cfs_rq;
  104.     u64 runtime = remaining;

  106.     rcu_read_lock();
  107.     list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq,
  108.                 throttled_list) {
  109.         struct rq *rq = rq_of(cfs_rq);

  111.         raw_spin_lock(&rq->lock);
  112.         if (!cfs_rq_throttled(cfs_rq))
  113.             goto next;

  115.         runtime = -cfs_rq->runtime_remaining + 1; // 发红包啦[20141128]! 每个处于trhottled获得
  116.         // -cfs_rq->runtime_remaining, 填平到数值0, 同时再额外附送1ns
  117.         // 其实对于当前cfs_rq只获得了可用的1ns,
  118.         // 但是发红包的人,可把这个cfs_rq签得账-cfs_rq->runtime_remaining都给偿还上了
  119.         // 调用这个函数发红包的人, 真是个runtime富裕的大好人[20141128]
  120.         // 不要想太美啦, 世上哪有好事,便宜让你占呀,
  121.         // 其实是do_sched_cfs_period_timer自己欠下的账, 自己在这个时机来还而已[gliethttp]
  122.         if (runtime > remaining)
  123.             runtime = remaining;
  124.         remaining -= runtime; // 红包发出去了runtime, 计算我还剩多少ns红包

  126.         cfs_rq->runtime_remaining += runtime; // 红包给你这个cfs_rq了[20141128]
  127.         cfs_rq->runtime_expires = expires; // 你cfs_rq使用我的expires吧, 我和我富裕出来的红包生命将在expires时刻因过期而终结

  129.         /* we check whether we're throttled above */
  130.         if (cfs_rq->runtime_remaining > 0) // 把自己throttled吧
  131.             unthrottle_cfs_rq(cfs_rq);

  133. next:
  134.         raw_spin_unlock(&rq->lock);

  136.         if (!remaining)
  137.             break;
  138.     }
  139.     rcu_read_unlock();

  141.     return remaining;
  142. }

  144. /*
  145.  * Responsible for refilling a task_group's bandwidth and unthrottling its
  146.  * cfs_rqs as appropriate. If there has been no activity within the last
  147.  * period the timer is deactivated until scheduling resumes; cfs_b->idle is
  148.  * used to track this state.
  149.  */
  150. // 新的cfs_b->period_timer带宽周期到达,
  151. // 带宽数值(单位ns纳秒): cfs_b->runtime (cfs_b->runtime = cfs_b->quota;)
  152. // 带宽有效期(单位ns纳秒): cfs_b->runtime_expires (cfs_b->runtime_expires = now + ktime_to_ns(cfs_b->period);)
  153. static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun)
  154. {
  155.     ......
  156.     /*
  157.      * This check is repeated as we are holding onto the new bandwidth
  158.      * while we unthrottle. This can potentially race with an unthrottled
  159.      * group trying to acquire new bandwidth from the global pool.
  160.      */
  161.     while (throttled && runtime > 0) {
  162.         raw_spin_unlock(&cfs_b->lock);
  163.         /* we can't nest cfs_b->lock while distributing bandwidth */
  164.         runtime = distribute_cfs_runtime(cfs_b, runtime,
  165.                          runtime_expires); // 把自己欠的账还回去
  166.         raw_spin_lock(&cfs_b->lock);

  168.         throttled = !list_empty(&cfs_b->throttled_cfs_rq);
  169.     }

  171.     /* return (any) remaining runtime */
  172.     cfs_b->runtime = runtime;
  173.     ......
  174. }


上一篇：linux内核组调度树的遍历算法 - walk_tg_tree_from
下一篇：没有了