diff --git a/study/kernel/01-process/05-schedule/04-cfs/01-cfs/README.md b/study/kernel/01-process/05-schedule/04-cfs/01-cfs/README.md index 5173bf3..41de58c 100644 --- a/study/kernel/01-process/05-schedule/04-cfs/01-cfs/README.md +++ b/study/kernel/01-process/05-schedule/04-cfs/01-cfs/README.md @@ -1,4 +1,4 @@ -Linux进程调度器的设计 +Linux进程调度CFS调度器 ======= diff --git a/study/kernel/01-process/05-schedule/04-cfs/01-cfs/fair.c b/study/kernel/01-process/05-schedule/04-cfs/01-cfs/fair.c index 0c2738b..035e415 100644 --- a/study/kernel/01-process/05-schedule/04-cfs/01-cfs/fair.c +++ b/study/kernel/01-process/05-schedule/04-cfs/01-cfs/fair.c @@ -62,7 +62,7 @@ unsigned int normalized_sysctl_sched_latency = 6000000ULL; * SCHED_TUNABLESCALING_LINEAR - scaled linear, *ncpus */ enum sched_tunable_scaling sysctl_sched_tunable_scaling - = SCHED_TUNABLESCALING_LOG; + = SCHED_TUNABLESCALING_LOG; /* * Minimal preemption granularity for CPU-bound tasks: @@ -118,20 +118,20 @@ unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; static inline void update_load_add(struct load_weight *lw, unsigned long inc) { - lw->weight += inc; - lw->inv_weight = 0; + lw->weight += inc; + lw->inv_weight = 0; } static inline void update_load_sub(struct load_weight *lw, unsigned long dec) { - lw->weight -= dec; - lw->inv_weight = 0; + lw->weight -= dec; + lw->inv_weight = 0; } static inline void update_load_set(struct load_weight *lw, unsigned long w) { - lw->weight = w; - lw->inv_weight = 0; + lw->weight = w; + lw->inv_weight = 0; } /* @@ -145,40 +145,40 @@ static inline void update_load_set(struct load_weight *lw, unsigned long w) */ static unsigned int get_update_sysctl_factor(void) { - unsigned int cpus = min_t(unsigned int, num_online_cpus(), 8); - unsigned int factor; + unsigned int cpus = min_t(unsigned int, num_online_cpus(), 8); + unsigned int factor; - switch (sysctl_sched_tunable_scaling) { - case SCHED_TUNABLESCALING_NONE: - factor = 1; - break; - case SCHED_TUNABLESCALING_LINEAR: - factor = cpus; - break; - case SCHED_TUNABLESCALING_LOG: - default: - factor = 1 + ilog2(cpus); - break; - } + switch (sysctl_sched_tunable_scaling) { + case SCHED_TUNABLESCALING_NONE: + factor = 1; + break; + case SCHED_TUNABLESCALING_LINEAR: + factor = cpus; + break; + case SCHED_TUNABLESCALING_LOG: + default: + factor = 1 + ilog2(cpus); + break; + } - return factor; + return factor; } static void update_sysctl(void) { - unsigned int factor = get_update_sysctl_factor(); + unsigned int factor = get_update_sysctl_factor(); #define SET_SYSCTL(name) \ - (sysctl_##name = (factor) * normalized_sysctl_##name) - SET_SYSCTL(sched_min_granularity); - SET_SYSCTL(sched_latency); - SET_SYSCTL(sched_wakeup_granularity); + (sysctl_##name = (factor) * normalized_sysctl_##name) + SET_SYSCTL(sched_min_granularity); + SET_SYSCTL(sched_latency); + SET_SYSCTL(sched_wakeup_granularity); #undef SET_SYSCTL } void sched_init_granularity(void) { - update_sysctl(); + update_sysctl(); } #define WMULT_CONST (~0U) @@ -186,19 +186,19 @@ void sched_init_granularity(void) static void __update_inv_weight(struct load_weight *lw) { - unsigned long w; + unsigned long w; - if (likely(lw->inv_weight)) - return; + if (likely(lw->inv_weight)) + return; - w = scale_load_down(lw->weight); + w = scale_load_down(lw->weight); - if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST)) - lw->inv_weight = 1; - else if (unlikely(!w)) - lw->inv_weight = WMULT_CONST; - else - lw->inv_weight = WMULT_CONST / w; + if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST)) + lw->inv_weight = 1; + else if (unlikely(!w)) + lw->inv_weight = WMULT_CONST; + else + lw->inv_weight = WMULT_CONST / w; } /* @@ -215,27 +215,27 @@ static void __update_inv_weight(struct load_weight *lw) */ static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw) { - u64 fact = scale_load_down(weight); - int shift = WMULT_SHIFT; + u64 fact = scale_load_down(weight); + int shift = WMULT_SHIFT; - __update_inv_weight(lw); - - if (unlikely(fact >> 32)) { - while (fact >> 32) { - fact >>= 1; - shift--; - } - } - - /* hint to use a 32x32->64 mul */ - fact = (u64)(u32)fact * lw->inv_weight; + __update_inv_weight(lw); + if (unlikely(fact >> 32)) { while (fact >> 32) { - fact >>= 1; - shift--; + fact >>= 1; + shift--; } + } - return mul_u64_u32_shr(delta_exec, fact, shift); + /* hint to use a 32x32->64 mul */ + fact = (u64)(u32)fact * lw->inv_weight; + + while (fact >> 32) { + fact >>= 1; + shift--; + } + + return mul_u64_u32_shr(delta_exec, fact, shift); } @@ -250,7 +250,7 @@ const struct sched_class fair_sched_class; /* cpu runqueue to which this cfs_rq is attached */ static inline struct rq *rq_of(struct cfs_rq *cfs_rq) { - return cfs_rq->rq; + return cfs_rq->rq; } /* An entity is a task if it doesn't "own" a runqueue */ @@ -259,147 +259,147 @@ static inline struct rq *rq_of(struct cfs_rq *cfs_rq) static inline struct task_struct *task_of(struct sched_entity *se) { #ifdef CONFIG_SCHED_DEBUG - WARN_ON_ONCE(!entity_is_task(se)); + WARN_ON_ONCE(!entity_is_task(se)); #endif - return container_of(se, struct task_struct, se); + return container_of(se, struct task_struct, se); } /* Walk up scheduling entities hierarchy */ #define for_each_sched_entity(se) \ - for (; se; se = se->parent) + for (; se; se = se->parent) static inline struct cfs_rq *task_cfs_rq(struct task_struct *p) { - return p->se.cfs_rq; + return p->se.cfs_rq; } /* runqueue on which this entity is (to be) queued */ static inline struct cfs_rq *cfs_rq_of(struct sched_entity *se) { - return se->cfs_rq; + return se->cfs_rq; } /* runqueue "owned" by this group */ static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp) { - return grp->my_q; + return grp->my_q; } static inline void list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) { - if (!cfs_rq->on_list) { - /* - * Ensure we either appear before our parent (if already - * enqueued) or force our parent to appear after us when it is - * enqueued. The fact that we always enqueue bottom-up - * reduces this to two cases. - */ - if (cfs_rq->tg->parent && - cfs_rq->tg->parent->cfs_rq[cpu_of(rq_of(cfs_rq))]->on_list) { - list_add_rcu(&cfs_rq->leaf_cfs_rq_list, - &rq_of(cfs_rq)->leaf_cfs_rq_list); - } else { - list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, - &rq_of(cfs_rq)->leaf_cfs_rq_list); - } - - cfs_rq->on_list = 1; + if (!cfs_rq->on_list) { + /* + * Ensure we either appear before our parent (if already + * enqueued) or force our parent to appear after us when it is + * enqueued. The fact that we always enqueue bottom-up + * reduces this to two cases. + */ + if (cfs_rq->tg->parent && + cfs_rq->tg->parent->cfs_rq[cpu_of(rq_of(cfs_rq))]->on_list) { + list_add_rcu(&cfs_rq->leaf_cfs_rq_list, + &rq_of(cfs_rq)->leaf_cfs_rq_list); + } else { + list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, + &rq_of(cfs_rq)->leaf_cfs_rq_list); } + + cfs_rq->on_list = 1; + } } static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) { - if (cfs_rq->on_list) { - list_del_rcu(&cfs_rq->leaf_cfs_rq_list); - cfs_rq->on_list = 0; - } + if (cfs_rq->on_list) { + list_del_rcu(&cfs_rq->leaf_cfs_rq_list); + cfs_rq->on_list = 0; + } } /* Iterate thr' all leaf cfs_rq's on a runqueue */ #define for_each_leaf_cfs_rq(rq, cfs_rq) \ - list_for_each_entry_rcu(cfs_rq, &rq->leaf_cfs_rq_list, leaf_cfs_rq_list) + list_for_each_entry_rcu(cfs_rq, &rq->leaf_cfs_rq_list, leaf_cfs_rq_list) /* Do the two (enqueued) entities belong to the same group ? */ static inline struct cfs_rq * is_same_group(struct sched_entity *se, struct sched_entity *pse) { - if (se->cfs_rq == pse->cfs_rq) - return se->cfs_rq; + if (se->cfs_rq == pse->cfs_rq) + return se->cfs_rq; - return NULL; + return NULL; } static inline struct sched_entity *parent_entity(struct sched_entity *se) { - return se->parent; + return se->parent; } static void find_matching_se(struct sched_entity **se, struct sched_entity **pse) { - int se_depth, pse_depth; + int se_depth, pse_depth; - /* - * preemption test can be made between sibling entities who are in the - * same cfs_rq i.e who have a common parent. Walk up the hierarchy of - * both tasks until we find their ancestors who are siblings of common - * parent. - */ + /* + * preemption test can be made between sibling entities who are in the + * same cfs_rq i.e who have a common parent. Walk up the hierarchy of + * both tasks until we find their ancestors who are siblings of common + * parent. + */ - /* First walk up until both entities are at same depth */ - se_depth = (*se)->depth; - pse_depth = (*pse)->depth; + /* First walk up until both entities are at same depth */ + se_depth = (*se)->depth; + pse_depth = (*pse)->depth; - while (se_depth > pse_depth) { - se_depth--; - *se = parent_entity(*se); - } + while (se_depth > pse_depth) { + se_depth--; + *se = parent_entity(*se); + } - while (pse_depth > se_depth) { - pse_depth--; - *pse = parent_entity(*pse); - } + while (pse_depth > se_depth) { + pse_depth--; + *pse = parent_entity(*pse); + } - while (!is_same_group(*se, *pse)) { - *se = parent_entity(*se); - *pse = parent_entity(*pse); - } + while (!is_same_group(*se, *pse)) { + *se = parent_entity(*se); + *pse = parent_entity(*pse); + } } #else /* !CONFIG_FAIR_GROUP_SCHED */ static inline struct task_struct *task_of(struct sched_entity *se) { - return container_of(se, struct task_struct, se); + return container_of(se, struct task_struct, se); } static inline struct rq *rq_of(struct cfs_rq *cfs_rq) { - return container_of(cfs_rq, struct rq, cfs); + return container_of(cfs_rq, struct rq, cfs); } #define entity_is_task(se) 1 #define for_each_sched_entity(se) \ - for (; se; se = NULL) + for (; se; se = NULL) static inline struct cfs_rq *task_cfs_rq(struct task_struct *p) { - return &task_rq(p)->cfs; + return &task_rq(p)->cfs; } static inline struct cfs_rq *cfs_rq_of(struct sched_entity *se) { - struct task_struct *p = task_of(se); - struct rq *rq = task_rq(p); + struct task_struct *p = task_of(se); + struct rq *rq = task_rq(p); - return &rq->cfs; + return &rq->cfs; } /* runqueue "owned" by this group */ static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp) { - return NULL; + return NULL; } static inline void list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) @@ -411,11 +411,11 @@ static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) } #define for_each_leaf_cfs_rq(rq, cfs_rq) \ - for (cfs_rq = &rq->cfs; cfs_rq; cfs_rq = NULL) + for (cfs_rq = &rq->cfs; cfs_rq; cfs_rq = NULL) static inline struct sched_entity *parent_entity(struct sched_entity *se) { - return NULL; + return NULL; } static inline void @@ -434,51 +434,51 @@ void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec); static inline u64 max_vruntime(u64 max_vruntime, u64 vruntime) { - s64 delta = (s64)(vruntime - max_vruntime); - if (delta > 0) - max_vruntime = vruntime; + s64 delta = (s64)(vruntime - max_vruntime); + if (delta > 0) + max_vruntime = vruntime; - return max_vruntime; + return max_vruntime; } static inline u64 min_vruntime(u64 min_vruntime, u64 vruntime) { - s64 delta = (s64)(vruntime - min_vruntime); - if (delta < 0) - min_vruntime = vruntime; + s64 delta = (s64)(vruntime - min_vruntime); + if (delta < 0) + min_vruntime = vruntime; - return min_vruntime; + return min_vruntime; } static inline int entity_before(struct sched_entity *a, - struct sched_entity *b) + struct sched_entity *b) { - return (s64)(a->vruntime - b->vruntime) < 0; + return (s64)(a->vruntime - b->vruntime) < 0; } static void update_min_vruntime(struct cfs_rq *cfs_rq) { - u64 vruntime = cfs_rq->min_vruntime; + u64 vruntime = cfs_rq->min_vruntime; - if (cfs_rq->curr) - vruntime = cfs_rq->curr->vruntime; + if (cfs_rq->curr) + vruntime = cfs_rq->curr->vruntime; - if (cfs_rq->rb_leftmost) { - struct sched_entity *se = rb_entry(cfs_rq->rb_leftmost, - struct sched_entity, - run_node); + if (cfs_rq->rb_leftmost) { + struct sched_entity *se = rb_entry(cfs_rq->rb_leftmost, + struct sched_entity, + run_node); - if (!cfs_rq->curr) - vruntime = se->vruntime; - else - vruntime = min_vruntime(vruntime, se->vruntime); - } + if (!cfs_rq->curr) + vruntime = se->vruntime; + else + vruntime = min_vruntime(vruntime, se->vruntime); + } - /* ensure we never gain time by being placed backwards. */ - cfs_rq->min_vruntime = max_vruntime(cfs_rq->min_vruntime, vruntime); + /* ensure we never gain time by being placed backwards. */ + cfs_rq->min_vruntime = max_vruntime(cfs_rq->min_vruntime, vruntime); #ifndef CONFIG_64BIT - smp_wmb(); - cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime; + smp_wmb(); + cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime; #endif } @@ -487,81 +487,81 @@ static void update_min_vruntime(struct cfs_rq *cfs_rq) */ static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { - struct rb_node **link = &cfs_rq->tasks_timeline.rb_node; - struct rb_node *parent = NULL; - struct sched_entity *entry; - int leftmost = 1; + struct rb_node **link = &cfs_rq->tasks_timeline.rb_node; + struct rb_node *parent = NULL; + struct sched_entity *entry; + int leftmost = 1; + /* + * Find the right place in the rbtree: + */ + while (*link) { + parent = *link; + entry = rb_entry(parent, struct sched_entity, run_node); /* - * Find the right place in the rbtree: + * We dont care about collisions. Nodes with + * the same key stay together. */ - while (*link) { - parent = *link; - entry = rb_entry(parent, struct sched_entity, run_node); - /* - * We dont care about collisions. Nodes with - * the same key stay together. - */ - if (entity_before(se, entry)) { - link = &parent->rb_left; - } else { - link = &parent->rb_right; - leftmost = 0; - } + if (entity_before(se, entry)) { + link = &parent->rb_left; + } else { + link = &parent->rb_right; + leftmost = 0; } + } - /* - * Maintain a cache of leftmost tree entries (it is frequently - * used): - */ - if (leftmost) - cfs_rq->rb_leftmost = &se->run_node; + /* + * Maintain a cache of leftmost tree entries (it is frequently + * used): + */ + if (leftmost) + cfs_rq->rb_leftmost = &se->run_node; - rb_link_node(&se->run_node, parent, link); - rb_insert_color(&se->run_node, &cfs_rq->tasks_timeline); + rb_link_node(&se->run_node, parent, link); + rb_insert_color(&se->run_node, &cfs_rq->tasks_timeline); } static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { - if (cfs_rq->rb_leftmost == &se->run_node) { - struct rb_node *next_node; + if (cfs_rq->rb_leftmost == &se->run_node) { + struct rb_node *next_node; - next_node = rb_next(&se->run_node); - cfs_rq->rb_leftmost = next_node; - } + next_node = rb_next(&se->run_node); + cfs_rq->rb_leftmost = next_node; + } - rb_erase(&se->run_node, &cfs_rq->tasks_timeline); + rb_erase(&se->run_node, &cfs_rq->tasks_timeline); } struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq) { - struct rb_node *left = cfs_rq->rb_leftmost; + struct rb_node *left = cfs_rq->rb_leftmost; - if (!left) - return NULL; + if (!left) + return NULL; - return rb_entry(left, struct sched_entity, run_node); + return rb_entry(left, struct sched_entity, run_node); } static struct sched_entity *__pick_next_entity(struct sched_entity *se) { - struct rb_node *next = rb_next(&se->run_node); + struct rb_node *next = rb_next(&se->run_node); - if (!next) - return NULL; + if (!next) + return NULL; - return rb_entry(next, struct sched_entity, run_node); + return rb_entry(next, struct sched_entity, run_node); } #ifdef CONFIG_SCHED_DEBUG struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq) { - struct rb_node *last = rb_last(&cfs_rq->tasks_timeline); + struct rb_node *last = rb_last(&cfs_rq->tasks_timeline); - if (!last) - return NULL; + if (!last) + return NULL; - return rb_entry(last, struct sched_entity, run_node); + return rb_entry(last, struct sched_entity, run_node); } /************************************************************** @@ -569,26 +569,26 @@ struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq) */ int sched_proc_update_handler(struct ctl_table *table, int write, - void __user *buffer, size_t *lenp, - loff_t *ppos) + void __user *buffer, size_t *lenp, + loff_t *ppos) { - int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); - unsigned int factor = get_update_sysctl_factor(); + int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + unsigned int factor = get_update_sysctl_factor(); - if (ret || !write) - return ret; + if (ret || !write) + return ret; - sched_nr_latency = DIV_ROUND_UP(sysctl_sched_latency, - sysctl_sched_min_granularity); + sched_nr_latency = DIV_ROUND_UP(sysctl_sched_latency, + sysctl_sched_min_granularity); #define WRT_SYSCTL(name) \ - (normalized_sysctl_##name = sysctl_##name / (factor)) - WRT_SYSCTL(sched_min_granularity); - WRT_SYSCTL(sched_latency); - WRT_SYSCTL(sched_wakeup_granularity); + (normalized_sysctl_##name = sysctl_##name / (factor)) + WRT_SYSCTL(sched_min_granularity); + WRT_SYSCTL(sched_latency); + WRT_SYSCTL(sched_wakeup_granularity); #undef WRT_SYSCTL - return 0; + return 0; } #endif @@ -597,10 +597,10 @@ int sched_proc_update_handler(struct ctl_table *table, int write, */ static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se) { - if (unlikely(se->load.weight != NICE_0_LOAD)) - delta = __calc_delta(delta, NICE_0_LOAD, &se->load); + if (unlikely(se->load.weight != NICE_0_LOAD)) + delta = __calc_delta(delta, NICE_0_LOAD, &se->load); - return delta; + return delta; } /* @@ -613,10 +613,10 @@ static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se) */ static u64 __sched_period(unsigned long nr_running) { - if (unlikely(nr_running > sched_nr_latency)) - return nr_running * sysctl_sched_min_granularity; - else - return sysctl_sched_latency; + if (unlikely(nr_running > sched_nr_latency)) + return nr_running * sysctl_sched_min_granularity; + else + return sysctl_sched_latency; } /* @@ -627,24 +627,24 @@ static u64 __sched_period(unsigned long nr_running) */ static u64 sched_slice(struct cfs_rq *cfs_rq, struct sched_entity *se) { - u64 slice = __sched_period(cfs_rq->nr_running + !se->on_rq); + u64 slice = __sched_period(cfs_rq->nr_running + !se->on_rq); - for_each_sched_entity(se) { - struct load_weight *load; - struct load_weight lw; + for_each_sched_entity(se) { + struct load_weight *load; + struct load_weight lw; - cfs_rq = cfs_rq_of(se); - load = &cfs_rq->load; + cfs_rq = cfs_rq_of(se); + load = &cfs_rq->load; - if (unlikely(!se->on_rq)) { - lw = cfs_rq->load; + if (unlikely(!se->on_rq)) { + lw = cfs_rq->load; - update_load_add(&lw, se->load.weight); - load = &lw; - } - slice = __calc_delta(slice, se->load.weight, load); + update_load_add(&lw, se->load.weight); + load = &lw; } - return slice; + slice = __calc_delta(slice, se->load.weight, load); + } + return slice; } /* @@ -654,7 +654,7 @@ static u64 sched_slice(struct cfs_rq *cfs_rq, struct sched_entity *se) */ static u64 sched_vslice(struct cfs_rq *cfs_rq, struct sched_entity *se) { - return calc_delta_fair(sched_slice(cfs_rq, se), se); + return calc_delta_fair(sched_slice(cfs_rq, se), se); } #ifdef CONFIG_SMP @@ -673,20 +673,20 @@ static unsigned long task_h_load(struct task_struct *p); /* Give new sched_entity start runnable values to heavy its load in infant time */ void init_entity_runnable_average(struct sched_entity *se) { - struct sched_avg *sa = &se->avg; + struct sched_avg *sa = &se->avg; - sa->last_update_time = 0; - /* - * sched_avg's period_contrib should be strictly less then 1024, so - * we give it 1023 to make sure it is almost a period (1024us), and - * will definitely be update (after enqueue). - */ - sa->period_contrib = 1023; - sa->load_avg = scale_load_down(se->load.weight); - sa->load_sum = sa->load_avg * LOAD_AVG_MAX; - sa->util_avg = scale_load_down(SCHED_LOAD_SCALE); - sa->util_sum = sa->util_avg * LOAD_AVG_MAX; - /* when this task enqueue'ed, it will contribute to its cfs_rq's load_avg */ + sa->last_update_time = 0; + /* + * sched_avg's period_contrib should be strictly less then 1024, so + * we give it 1023 to make sure it is almost a period (1024us), and + * will definitely be update (after enqueue). + */ + sa->period_contrib = 1023; + sa->load_avg = scale_load_down(se->load.weight); + sa->load_sum = sa->load_avg * LOAD_AVG_MAX; + sa->util_avg = scale_load_down(SCHED_LOAD_SCALE); + sa->util_sum = sa->util_avg * LOAD_AVG_MAX; + /* when this task enqueue'ed, it will contribute to its cfs_rq's load_avg */ } static inline unsigned long cfs_rq_runnable_load_avg(struct cfs_rq *cfs_rq); @@ -702,83 +702,83 @@ void init_entity_runnable_average(struct sched_entity *se) */ static void update_curr(struct cfs_rq *cfs_rq) { - struct sched_entity *curr = cfs_rq->curr; - u64 now = rq_clock_task(rq_of(cfs_rq)); - u64 delta_exec; + struct sched_entity *curr = cfs_rq->curr; + u64 now = rq_clock_task(rq_of(cfs_rq)); + u64 delta_exec; - if (unlikely(!curr)) - return; + if (unlikely(!curr)) + return; - delta_exec = now - curr->exec_start; - if (unlikely((s64)delta_exec <= 0)) - return; + delta_exec = now - curr->exec_start; + if (unlikely((s64)delta_exec <= 0)) + return; - curr->exec_start = now; + curr->exec_start = now; - schedstat_set(curr->statistics.exec_max, - max(delta_exec, curr->statistics.exec_max)); + schedstat_set(curr->statistics.exec_max, + max(delta_exec, curr->statistics.exec_max)); - curr->sum_exec_runtime += delta_exec; - schedstat_add(cfs_rq, exec_clock, delta_exec); + curr->sum_exec_runtime += delta_exec; + schedstat_add(cfs_rq, exec_clock, delta_exec); - curr->vruntime += calc_delta_fair(delta_exec, curr); - update_min_vruntime(cfs_rq); + curr->vruntime += calc_delta_fair(delta_exec, curr); + update_min_vruntime(cfs_rq); - if (entity_is_task(curr)) { - struct task_struct *curtask = task_of(curr); + if (entity_is_task(curr)) { + struct task_struct *curtask = task_of(curr); - trace_sched_stat_runtime(curtask, delta_exec, curr->vruntime); - cpuacct_charge(curtask, delta_exec); - account_group_exec_runtime(curtask, delta_exec); - } + trace_sched_stat_runtime(curtask, delta_exec, curr->vruntime); + cpuacct_charge(curtask, delta_exec); + account_group_exec_runtime(curtask, delta_exec); + } - account_cfs_rq_runtime(cfs_rq, delta_exec); + account_cfs_rq_runtime(cfs_rq, delta_exec); } static void update_curr_fair(struct rq *rq) { - update_curr(cfs_rq_of(&rq->curr->se)); + update_curr(cfs_rq_of(&rq->curr->se)); } #ifdef CONFIG_SCHEDSTATS static inline void update_stats_wait_start(struct cfs_rq *cfs_rq, struct sched_entity *se) { - u64 wait_start = rq_clock(rq_of(cfs_rq)); + u64 wait_start = rq_clock(rq_of(cfs_rq)); - if (entity_is_task(se) && task_on_rq_migrating(task_of(se)) && - likely(wait_start > se->statistics.wait_start)) - wait_start -= se->statistics.wait_start; + if (entity_is_task(se) && task_on_rq_migrating(task_of(se)) && + likely(wait_start > se->statistics.wait_start)) + wait_start -= se->statistics.wait_start; - se->statistics.wait_start = wait_start; + se->statistics.wait_start = wait_start; } static void update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se) { - struct task_struct *p; - u64 delta; + struct task_struct *p; + u64 delta; - delta = rq_clock(rq_of(cfs_rq)) - se->statistics.wait_start; + delta = rq_clock(rq_of(cfs_rq)) - se->statistics.wait_start; - if (entity_is_task(se)) { - p = task_of(se); - if (task_on_rq_migrating(p)) { - /* - * Preserve migrating task's wait time so wait_start - * time stamp can be adjusted to accumulate wait time - * prior to migration. - */ - se->statistics.wait_start = delta; - return; - } - trace_sched_stat_wait(p, delta); + if (entity_is_task(se)) { + p = task_of(se); + if (task_on_rq_migrating(p)) { + /* + * Preserve migrating task's wait time so wait_start + * time stamp can be adjusted to accumulate wait time + * prior to migration. + */ + se->statistics.wait_start = delta; + return; } + trace_sched_stat_wait(p, delta); + } - se->statistics.wait_max = max(se->statistics.wait_max, delta); - se->statistics.wait_count++; - se->statistics.wait_sum += delta; - se->statistics.wait_start = 0; + se->statistics.wait_max = max(se->statistics.wait_max, delta); + se->statistics.wait_count++; + se->statistics.wait_sum += delta; + se->statistics.wait_start = 0; } /* @@ -787,34 +787,34 @@ update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se) static inline void update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) { - /* - * Are we enqueueing a waiting task? (for current tasks - * a dequeue/enqueue event is a NOP) - */ - if (se != cfs_rq->curr) - update_stats_wait_start(cfs_rq, se); + /* + * Are we enqueueing a waiting task? (for current tasks + * a dequeue/enqueue event is a NOP) + */ + if (se != cfs_rq->curr) + update_stats_wait_start(cfs_rq, se); } static inline void update_stats_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) { - /* - * Mark the end of the wait period if dequeueing a - * waiting task: - */ - if (se != cfs_rq->curr) - update_stats_wait_end(cfs_rq, se); + /* + * Mark the end of the wait period if dequeueing a + * waiting task: + */ + if (se != cfs_rq->curr) + update_stats_wait_end(cfs_rq, se); - if (flags & DEQUEUE_SLEEP) { - if (entity_is_task(se)) { - struct task_struct *tsk = task_of(se); + if (flags & DEQUEUE_SLEEP) { + if (entity_is_task(se)) { + struct task_struct *tsk = task_of(se); - if (tsk->state & TASK_INTERRUPTIBLE) - se->statistics.sleep_start = rq_clock(rq_of(cfs_rq)); - if (tsk->state & TASK_UNINTERRUPTIBLE) - se->statistics.block_start = rq_clock(rq_of(cfs_rq)); - } + if (tsk->state & TASK_INTERRUPTIBLE) + se->statistics.sleep_start = rq_clock(rq_of(cfs_rq)); + if (tsk->state & TASK_UNINTERRUPTIBLE) + se->statistics.block_start = rq_clock(rq_of(cfs_rq)); } + } } #else @@ -845,10 +845,10 @@ update_stats_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) static inline void update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se) { - /* - * We are starting a new run period: - */ - se->exec_start = rq_clock_task(rq_of(cfs_rq)); + /* + * We are starting a new run period: + */ + se->exec_start = rq_clock_task(rq_of(cfs_rq)); } /************************************************** @@ -872,21 +872,21 @@ unsigned int sysctl_numa_balancing_scan_delay = 1000; static unsigned int task_nr_scan_windows(struct task_struct *p) { - unsigned long rss = 0; - unsigned long nr_scan_pages; + unsigned long rss = 0; + unsigned long nr_scan_pages; - /* - * Calculations based on RSS as non-present and empty pages are skipped - * by the PTE scanner and NUMA hinting faults should be trapped based - * on resident pages - */ - nr_scan_pages = sysctl_numa_balancing_scan_size << (20 - PAGE_SHIFT); - rss = get_mm_rss(p->mm); - if (!rss) - rss = nr_scan_pages; + /* + * Calculations based on RSS as non-present and empty pages are skipped + * by the PTE scanner and NUMA hinting faults should be trapped based + * on resident pages + */ + nr_scan_pages = sysctl_numa_balancing_scan_size << (20 - PAGE_SHIFT); + rss = get_mm_rss(p->mm); + if (!rss) + rss = nr_scan_pages; - rss = round_up(rss, nr_scan_pages); - return rss / nr_scan_pages; + rss = round_up(rss, nr_scan_pages); + return rss / nr_scan_pages; } /* For sanitys sake, never scan more PTEs than MAX_SCAN_WINDOW MB/sec. */ @@ -894,58 +894,58 @@ static unsigned int task_nr_scan_windows(struct task_struct *p) static unsigned int task_scan_min(struct task_struct *p) { - unsigned int scan_size = READ_ONCE(sysctl_numa_balancing_scan_size); - unsigned int scan, floor; - unsigned int windows = 1; + unsigned int scan_size = READ_ONCE(sysctl_numa_balancing_scan_size); + unsigned int scan, floor; + unsigned int windows = 1; - if (scan_size < MAX_SCAN_WINDOW) - windows = MAX_SCAN_WINDOW / scan_size; - floor = 1000 / windows; + if (scan_size < MAX_SCAN_WINDOW) + windows = MAX_SCAN_WINDOW / scan_size; + floor = 1000 / windows; - scan = sysctl_numa_balancing_scan_period_min / task_nr_scan_windows(p); - return max_t(unsigned int, floor, scan); + scan = sysctl_numa_balancing_scan_period_min / task_nr_scan_windows(p); + return max_t(unsigned int, floor, scan); } static unsigned int task_scan_max(struct task_struct *p) { - unsigned int smin = task_scan_min(p); - unsigned int smax; + unsigned int smin = task_scan_min(p); + unsigned int smax; - /* Watch for min being lower than max due to floor calculations */ - smax = sysctl_numa_balancing_scan_period_max / task_nr_scan_windows(p); - return max(smin, smax); + /* Watch for min being lower than max due to floor calculations */ + smax = sysctl_numa_balancing_scan_period_max / task_nr_scan_windows(p); + return max(smin, smax); } static void account_numa_enqueue(struct rq *rq, struct task_struct *p) { - rq->nr_numa_running += (p->numa_preferred_nid != -1); - rq->nr_preferred_running += (p->numa_preferred_nid == task_node(p)); + rq->nr_numa_running += (p->numa_preferred_nid != -1); + rq->nr_preferred_running += (p->numa_preferred_nid == task_node(p)); } static void account_numa_dequeue(struct rq *rq, struct task_struct *p) { - rq->nr_numa_running -= (p->numa_preferred_nid != -1); - rq->nr_preferred_running -= (p->numa_preferred_nid == task_node(p)); + rq->nr_numa_running -= (p->numa_preferred_nid != -1); + rq->nr_preferred_running -= (p->numa_preferred_nid == task_node(p)); } struct numa_group { - atomic_t refcount; + atomic_t refcount; - spinlock_t lock; /* nr_tasks, tasks */ - int nr_tasks; - pid_t gid; - int active_nodes; + spinlock_t lock; /* nr_tasks, tasks */ + int nr_tasks; + pid_t gid; + int active_nodes; - struct rcu_head rcu; - unsigned long total_faults; - unsigned long max_faults_cpu; - /* - * Faults_cpu is used to decide whether memory should move - * towards the CPU. As a consequence, these stats are weighted - * more by CPU use than by memory faults. - */ - unsigned long *faults_cpu; - unsigned long faults[0]; + struct rcu_head rcu; + unsigned long total_faults; + unsigned long max_faults_cpu; + /* + * Faults_cpu is used to decide whether memory should move + * towards the CPU. As a consequence, these stats are weighted + * more by CPU use than by memory faults. + */ + unsigned long *faults_cpu; + unsigned long faults[0]; }; /* Shared or private faults. */ @@ -959,7 +959,7 @@ struct numa_group { pid_t task_numa_group_id(struct task_struct *p) { - return p->numa_group ? p->numa_group->gid : 0; + return p->numa_group ? p->numa_group->gid : 0; } /* @@ -970,31 +970,31 @@ pid_t task_numa_group_id(struct task_struct *p) */ static inline int task_faults_idx(enum numa_faults_stats s, int nid, int priv) { - return NR_NUMA_HINT_FAULT_TYPES * (s * nr_node_ids + nid) + priv; + return NR_NUMA_HINT_FAULT_TYPES * (s * nr_node_ids + nid) + priv; } static inline unsigned long task_faults(struct task_struct *p, int nid) { - if (!p->numa_faults) - return 0; + if (!p->numa_faults) + return 0; - return p->numa_faults[task_faults_idx(NUMA_MEM, nid, 0)] + - p->numa_faults[task_faults_idx(NUMA_MEM, nid, 1)]; + return p->numa_faults[task_faults_idx(NUMA_MEM, nid, 0)] + + p->numa_faults[task_faults_idx(NUMA_MEM, nid, 1)]; } static inline unsigned long group_faults(struct task_struct *p, int nid) { - if (!p->numa_group) - return 0; + if (!p->numa_group) + return 0; - return p->numa_group->faults[task_faults_idx(NUMA_MEM, nid, 0)] + - p->numa_group->faults[task_faults_idx(NUMA_MEM, nid, 1)]; + return p->numa_group->faults[task_faults_idx(NUMA_MEM, nid, 0)] + + p->numa_group->faults[task_faults_idx(NUMA_MEM, nid, 1)]; } static inline unsigned long group_faults_cpu(struct numa_group *group, int nid) { - return group->faults_cpu[task_faults_idx(NUMA_MEM, nid, 0)] + - group->faults_cpu[task_faults_idx(NUMA_MEM, nid, 1)]; + return group->faults_cpu[task_faults_idx(NUMA_MEM, nid, 0)] + + group->faults_cpu[task_faults_idx(NUMA_MEM, nid, 1)]; } /* @@ -1006,72 +1006,72 @@ static inline unsigned long group_faults_cpu(struct numa_group *group, int nid) static bool numa_is_active_node(int nid, struct numa_group *ng) { - return group_faults_cpu(ng, nid) * ACTIVE_NODE_FRACTION > ng->max_faults_cpu; + return group_faults_cpu(ng, nid) * ACTIVE_NODE_FRACTION > ng->max_faults_cpu; } /* Handle placement on systems where not all nodes are directly connected. */ static unsigned long score_nearby_nodes(struct task_struct *p, int nid, - int maxdist, bool task) + int maxdist, bool task) { - unsigned long score = 0; - int node; + unsigned long score = 0; + int node; + + /* + * All nodes are directly connected, and the same distance + * from each other. No need for fancy placement algorithms. + */ + if (sched_numa_topology_type == NUMA_DIRECT) + return 0; + + /* + * This code is called for each node, introducing N^2 complexity, + * which should be ok given the number of nodes rarely exceeds 8. + */ + for_each_online_node(node) { + unsigned long faults; + int dist = node_distance(nid, node); /* - * All nodes are directly connected, and the same distance - * from each other. No need for fancy placement algorithms. + * The furthest away nodes in the system are not interesting + * for placement; nid was already counted. */ - if (sched_numa_topology_type == NUMA_DIRECT) - return 0; + if (dist == sched_max_numa_distance || node == nid) + continue; /* - * This code is called for each node, introducing N^2 complexity, - * which should be ok given the number of nodes rarely exceeds 8. + * On systems with a backplane NUMA topology, compare groups + * of nodes, and move tasks towards the group with the most + * memory accesses. When comparing two nodes at distance + * "hoplimit", only nodes closer by than "hoplimit" are part + * of each group. Skip other nodes. */ - for_each_online_node(node) { - unsigned long faults; - int dist = node_distance(nid, node); + if (sched_numa_topology_type == NUMA_BACKPLANE && + dist > maxdist) + continue; - /* - * The furthest away nodes in the system are not interesting - * for placement; nid was already counted. - */ - if (dist == sched_max_numa_distance || node == nid) - continue; + /* Add up the faults from nearby nodes. */ + if (task) + faults = task_faults(p, node); + else + faults = group_faults(p, node); - /* - * On systems with a backplane NUMA topology, compare groups - * of nodes, and move tasks towards the group with the most - * memory accesses. When comparing two nodes at distance - * "hoplimit", only nodes closer by than "hoplimit" are part - * of each group. Skip other nodes. - */ - if (sched_numa_topology_type == NUMA_BACKPLANE && - dist > maxdist) - continue; - - /* Add up the faults from nearby nodes. */ - if (task) - faults = task_faults(p, node); - else - faults = group_faults(p, node); - - /* - * On systems with a glueless mesh NUMA topology, there are - * no fixed "groups of nodes". Instead, nodes that are not - * directly connected bounce traffic through intermediate - * nodes; a numa_group can occupy any set of nodes. - * The further away a node is, the less the faults count. - * This seems to result in good task placement. - */ - if (sched_numa_topology_type == NUMA_GLUELESS_MESH) { - faults *= (sched_max_numa_distance - dist); - faults /= (sched_max_numa_distance - LOCAL_DISTANCE); - } - - score += faults; + /* + * On systems with a glueless mesh NUMA topology, there are + * no fixed "groups of nodes". Instead, nodes that are not + * directly connected bounce traffic through intermediate + * nodes; a numa_group can occupy any set of nodes. + * The further away a node is, the less the faults count. + * This seems to result in good task placement. + */ + if (sched_numa_topology_type == NUMA_GLUELESS_MESH) { + faults *= (sched_max_numa_distance - dist); + faults /= (sched_max_numa_distance - LOCAL_DISTANCE); } - return score; + score += faults; + } + + return score; } /* @@ -1081,100 +1081,100 @@ static unsigned long score_nearby_nodes(struct task_struct *p, int nid, * evenly spread out between numa nodes. */ static inline unsigned long task_weight(struct task_struct *p, int nid, - int dist) + int dist) { - unsigned long faults, total_faults; + unsigned long faults, total_faults; - if (!p->numa_faults) - return 0; + if (!p->numa_faults) + return 0; - total_faults = p->total_numa_faults; + total_faults = p->total_numa_faults; - if (!total_faults) - return 0; + if (!total_faults) + return 0; - faults = task_faults(p, nid); - faults += score_nearby_nodes(p, nid, dist, true); + faults = task_faults(p, nid); + faults += score_nearby_nodes(p, nid, dist, true); - return 1000 * faults / total_faults; + return 1000 * faults / total_faults; } static inline unsigned long group_weight(struct task_struct *p, int nid, - int dist) + int dist) { - unsigned long faults, total_faults; + unsigned long faults, total_faults; - if (!p->numa_group) - return 0; + if (!p->numa_group) + return 0; - total_faults = p->numa_group->total_faults; + total_faults = p->numa_group->total_faults; - if (!total_faults) - return 0; + if (!total_faults) + return 0; - faults = group_faults(p, nid); - faults += score_nearby_nodes(p, nid, dist, false); + faults = group_faults(p, nid); + faults += score_nearby_nodes(p, nid, dist, false); - return 1000 * faults / total_faults; + return 1000 * faults / total_faults; } bool should_numa_migrate_memory(struct task_struct *p, struct page * page, - int src_nid, int dst_cpu) + int src_nid, int dst_cpu) { - struct numa_group *ng = p->numa_group; - int dst_nid = cpu_to_node(dst_cpu); - int last_cpupid, this_cpupid; + struct numa_group *ng = p->numa_group; + int dst_nid = cpu_to_node(dst_cpu); + int last_cpupid, this_cpupid; - this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid); + this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid); - /* - * Multi-stage node selection is used in conjunction with a periodic - * migration fault to build a temporal task<->page relation. By using - * a two-stage filter we remove short/unlikely relations. - * - * Using P(p) ~ n_p / n_t as per frequentist probability, we can equate - * a task's usage of a particular page (n_p) per total usage of this - * page (n_t) (in a given time-span) to a probability. - * - * Our periodic faults will sample this probability and getting the - * same result twice in a row, given these samples are fully - * independent, is then given by P(n)^2, provided our sample period - * is sufficiently short compared to the usage pattern. - * - * This quadric squishes small probabilities, making it less likely we - * act on an unlikely task<->page relation. - */ - last_cpupid = page_cpupid_xchg_last(page, this_cpupid); - if (!cpupid_pid_unset(last_cpupid) && - cpupid_to_nid(last_cpupid) != dst_nid) - return false; + /* + * Multi-stage node selection is used in conjunction with a periodic + * migration fault to build a temporal task<->page relation. By using + * a two-stage filter we remove short/unlikely relations. + * + * Using P(p) ~ n_p / n_t as per frequentist probability, we can equate + * a task's usage of a particular page (n_p) per total usage of this + * page (n_t) (in a given time-span) to a probability. + * + * Our periodic faults will sample this probability and getting the + * same result twice in a row, given these samples are fully + * independent, is then given by P(n)^2, provided our sample period + * is sufficiently short compared to the usage pattern. + * + * This quadric squishes small probabilities, making it less likely we + * act on an unlikely task<->page relation. + */ + last_cpupid = page_cpupid_xchg_last(page, this_cpupid); + if (!cpupid_pid_unset(last_cpupid) && + cpupid_to_nid(last_cpupid) != dst_nid) + return false; - /* Always allow migrate on private faults */ - if (cpupid_match_pid(p, last_cpupid)) - return true; + /* Always allow migrate on private faults */ + if (cpupid_match_pid(p, last_cpupid)) + return true; - /* A shared fault, but p->numa_group has not been set up yet. */ - if (!ng) - return true; + /* A shared fault, but p->numa_group has not been set up yet. */ + if (!ng) + return true; - /* - * Destination node is much more heavily used than the source - * node? Allow migration. - */ - if (group_faults_cpu(ng, dst_nid) > group_faults_cpu(ng, src_nid) * - ACTIVE_NODE_FRACTION) - return true; + /* + * Destination node is much more heavily used than the source + * node? Allow migration. + */ + if (group_faults_cpu(ng, dst_nid) > group_faults_cpu(ng, src_nid) * + ACTIVE_NODE_FRACTION) + return true; - /* - * Distribute memory according to CPU & memory use on each node, - * with 3/4 hysteresis to avoid unnecessary memory migrations: - * - * faults_cpu(dst) 3 faults_cpu(src) - * --------------- * - > --------------- - * faults_mem(dst) 4 faults_mem(src) - */ - return group_faults_cpu(ng, dst_nid) * group_faults(p, src_nid) * 3 > - group_faults_cpu(ng, src_nid) * group_faults(p, dst_nid) * 4; + /* + * Distribute memory according to CPU & memory use on each node, + * with 3/4 hysteresis to avoid unnecessary memory migrations: + * + * faults_cpu(dst) 3 faults_cpu(src) + * --------------- * - > --------------- + * faults_mem(dst) 4 faults_mem(src) + */ + return group_faults_cpu(ng, dst_nid) * group_faults(p, src_nid) * 3 > + group_faults_cpu(ng, src_nid) * group_faults(p, dst_nid) * 4; } static unsigned long weighted_cpuload(const int cpu); @@ -1185,15 +1185,15 @@ static long effective_load(struct task_group *tg, int cpu, long wl, long wg); /* Cached statistics for all CPUs within a node */ struct numa_stats { - unsigned long nr_running; - unsigned long load; + unsigned long nr_running; + unsigned long load; - /* Total compute capacity of CPUs on a node */ - unsigned long compute_capacity; + /* Total compute capacity of CPUs on a node */ + unsigned long compute_capacity; - /* Approximate capacity in terms of runnable tasks on a node */ - unsigned long task_capacity; - int has_free_capacity; + /* Approximate capacity in terms of runnable tasks on a node */ + unsigned long task_capacity; + int has_free_capacity; }; /* @@ -1201,109 +1201,109 @@ struct numa_stats { */ static void update_numa_stats(struct numa_stats *ns, int nid) { - int smt, cpu, cpus = 0; - unsigned long capacity; + int smt, cpu, cpus = 0; + unsigned long capacity; - memset(ns, 0, sizeof(*ns)); - for_each_cpu(cpu, cpumask_of_node(nid)) { - struct rq *rq = cpu_rq(cpu); + memset(ns, 0, sizeof(*ns)); + for_each_cpu(cpu, cpumask_of_node(nid)) { + struct rq *rq = cpu_rq(cpu); - ns->nr_running += rq->nr_running; - ns->load += weighted_cpuload(cpu); - ns->compute_capacity += capacity_of(cpu); + ns->nr_running += rq->nr_running; + ns->load += weighted_cpuload(cpu); + ns->compute_capacity += capacity_of(cpu); - cpus++; - } + cpus++; + } - /* - * If we raced with hotplug and there are no CPUs left in our mask - * the @ns structure is NULL'ed and task_numa_compare() will - * not find this node attractive. - * - * We'll either bail at !has_free_capacity, or we'll detect a huge - * imbalance and bail there. - */ - if (!cpus) - return; + /* + * If we raced with hotplug and there are no CPUs left in our mask + * the @ns structure is NULL'ed and task_numa_compare() will + * not find this node attractive. + * + * We'll either bail at !has_free_capacity, or we'll detect a huge + * imbalance and bail there. + */ + if (!cpus) + return; - /* smt := ceil(cpus / capacity), assumes: 1 < smt_power < 2 */ - smt = DIV_ROUND_UP(SCHED_CAPACITY_SCALE * cpus, ns->compute_capacity); - capacity = cpus / smt; /* cores */ + /* smt := ceil(cpus / capacity), assumes: 1 < smt_power < 2 */ + smt = DIV_ROUND_UP(SCHED_CAPACITY_SCALE * cpus, ns->compute_capacity); + capacity = cpus / smt; /* cores */ - ns->task_capacity = min_t(unsigned, capacity, - DIV_ROUND_CLOSEST(ns->compute_capacity, SCHED_CAPACITY_SCALE)); - ns->has_free_capacity = (ns->nr_running < ns->task_capacity); + ns->task_capacity = min_t(unsigned, capacity, + DIV_ROUND_CLOSEST(ns->compute_capacity, SCHED_CAPACITY_SCALE)); + ns->has_free_capacity = (ns->nr_running < ns->task_capacity); } struct task_numa_env { - struct task_struct *p; + struct task_struct *p; - int src_cpu, src_nid; - int dst_cpu, dst_nid; + int src_cpu, src_nid; + int dst_cpu, dst_nid; - struct numa_stats src_stats, dst_stats; + struct numa_stats src_stats, dst_stats; - int imbalance_pct; - int dist; + int imbalance_pct; + int dist; - struct task_struct *best_task; - long best_imp; - int best_cpu; + struct task_struct *best_task; + long best_imp; + int best_cpu; }; static void task_numa_assign(struct task_numa_env *env, - struct task_struct *p, long imp) + struct task_struct *p, long imp) { - if (env->best_task) - put_task_struct(env->best_task); + if (env->best_task) + put_task_struct(env->best_task); - env->best_task = p; - env->best_imp = imp; - env->best_cpu = env->dst_cpu; + env->best_task = p; + env->best_imp = imp; + env->best_cpu = env->dst_cpu; } static bool load_too_imbalanced(long src_load, long dst_load, - struct task_numa_env *env) + struct task_numa_env *env) { - long imb, old_imb; - long orig_src_load, orig_dst_load; - long src_capacity, dst_capacity; + long imb, old_imb; + long orig_src_load, orig_dst_load; + long src_capacity, dst_capacity; - /* - * The load is corrected for the CPU capacity available on each node. - * - * src_load dst_load - * ------------ vs --------- - * src_capacity dst_capacity - */ - src_capacity = env->src_stats.compute_capacity; - dst_capacity = env->dst_stats.compute_capacity; + /* + * The load is corrected for the CPU capacity available on each node. + * + * src_load dst_load + * ------------ vs --------- + * src_capacity dst_capacity + */ + src_capacity = env->src_stats.compute_capacity; + dst_capacity = env->dst_stats.compute_capacity; - /* We care about the slope of the imbalance, not the direction. */ - if (dst_load < src_load) - swap(dst_load, src_load); + /* We care about the slope of the imbalance, not the direction. */ + if (dst_load < src_load) + swap(dst_load, src_load); - /* Is the difference below the threshold? */ - imb = dst_load * src_capacity * 100 - - src_load * dst_capacity * env->imbalance_pct; - if (imb <= 0) - return false; + /* Is the difference below the threshold? */ + imb = dst_load * src_capacity * 100 - + src_load * dst_capacity * env->imbalance_pct; + if (imb <= 0) + return false; - /* - * The imbalance is above the allowed threshold. - * Compare it with the old imbalance. - */ - orig_src_load = env->src_stats.load; - orig_dst_load = env->dst_stats.load; + /* + * The imbalance is above the allowed threshold. + * Compare it with the old imbalance. + */ + orig_src_load = env->src_stats.load; + orig_dst_load = env->dst_stats.load; - if (orig_dst_load < orig_src_load) - swap(orig_dst_load, orig_src_load); + if (orig_dst_load < orig_src_load) + swap(orig_dst_load, orig_src_load); - old_imb = orig_dst_load * src_capacity * 100 - - orig_src_load * dst_capacity * env->imbalance_pct; + old_imb = orig_dst_load * src_capacity * 100 - + orig_src_load * dst_capacity * env->imbalance_pct; - /* Would this change make things worse? */ - return (imb > old_imb); + /* Would this change make things worse? */ + return (imb > old_imb); } /* @@ -1313,354 +1313,354 @@ static bool load_too_imbalanced(long src_load, long dst_load, * be exchanged with the source task */ static void task_numa_compare(struct task_numa_env *env, - long taskimp, long groupimp) + long taskimp, long groupimp) { - struct rq *src_rq = cpu_rq(env->src_cpu); - struct rq *dst_rq = cpu_rq(env->dst_cpu); - struct task_struct *cur; - long src_load, dst_load; - long load; - long imp = env->p->numa_group ? groupimp : taskimp; - long moveimp = imp; - int dist = env->dist; - bool assigned = false; + struct rq *src_rq = cpu_rq(env->src_cpu); + struct rq *dst_rq = cpu_rq(env->dst_cpu); + struct task_struct *cur; + long src_load, dst_load; + long load; + long imp = env->p->numa_group ? groupimp : taskimp; + long moveimp = imp; + int dist = env->dist; + bool assigned = false; - rcu_read_lock(); + rcu_read_lock(); - raw_spin_lock_irq(&dst_rq->lock); - cur = dst_rq->curr; + raw_spin_lock_irq(&dst_rq->lock); + cur = dst_rq->curr; + /* + * No need to move the exiting task or idle task. + */ + if ((cur->flags & PF_EXITING) || is_idle_task(cur)) + cur = NULL; + else { /* - * No need to move the exiting task or idle task. + * The task_struct must be protected here to protect the + * p->numa_faults access in the task_weight since the + * numa_faults could already be freed in the following path: + * finish_task_switch() + * --> put_task_struct() + * --> __put_task_struct() + * --> task_numa_free() */ - if ((cur->flags & PF_EXITING) || is_idle_task(cur)) - cur = NULL; - else { - /* - * The task_struct must be protected here to protect the - * p->numa_faults access in the task_weight since the - * numa_faults could already be freed in the following path: - * finish_task_switch() - * --> put_task_struct() - * --> __put_task_struct() - * --> task_numa_free() - */ - get_task_struct(cur); + get_task_struct(cur); + } + + raw_spin_unlock_irq(&dst_rq->lock); + + /* + * Because we have preemption enabled we can get migrated around and + * end try selecting ourselves (current == env->p) as a swap candidate. + */ + if (cur == env->p) + goto unlock; + + /* + * "imp" is the fault differential for the source task between the + * source and destination node. Calculate the total differential for + * the source task and potential destination task. The more negative + * the value is, the more rmeote accesses that would be expected to + * be incurred if the tasks were swapped. + */ + if (cur) { + /* Skip this swap candidate if cannot move to the source cpu */ + if (!cpumask_test_cpu(env->src_cpu, tsk_cpus_allowed(cur))) + goto unlock; + + /* + * If dst and source tasks are in the same NUMA group, or not + * in any group then look only at task weights. + */ + if (cur->numa_group == env->p->numa_group) { + imp = taskimp + task_weight(cur, env->src_nid, dist) - + task_weight(cur, env->dst_nid, dist); + /* + * Add some hysteresis to prevent swapping the + * tasks within a group over tiny differences. + */ + if (cur->numa_group) + imp -= imp/16; + } else { + /* + * Compare the group weights. If a task is all by + * itself (not part of a group), use the task weight + * instead. + */ + if (cur->numa_group) + imp += group_weight(cur, env->src_nid, dist) - + group_weight(cur, env->dst_nid, dist); + else + imp += task_weight(cur, env->src_nid, dist) - + task_weight(cur, env->dst_nid, dist); } + } - raw_spin_unlock_irq(&dst_rq->lock); + if (imp <= env->best_imp && moveimp <= env->best_imp) + goto unlock; - /* - * Because we have preemption enabled we can get migrated around and - * end try selecting ourselves (current == env->p) as a swap candidate. - */ - if (cur == env->p) - goto unlock; + if (!cur) { + /* Is there capacity at our destination? */ + if (env->src_stats.nr_running <= env->src_stats.task_capacity && + !env->dst_stats.has_free_capacity) + goto unlock; - /* - * "imp" is the fault differential for the source task between the - * source and destination node. Calculate the total differential for - * the source task and potential destination task. The more negative - * the value is, the more rmeote accesses that would be expected to - * be incurred if the tasks were swapped. - */ - if (cur) { - /* Skip this swap candidate if cannot move to the source cpu */ - if (!cpumask_test_cpu(env->src_cpu, tsk_cpus_allowed(cur))) - goto unlock; + goto balance; + } - /* - * If dst and source tasks are in the same NUMA group, or not - * in any group then look only at task weights. - */ - if (cur->numa_group == env->p->numa_group) { - imp = taskimp + task_weight(cur, env->src_nid, dist) - - task_weight(cur, env->dst_nid, dist); - /* - * Add some hysteresis to prevent swapping the - * tasks within a group over tiny differences. - */ - if (cur->numa_group) - imp -= imp/16; - } else { - /* - * Compare the group weights. If a task is all by - * itself (not part of a group), use the task weight - * instead. - */ - if (cur->numa_group) - imp += group_weight(cur, env->src_nid, dist) - - group_weight(cur, env->dst_nid, dist); - else - imp += task_weight(cur, env->src_nid, dist) - - task_weight(cur, env->dst_nid, dist); - } - } + /* Balance doesn't matter much if we're running a task per cpu */ + if (imp > env->best_imp && src_rq->nr_running == 1 && + dst_rq->nr_running == 1) + goto assign; - if (imp <= env->best_imp && moveimp <= env->best_imp) - goto unlock; - - if (!cur) { - /* Is there capacity at our destination? */ - if (env->src_stats.nr_running <= env->src_stats.task_capacity && - !env->dst_stats.has_free_capacity) - goto unlock; - - goto balance; - } - - /* Balance doesn't matter much if we're running a task per cpu */ - if (imp > env->best_imp && src_rq->nr_running == 1 && - dst_rq->nr_running == 1) - goto assign; - - /* - * In the overloaded case, try and keep the load balanced. - */ + /* + * In the overloaded case, try and keep the load balanced. + */ balance: - load = task_h_load(env->p); - dst_load = env->dst_stats.load + load; - src_load = env->src_stats.load - load; - - if (moveimp > imp && moveimp > env->best_imp) { - /* - * If the improvement from just moving env->p direction is - * better than swapping tasks around, check if a move is - * possible. Store a slightly smaller score than moveimp, - * so an actually idle CPU will win. - */ - if (!load_too_imbalanced(src_load, dst_load, env)) { - imp = moveimp - 1; - put_task_struct(cur); - cur = NULL; - goto assign; - } - } - - if (imp <= env->best_imp) - goto unlock; - - if (cur) { - load = task_h_load(cur); - dst_load -= load; - src_load += load; - } - - if (load_too_imbalanced(src_load, dst_load, env)) - goto unlock; + load = task_h_load(env->p); + dst_load = env->dst_stats.load + load; + src_load = env->src_stats.load - load; + if (moveimp > imp && moveimp > env->best_imp) { /* - * One idle CPU per node is evaluated for a task numa move. - * Call select_idle_sibling to maybe find a better one. + * If the improvement from just moving env->p direction is + * better than swapping tasks around, check if a move is + * possible. Store a slightly smaller score than moveimp, + * so an actually idle CPU will win. */ - if (!cur) - env->dst_cpu = select_idle_sibling(env->p, env->dst_cpu); + if (!load_too_imbalanced(src_load, dst_load, env)) { + imp = moveimp - 1; + put_task_struct(cur); + cur = NULL; + goto assign; + } + } + + if (imp <= env->best_imp) + goto unlock; + + if (cur) { + load = task_h_load(cur); + dst_load -= load; + src_load += load; + } + + if (load_too_imbalanced(src_load, dst_load, env)) + goto unlock; + + /* + * One idle CPU per node is evaluated for a task numa move. + * Call select_idle_sibling to maybe find a better one. + */ + if (!cur) + env->dst_cpu = select_idle_sibling(env->p, env->dst_cpu); assign: - assigned = true; - task_numa_assign(env, cur, imp); + assigned = true; + task_numa_assign(env, cur, imp); unlock: - rcu_read_unlock(); - /* - * The dst_rq->curr isn't assigned. The protection for task_struct is - * finished. - */ - if (cur && !assigned) - put_task_struct(cur); + rcu_read_unlock(); + /* + * The dst_rq->curr isn't assigned. The protection for task_struct is + * finished. + */ + if (cur && !assigned) + put_task_struct(cur); } static void task_numa_find_cpu(struct task_numa_env *env, - long taskimp, long groupimp) + long taskimp, long groupimp) { - int cpu; + int cpu; - for_each_cpu(cpu, cpumask_of_node(env->dst_nid)) { - /* Skip this CPU if the source task cannot migrate */ - if (!cpumask_test_cpu(cpu, tsk_cpus_allowed(env->p))) - continue; + for_each_cpu(cpu, cpumask_of_node(env->dst_nid)) { + /* Skip this CPU if the source task cannot migrate */ + if (!cpumask_test_cpu(cpu, tsk_cpus_allowed(env->p))) + continue; - env->dst_cpu = cpu; - task_numa_compare(env, taskimp, groupimp); - } + env->dst_cpu = cpu; + task_numa_compare(env, taskimp, groupimp); + } } /* Only move tasks to a NUMA node less busy than the current node. */ static bool numa_has_capacity(struct task_numa_env *env) { - struct numa_stats *src = &env->src_stats; - struct numa_stats *dst = &env->dst_stats; - - if (src->has_free_capacity && !dst->has_free_capacity) - return false; - - /* - * Only consider a task move if the source has a higher load - * than the destination, corrected for CPU capacity on each node. - * - * src->load dst->load - * --------------------- vs --------------------- - * src->compute_capacity dst->compute_capacity - */ - if (src->load * dst->compute_capacity * env->imbalance_pct > - - dst->load * src->compute_capacity * 100) - return true; + struct numa_stats *src = &env->src_stats; + struct numa_stats *dst = &env->dst_stats; + if (src->has_free_capacity && !dst->has_free_capacity) return false; + + /* + * Only consider a task move if the source has a higher load + * than the destination, corrected for CPU capacity on each node. + * + * src->load dst->load + * --------------------- vs --------------------- + * src->compute_capacity dst->compute_capacity + */ + if (src->load * dst->compute_capacity * env->imbalance_pct > + + dst->load * src->compute_capacity * 100) + return true; + + return false; } static int task_numa_migrate(struct task_struct *p) { - struct task_numa_env env = { - .p = p, + struct task_numa_env env = { + .p = p, - .src_cpu = task_cpu(p), - .src_nid = task_node(p), + .src_cpu = task_cpu(p), + .src_nid = task_node(p), - .imbalance_pct = 112, + .imbalance_pct = 112, - .best_task = NULL, - .best_imp = 0, - .best_cpu = -1, - }; - struct sched_domain *sd; - unsigned long taskweight, groupweight; - int nid, ret, dist; - long taskimp, groupimp; + .best_task = NULL, + .best_imp = 0, + .best_cpu = -1, + }; + struct sched_domain *sd; + unsigned long taskweight, groupweight; + int nid, ret, dist; + long taskimp, groupimp; - /* - * Pick the lowest SD_NUMA domain, as that would have the smallest - * imbalance and would be the first to start moving tasks about. - * - * And we want to avoid any moving of tasks about, as that would create - * random movement of tasks -- counter the numa conditions we're trying - * to satisfy here. - */ - rcu_read_lock(); - sd = rcu_dereference(per_cpu(sd_numa, env.src_cpu)); - if (sd) - env.imbalance_pct = 100 + (sd->imbalance_pct - 100) / 2; - rcu_read_unlock(); + /* + * Pick the lowest SD_NUMA domain, as that would have the smallest + * imbalance and would be the first to start moving tasks about. + * + * And we want to avoid any moving of tasks about, as that would create + * random movement of tasks -- counter the numa conditions we're trying + * to satisfy here. + */ + rcu_read_lock(); + sd = rcu_dereference(per_cpu(sd_numa, env.src_cpu)); + if (sd) + env.imbalance_pct = 100 + (sd->imbalance_pct - 100) / 2; + rcu_read_unlock(); - /* - * Cpusets can break the scheduler domain tree into smaller - * balance domains, some of which do not cross NUMA boundaries. - * Tasks that are "trapped" in such domains cannot be migrated - * elsewhere, so there is no point in (re)trying. - */ - if (unlikely(!sd)) { - p->numa_preferred_nid = task_node(p); - return -EINVAL; - } + /* + * Cpusets can break the scheduler domain tree into smaller + * balance domains, some of which do not cross NUMA boundaries. + * Tasks that are "trapped" in such domains cannot be migrated + * elsewhere, so there is no point in (re)trying. + */ + if (unlikely(!sd)) { + p->numa_preferred_nid = task_node(p); + return -EINVAL; + } - env.dst_nid = p->numa_preferred_nid; - dist = env.dist = node_distance(env.src_nid, env.dst_nid); - taskweight = task_weight(p, env.src_nid, dist); - groupweight = group_weight(p, env.src_nid, dist); - update_numa_stats(&env.src_stats, env.src_nid); - taskimp = task_weight(p, env.dst_nid, dist) - taskweight; - groupimp = group_weight(p, env.dst_nid, dist) - groupweight; - update_numa_stats(&env.dst_stats, env.dst_nid); + env.dst_nid = p->numa_preferred_nid; + dist = env.dist = node_distance(env.src_nid, env.dst_nid); + taskweight = task_weight(p, env.src_nid, dist); + groupweight = group_weight(p, env.src_nid, dist); + update_numa_stats(&env.src_stats, env.src_nid); + taskimp = task_weight(p, env.dst_nid, dist) - taskweight; + groupimp = group_weight(p, env.dst_nid, dist) - groupweight; + update_numa_stats(&env.dst_stats, env.dst_nid); - /* Try to find a spot on the preferred nid. */ - if (numa_has_capacity(&env)) + /* Try to find a spot on the preferred nid. */ + if (numa_has_capacity(&env)) + task_numa_find_cpu(&env, taskimp, groupimp); + + /* + * Look at other nodes in these cases: + * - there is no space available on the preferred_nid + * - the task is part of a numa_group that is interleaved across + * multiple NUMA nodes; in order to better consolidate the group, + * we need to check other locations. + */ + if (env.best_cpu == -1 || (p->numa_group && p->numa_group->active_nodes > 1)) { + for_each_online_node(nid) { + if (nid == env.src_nid || nid == p->numa_preferred_nid) + continue; + + dist = node_distance(env.src_nid, env.dst_nid); + if (sched_numa_topology_type == NUMA_BACKPLANE && + dist != env.dist) { + taskweight = task_weight(p, env.src_nid, dist); + groupweight = group_weight(p, env.src_nid, dist); + } + + /* Only consider nodes where both task and groups benefit */ + taskimp = task_weight(p, nid, dist) - taskweight; + groupimp = group_weight(p, nid, dist) - groupweight; + if (taskimp < 0 && groupimp < 0) + continue; + + env.dist = dist; + env.dst_nid = nid; + update_numa_stats(&env.dst_stats, env.dst_nid); + if (numa_has_capacity(&env)) task_numa_find_cpu(&env, taskimp, groupimp); - - /* - * Look at other nodes in these cases: - * - there is no space available on the preferred_nid - * - the task is part of a numa_group that is interleaved across - * multiple NUMA nodes; in order to better consolidate the group, - * we need to check other locations. - */ - if (env.best_cpu == -1 || (p->numa_group && p->numa_group->active_nodes > 1)) { - for_each_online_node(nid) { - if (nid == env.src_nid || nid == p->numa_preferred_nid) - continue; - - dist = node_distance(env.src_nid, env.dst_nid); - if (sched_numa_topology_type == NUMA_BACKPLANE && - dist != env.dist) { - taskweight = task_weight(p, env.src_nid, dist); - groupweight = group_weight(p, env.src_nid, dist); - } - - /* Only consider nodes where both task and groups benefit */ - taskimp = task_weight(p, nid, dist) - taskweight; - groupimp = group_weight(p, nid, dist) - groupweight; - if (taskimp < 0 && groupimp < 0) - continue; - - env.dist = dist; - env.dst_nid = nid; - update_numa_stats(&env.dst_stats, env.dst_nid); - if (numa_has_capacity(&env)) - task_numa_find_cpu(&env, taskimp, groupimp); - } } + } - /* - * If the task is part of a workload that spans multiple NUMA nodes, - * and is migrating into one of the workload's active nodes, remember - * this node as the task's preferred numa node, so the workload can - * settle down. - * A task that migrated to a second choice node will be better off - * trying for a better one later. Do not set the preferred node here. - */ - if (p->numa_group) { - struct numa_group *ng = p->numa_group; + /* + * If the task is part of a workload that spans multiple NUMA nodes, + * and is migrating into one of the workload's active nodes, remember + * this node as the task's preferred numa node, so the workload can + * settle down. + * A task that migrated to a second choice node will be better off + * trying for a better one later. Do not set the preferred node here. + */ + if (p->numa_group) { + struct numa_group *ng = p->numa_group; - if (env.best_cpu == -1) - nid = env.src_nid; - else - nid = env.dst_nid; - - if (ng->active_nodes > 1 && numa_is_active_node(env.dst_nid, ng)) - sched_setnuma(p, env.dst_nid); - } - - /* No better CPU than the current one was found. */ if (env.best_cpu == -1) - return -EAGAIN; + nid = env.src_nid; + else + nid = env.dst_nid; - /* - * Reset the scan period if the task is being rescheduled on an - * alternative node to recheck if the tasks is now properly placed. - */ - p->numa_scan_period = task_scan_min(p); + if (ng->active_nodes > 1 && numa_is_active_node(env.dst_nid, ng)) + sched_setnuma(p, env.dst_nid); + } - if (env.best_task == NULL) { - ret = migrate_task_to(p, env.best_cpu); - if (ret != 0) - trace_sched_stick_numa(p, env.src_cpu, env.best_cpu); - return ret; - } + /* No better CPU than the current one was found. */ + if (env.best_cpu == -1) + return -EAGAIN; - ret = migrate_swap(p, env.best_task); + /* + * Reset the scan period if the task is being rescheduled on an + * alternative node to recheck if the tasks is now properly placed. + */ + p->numa_scan_period = task_scan_min(p); + + if (env.best_task == NULL) { + ret = migrate_task_to(p, env.best_cpu); if (ret != 0) - trace_sched_stick_numa(p, env.src_cpu, task_cpu(env.best_task)); - put_task_struct(env.best_task); + trace_sched_stick_numa(p, env.src_cpu, env.best_cpu); return ret; + } + + ret = migrate_swap(p, env.best_task); + if (ret != 0) + trace_sched_stick_numa(p, env.src_cpu, task_cpu(env.best_task)); + put_task_struct(env.best_task); + return ret; } /* Attempt to migrate a task to a CPU on the preferred node. */ static void numa_migrate_preferred(struct task_struct *p) { - unsigned long interval = HZ; + unsigned long interval = HZ; - /* This task has no NUMA fault statistics yet */ - if (unlikely(p->numa_preferred_nid == -1 || !p->numa_faults)) - return; + /* This task has no NUMA fault statistics yet */ + if (unlikely(p->numa_preferred_nid == -1 || !p->numa_faults)) + return; - /* Periodically retry migrating the task to the preferred node */ - interval = min(interval, msecs_to_jiffies(p->numa_scan_period) / 16); - p->numa_migrate_retry = jiffies + interval; + /* Periodically retry migrating the task to the preferred node */ + interval = min(interval, msecs_to_jiffies(p->numa_scan_period) / 16); + p->numa_migrate_retry = jiffies + interval; - /* Success if task is already running on preferred CPU */ - if (task_node(p) == p->numa_preferred_nid) - return; + /* Success if task is already running on preferred CPU */ + if (task_node(p) == p->numa_preferred_nid) + return; - /* Otherwise, try migrate to a CPU on the preferred node */ - task_numa_migrate(p); + /* Otherwise, try migrate to a CPU on the preferred node */ + task_numa_migrate(p); } /* @@ -1671,23 +1671,23 @@ static void numa_migrate_preferred(struct task_struct *p) */ static void numa_group_count_active_nodes(struct numa_group *numa_group) { - unsigned long faults, max_faults = 0; - int nid, active_nodes = 0; + unsigned long faults, max_faults = 0; + int nid, active_nodes = 0; - for_each_online_node(nid) { - faults = group_faults_cpu(numa_group, nid); - if (faults > max_faults) - max_faults = faults; - } + for_each_online_node(nid) { + faults = group_faults_cpu(numa_group, nid); + if (faults > max_faults) + max_faults = faults; + } - for_each_online_node(nid) { - faults = group_faults_cpu(numa_group, nid); - if (faults * ACTIVE_NODE_FRACTION > max_faults) - active_nodes++; - } + for_each_online_node(nid) { + faults = group_faults_cpu(numa_group, nid); + if (faults * ACTIVE_NODE_FRACTION > max_faults) + active_nodes++; + } - numa_group->max_faults_cpu = max_faults; - numa_group->active_nodes = active_nodes; + numa_group->max_faults_cpu = max_faults; + numa_group->active_nodes = active_nodes; } /* @@ -1707,63 +1707,63 @@ static void numa_group_count_active_nodes(struct numa_group *numa_group) * Otherwise, decrease the scan period. */ static void update_task_scan_period(struct task_struct *p, - unsigned long shared, unsigned long private) + unsigned long shared, unsigned long private) { - unsigned int period_slot; - int ratio; - int diff; + unsigned int period_slot; + int ratio; + int diff; - unsigned long remote = p->numa_faults_locality[0]; - unsigned long local = p->numa_faults_locality[1]; + unsigned long remote = p->numa_faults_locality[0]; + unsigned long local = p->numa_faults_locality[1]; + + /* + * If there were no record hinting faults then either the task is + * completely idle or all activity is areas that are not of interest + * to automatic numa balancing. Related to that, if there were failed + * migration then it implies we are migrating too quickly or the local + * node is overloaded. In either case, scan slower + */ + if (local + shared == 0 || p->numa_faults_locality[2]) { + p->numa_scan_period = min(p->numa_scan_period_max, + p->numa_scan_period << 1); + + p->mm->numa_next_scan = jiffies + + msecs_to_jiffies(p->numa_scan_period); + + return; + } + + /* + * Prepare to scale scan period relative to the current period. + * == NUMA_PERIOD_THRESHOLD scan period stays the same + * < NUMA_PERIOD_THRESHOLD scan period decreases (scan faster) + * >= NUMA_PERIOD_THRESHOLD scan period increases (scan slower) + */ + period_slot = DIV_ROUND_UP(p->numa_scan_period, NUMA_PERIOD_SLOTS); + ratio = (local * NUMA_PERIOD_SLOTS) / (local + remote); + if (ratio >= NUMA_PERIOD_THRESHOLD) { + int slot = ratio - NUMA_PERIOD_THRESHOLD; + if (!slot) + slot = 1; + diff = slot * period_slot; + } else { + diff = -(NUMA_PERIOD_THRESHOLD - ratio) * period_slot; /* - * If there were no record hinting faults then either the task is - * completely idle or all activity is areas that are not of interest - * to automatic numa balancing. Related to that, if there were failed - * migration then it implies we are migrating too quickly or the local - * node is overloaded. In either case, scan slower + * Scale scan rate increases based on sharing. There is an + * inverse relationship between the degree of sharing and + * the adjustment made to the scanning period. Broadly + * speaking the intent is that there is little point + * scanning faster if shared accesses dominate as it may + * simply bounce migrations uselessly */ - if (local + shared == 0 || p->numa_faults_locality[2]) { - p->numa_scan_period = min(p->numa_scan_period_max, - p->numa_scan_period << 1); + ratio = DIV_ROUND_UP(private * NUMA_PERIOD_SLOTS, (private + shared + 1)); + diff = (diff * ratio) / NUMA_PERIOD_SLOTS; + } - p->mm->numa_next_scan = jiffies + - msecs_to_jiffies(p->numa_scan_period); - - return; - } - - /* - * Prepare to scale scan period relative to the current period. - * == NUMA_PERIOD_THRESHOLD scan period stays the same - * < NUMA_PERIOD_THRESHOLD scan period decreases (scan faster) - * >= NUMA_PERIOD_THRESHOLD scan period increases (scan slower) - */ - period_slot = DIV_ROUND_UP(p->numa_scan_period, NUMA_PERIOD_SLOTS); - ratio = (local * NUMA_PERIOD_SLOTS) / (local + remote); - if (ratio >= NUMA_PERIOD_THRESHOLD) { - int slot = ratio - NUMA_PERIOD_THRESHOLD; - if (!slot) - slot = 1; - diff = slot * period_slot; - } else { - diff = -(NUMA_PERIOD_THRESHOLD - ratio) * period_slot; - - /* - * Scale scan rate increases based on sharing. There is an - * inverse relationship between the degree of sharing and - * the adjustment made to the scanning period. Broadly - * speaking the intent is that there is little point - * scanning faster if shared accesses dominate as it may - * simply bounce migrations uselessly - */ - ratio = DIV_ROUND_UP(private * NUMA_PERIOD_SLOTS, (private + shared + 1)); - diff = (diff * ratio) / NUMA_PERIOD_SLOTS; - } - - p->numa_scan_period = clamp(p->numa_scan_period + diff, - task_scan_min(p), task_scan_max(p)); - memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); + p->numa_scan_period = clamp(p->numa_scan_period + diff, + task_scan_min(p), task_scan_max(p)); + memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); } /* @@ -1775,23 +1775,23 @@ static void update_task_scan_period(struct task_struct *p, */ static u64 numa_get_avg_runtime(struct task_struct *p, u64 *period) { - u64 runtime, delta, now; - /* Use the start of this time slice to avoid calculations. */ - now = p->se.exec_start; - runtime = p->se.sum_exec_runtime; + u64 runtime, delta, now; + /* Use the start of this time slice to avoid calculations. */ + now = p->se.exec_start; + runtime = p->se.sum_exec_runtime; - if (p->last_task_numa_placement) { - delta = runtime - p->last_sum_exec_runtime; - *period = now - p->last_task_numa_placement; - } else { - delta = p->se.avg.load_sum / p->se.load.weight; - *period = LOAD_AVG_MAX; - } + if (p->last_task_numa_placement) { + delta = runtime - p->last_sum_exec_runtime; + *period = now - p->last_task_numa_placement; + } else { + delta = p->se.avg.load_sum / p->se.load.weight; + *period = LOAD_AVG_MAX; + } - p->last_sum_exec_runtime = runtime; - p->last_task_numa_placement = now; + p->last_sum_exec_runtime = runtime; + p->last_task_numa_placement = now; - return delta; + return delta; } /* @@ -1801,337 +1801,337 @@ static u64 numa_get_avg_runtime(struct task_struct *p, u64 *period) */ static int preferred_group_nid(struct task_struct *p, int nid) { - nodemask_t nodes; - int dist; + nodemask_t nodes; + int dist; - /* Direct connections between all NUMA nodes. */ - if (sched_numa_topology_type == NUMA_DIRECT) - return nid; - - /* - * On a system with glueless mesh NUMA topology, group_weight - * scores nodes according to the number of NUMA hinting faults on - * both the node itself, and on nearby nodes. - */ - if (sched_numa_topology_type == NUMA_GLUELESS_MESH) { - unsigned long score, max_score = 0; - int node, max_node = nid; - - dist = sched_max_numa_distance; - - for_each_online_node(node) { - score = group_weight(p, node, dist); - if (score > max_score) { - max_score = score; - max_node = node; - } - } - return max_node; - } - - /* - * Finding the preferred nid in a system with NUMA backplane - * interconnect topology is more involved. The goal is to locate - * tasks from numa_groups near each other in the system, and - * untangle workloads from different sides of the system. This requires - * searching down the hierarchy of node groups, recursively searching - * inside the highest scoring group of nodes. The nodemask tricks - * keep the complexity of the search down. - */ - nodes = node_online_map; - for (dist = sched_max_numa_distance; dist > LOCAL_DISTANCE; dist--) { - unsigned long max_faults = 0; - nodemask_t max_group = NODE_MASK_NONE; - int a, b; - - /* Are there nodes at this distance from each other? */ - if (!find_numa_distance(dist)) - continue; - - for_each_node_mask(a, nodes) { - unsigned long faults = 0; - nodemask_t this_group; - nodes_clear(this_group); - - /* Sum group's NUMA faults; includes a==b case. */ - for_each_node_mask(b, nodes) { - if (node_distance(a, b) < dist) { - faults += group_faults(p, b); - node_set(b, this_group); - node_clear(b, nodes); - } - } - - /* Remember the top group. */ - if (faults > max_faults) { - max_faults = faults; - max_group = this_group; - /* - * subtle: at the smallest distance there is - * just one node left in each "group", the - * winner is the preferred nid. - */ - nid = a; - } - } - /* Next round, evaluate the nodes within max_group. */ - if (!max_faults) - break; - nodes = max_group; - } + /* Direct connections between all NUMA nodes. */ + if (sched_numa_topology_type == NUMA_DIRECT) return nid; + + /* + * On a system with glueless mesh NUMA topology, group_weight + * scores nodes according to the number of NUMA hinting faults on + * both the node itself, and on nearby nodes. + */ + if (sched_numa_topology_type == NUMA_GLUELESS_MESH) { + unsigned long score, max_score = 0; + int node, max_node = nid; + + dist = sched_max_numa_distance; + + for_each_online_node(node) { + score = group_weight(p, node, dist); + if (score > max_score) { + max_score = score; + max_node = node; + } + } + return max_node; + } + + /* + * Finding the preferred nid in a system with NUMA backplane + * interconnect topology is more involved. The goal is to locate + * tasks from numa_groups near each other in the system, and + * untangle workloads from different sides of the system. This requires + * searching down the hierarchy of node groups, recursively searching + * inside the highest scoring group of nodes. The nodemask tricks + * keep the complexity of the search down. + */ + nodes = node_online_map; + for (dist = sched_max_numa_distance; dist > LOCAL_DISTANCE; dist--) { + unsigned long max_faults = 0; + nodemask_t max_group = NODE_MASK_NONE; + int a, b; + + /* Are there nodes at this distance from each other? */ + if (!find_numa_distance(dist)) + continue; + + for_each_node_mask(a, nodes) { + unsigned long faults = 0; + nodemask_t this_group; + nodes_clear(this_group); + + /* Sum group's NUMA faults; includes a==b case. */ + for_each_node_mask(b, nodes) { + if (node_distance(a, b) < dist) { + faults += group_faults(p, b); + node_set(b, this_group); + node_clear(b, nodes); + } + } + + /* Remember the top group. */ + if (faults > max_faults) { + max_faults = faults; + max_group = this_group; + /* + * subtle: at the smallest distance there is + * just one node left in each "group", the + * winner is the preferred nid. + */ + nid = a; + } + } + /* Next round, evaluate the nodes within max_group. */ + if (!max_faults) + break; + nodes = max_group; + } + return nid; } static void task_numa_placement(struct task_struct *p) { - int seq, nid, max_nid = -1, max_group_nid = -1; - unsigned long max_faults = 0, max_group_faults = 0; - unsigned long fault_types[2] = { 0, 0 }; - unsigned long total_faults; - u64 runtime, period; - spinlock_t *group_lock = NULL; + int seq, nid, max_nid = -1, max_group_nid = -1; + unsigned long max_faults = 0, max_group_faults = 0; + unsigned long fault_types[2] = { 0, 0 }; + unsigned long total_faults; + u64 runtime, period; + spinlock_t *group_lock = NULL; - /* - * The p->mm->numa_scan_seq field gets updated without - * exclusive access. Use READ_ONCE() here to ensure - * that the field is read in a single access: - */ - seq = READ_ONCE(p->mm->numa_scan_seq); - if (p->numa_scan_seq == seq) - return; - p->numa_scan_seq = seq; - p->numa_scan_period_max = task_scan_max(p); + /* + * The p->mm->numa_scan_seq field gets updated without + * exclusive access. Use READ_ONCE() here to ensure + * that the field is read in a single access: + */ + seq = READ_ONCE(p->mm->numa_scan_seq); + if (p->numa_scan_seq == seq) + return; + p->numa_scan_seq = seq; + p->numa_scan_period_max = task_scan_max(p); - total_faults = p->numa_faults_locality[0] + - p->numa_faults_locality[1]; - runtime = numa_get_avg_runtime(p, &period); + total_faults = p->numa_faults_locality[0] + + p->numa_faults_locality[1]; + runtime = numa_get_avg_runtime(p, &period); - /* If the task is part of a group prevent parallel updates to group stats */ - if (p->numa_group) { - group_lock = &p->numa_group->lock; - spin_lock_irq(group_lock); + /* If the task is part of a group prevent parallel updates to group stats */ + if (p->numa_group) { + group_lock = &p->numa_group->lock; + spin_lock_irq(group_lock); + } + + /* Find the node with the highest number of faults */ + for_each_online_node(nid) { + /* Keep track of the offsets in numa_faults array */ + int mem_idx, membuf_idx, cpu_idx, cpubuf_idx; + unsigned long faults = 0, group_faults = 0; + int priv; + + for (priv = 0; priv < NR_NUMA_HINT_FAULT_TYPES; priv++) { + long diff, f_diff, f_weight; + + mem_idx = task_faults_idx(NUMA_MEM, nid, priv); + membuf_idx = task_faults_idx(NUMA_MEMBUF, nid, priv); + cpu_idx = task_faults_idx(NUMA_CPU, nid, priv); + cpubuf_idx = task_faults_idx(NUMA_CPUBUF, nid, priv); + + /* Decay existing window, copy faults since last scan */ + diff = p->numa_faults[membuf_idx] - p->numa_faults[mem_idx] / 2; + fault_types[priv] += p->numa_faults[membuf_idx]; + p->numa_faults[membuf_idx] = 0; + + /* + * Normalize the faults_from, so all tasks in a group + * count according to CPU use, instead of by the raw + * number of faults. Tasks with little runtime have + * little over-all impact on throughput, and thus their + * faults are less important. + */ + f_weight = div64_u64(runtime << 16, period + 1); + f_weight = (f_weight * p->numa_faults[cpubuf_idx]) / + (total_faults + 1); + f_diff = f_weight - p->numa_faults[cpu_idx] / 2; + p->numa_faults[cpubuf_idx] = 0; + + p->numa_faults[mem_idx] += diff; + p->numa_faults[cpu_idx] += f_diff; + faults += p->numa_faults[mem_idx]; + p->total_numa_faults += diff; + if (p->numa_group) { + /* + * safe because we can only change our own group + * + * mem_idx represents the offset for a given + * nid and priv in a specific region because it + * is at the beginning of the numa_faults array. + */ + p->numa_group->faults[mem_idx] += diff; + p->numa_group->faults_cpu[mem_idx] += f_diff; + p->numa_group->total_faults += diff; + group_faults += p->numa_group->faults[mem_idx]; + } } - /* Find the node with the highest number of faults */ - for_each_online_node(nid) { - /* Keep track of the offsets in numa_faults array */ - int mem_idx, membuf_idx, cpu_idx, cpubuf_idx; - unsigned long faults = 0, group_faults = 0; - int priv; - - for (priv = 0; priv < NR_NUMA_HINT_FAULT_TYPES; priv++) { - long diff, f_diff, f_weight; - - mem_idx = task_faults_idx(NUMA_MEM, nid, priv); - membuf_idx = task_faults_idx(NUMA_MEMBUF, nid, priv); - cpu_idx = task_faults_idx(NUMA_CPU, nid, priv); - cpubuf_idx = task_faults_idx(NUMA_CPUBUF, nid, priv); - - /* Decay existing window, copy faults since last scan */ - diff = p->numa_faults[membuf_idx] - p->numa_faults[mem_idx] / 2; - fault_types[priv] += p->numa_faults[membuf_idx]; - p->numa_faults[membuf_idx] = 0; - - /* - * Normalize the faults_from, so all tasks in a group - * count according to CPU use, instead of by the raw - * number of faults. Tasks with little runtime have - * little over-all impact on throughput, and thus their - * faults are less important. - */ - f_weight = div64_u64(runtime << 16, period + 1); - f_weight = (f_weight * p->numa_faults[cpubuf_idx]) / - (total_faults + 1); - f_diff = f_weight - p->numa_faults[cpu_idx] / 2; - p->numa_faults[cpubuf_idx] = 0; - - p->numa_faults[mem_idx] += diff; - p->numa_faults[cpu_idx] += f_diff; - faults += p->numa_faults[mem_idx]; - p->total_numa_faults += diff; - if (p->numa_group) { - /* - * safe because we can only change our own group - * - * mem_idx represents the offset for a given - * nid and priv in a specific region because it - * is at the beginning of the numa_faults array. - */ - p->numa_group->faults[mem_idx] += diff; - p->numa_group->faults_cpu[mem_idx] += f_diff; - p->numa_group->total_faults += diff; - group_faults += p->numa_group->faults[mem_idx]; - } - } - - if (faults > max_faults) { - max_faults = faults; - max_nid = nid; - } - - if (group_faults > max_group_faults) { - max_group_faults = group_faults; - max_group_nid = nid; - } + if (faults > max_faults) { + max_faults = faults; + max_nid = nid; } - update_task_scan_period(p, fault_types[0], fault_types[1]); - - if (p->numa_group) { - numa_group_count_active_nodes(p->numa_group); - spin_unlock_irq(group_lock); - max_nid = preferred_group_nid(p, max_group_nid); + if (group_faults > max_group_faults) { + max_group_faults = group_faults; + max_group_nid = nid; } + } - if (max_faults) { - /* Set the new preferred node */ - if (max_nid != p->numa_preferred_nid) - sched_setnuma(p, max_nid); + update_task_scan_period(p, fault_types[0], fault_types[1]); - if (task_node(p) != p->numa_preferred_nid) - numa_migrate_preferred(p); - } + if (p->numa_group) { + numa_group_count_active_nodes(p->numa_group); + spin_unlock_irq(group_lock); + max_nid = preferred_group_nid(p, max_group_nid); + } + + if (max_faults) { + /* Set the new preferred node */ + if (max_nid != p->numa_preferred_nid) + sched_setnuma(p, max_nid); + + if (task_node(p) != p->numa_preferred_nid) + numa_migrate_preferred(p); + } } static inline int get_numa_group(struct numa_group *grp) { - return atomic_inc_not_zero(&grp->refcount); + return atomic_inc_not_zero(&grp->refcount); } static inline void put_numa_group(struct numa_group *grp) { - if (atomic_dec_and_test(&grp->refcount)) - kfree_rcu(grp, rcu); + if (atomic_dec_and_test(&grp->refcount)) + kfree_rcu(grp, rcu); } static void task_numa_group(struct task_struct *p, int cpupid, int flags, - int *priv) + int *priv) { - struct numa_group *grp, *my_grp; - struct task_struct *tsk; - bool join = false; - int cpu = cpupid_to_cpu(cpupid); - int i; + struct numa_group *grp, *my_grp; + struct task_struct *tsk; + bool join = false; + int cpu = cpupid_to_cpu(cpupid); + int i; - if (unlikely(!p->numa_group)) { - unsigned int size = sizeof(struct numa_group) + - 4*nr_node_ids*sizeof(unsigned long); + if (unlikely(!p->numa_group)) { + unsigned int size = sizeof(struct numa_group) + + 4*nr_node_ids*sizeof(unsigned long); - grp = kzalloc(size, GFP_KERNEL | __GFP_NOWARN); - if (!grp) - return; - - atomic_set(&grp->refcount, 1); - grp->active_nodes = 1; - grp->max_faults_cpu = 0; - spin_lock_init(&grp->lock); - grp->gid = p->pid; - /* Second half of the array tracks nids where faults happen */ - grp->faults_cpu = grp->faults + NR_NUMA_HINT_FAULT_TYPES * - nr_node_ids; - - for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) - grp->faults[i] = p->numa_faults[i]; - - grp->total_faults = p->total_numa_faults; - - grp->nr_tasks++; - rcu_assign_pointer(p->numa_group, grp); - } - - rcu_read_lock(); - tsk = READ_ONCE(cpu_rq(cpu)->curr); - - if (!cpupid_match_pid(tsk, cpupid)) - goto no_join; - - grp = rcu_dereference(tsk->numa_group); + grp = kzalloc(size, GFP_KERNEL | __GFP_NOWARN); if (!grp) - goto no_join; + return; - my_grp = p->numa_group; - if (grp == my_grp) - goto no_join; + atomic_set(&grp->refcount, 1); + grp->active_nodes = 1; + grp->max_faults_cpu = 0; + spin_lock_init(&grp->lock); + grp->gid = p->pid; + /* Second half of the array tracks nids where faults happen */ + grp->faults_cpu = grp->faults + NR_NUMA_HINT_FAULT_TYPES * + nr_node_ids; - /* - * Only join the other group if its bigger; if we're the bigger group, - * the other task will join us. - */ - if (my_grp->nr_tasks > grp->nr_tasks) - goto no_join; + for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) + grp->faults[i] = p->numa_faults[i]; - /* - * Tie-break on the grp address. - */ - if (my_grp->nr_tasks == grp->nr_tasks && my_grp > grp) - goto no_join; + grp->total_faults = p->total_numa_faults; - /* Always join threads in the same process. */ - if (tsk->mm == current->mm) - join = true; - - /* Simple filter to avoid false positives due to PID collisions */ - if (flags & TNF_SHARED) - join = true; - - /* Update priv based on whether false sharing was detected */ - *priv = !join; - - if (join && !get_numa_group(grp)) - goto no_join; - - rcu_read_unlock(); - - if (!join) - return; - - BUG_ON(irqs_disabled()); - double_lock_irq(&my_grp->lock, &grp->lock); - - for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) { - my_grp->faults[i] -= p->numa_faults[i]; - grp->faults[i] += p->numa_faults[i]; - } - my_grp->total_faults -= p->total_numa_faults; - grp->total_faults += p->total_numa_faults; - - my_grp->nr_tasks--; grp->nr_tasks++; - - spin_unlock(&my_grp->lock); - spin_unlock_irq(&grp->lock); - rcu_assign_pointer(p->numa_group, grp); + } - put_numa_group(my_grp); + rcu_read_lock(); + tsk = READ_ONCE(cpu_rq(cpu)->curr); + + if (!cpupid_match_pid(tsk, cpupid)) + goto no_join; + + grp = rcu_dereference(tsk->numa_group); + if (!grp) + goto no_join; + + my_grp = p->numa_group; + if (grp == my_grp) + goto no_join; + + /* + * Only join the other group if its bigger; if we're the bigger group, + * the other task will join us. + */ + if (my_grp->nr_tasks > grp->nr_tasks) + goto no_join; + + /* + * Tie-break on the grp address. + */ + if (my_grp->nr_tasks == grp->nr_tasks && my_grp > grp) + goto no_join; + + /* Always join threads in the same process. */ + if (tsk->mm == current->mm) + join = true; + + /* Simple filter to avoid false positives due to PID collisions */ + if (flags & TNF_SHARED) + join = true; + + /* Update priv based on whether false sharing was detected */ + *priv = !join; + + if (join && !get_numa_group(grp)) + goto no_join; + + rcu_read_unlock(); + + if (!join) return; + BUG_ON(irqs_disabled()); + double_lock_irq(&my_grp->lock, &grp->lock); + + for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) { + my_grp->faults[i] -= p->numa_faults[i]; + grp->faults[i] += p->numa_faults[i]; + } + my_grp->total_faults -= p->total_numa_faults; + grp->total_faults += p->total_numa_faults; + + my_grp->nr_tasks--; + grp->nr_tasks++; + + spin_unlock(&my_grp->lock); + spin_unlock_irq(&grp->lock); + + rcu_assign_pointer(p->numa_group, grp); + + put_numa_group(my_grp); + return; + no_join: - rcu_read_unlock(); - return; + rcu_read_unlock(); + return; } void task_numa_free(struct task_struct *p) { - struct numa_group *grp = p->numa_group; - void *numa_faults = p->numa_faults; - unsigned long flags; - int i; + struct numa_group *grp = p->numa_group; + void *numa_faults = p->numa_faults; + unsigned long flags; + int i; - if (grp) { - spin_lock_irqsave(&grp->lock, flags); - for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) - grp->faults[i] -= p->numa_faults[i]; - grp->total_faults -= p->total_numa_faults; + if (grp) { + spin_lock_irqsave(&grp->lock, flags); + for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) + grp->faults[i] -= p->numa_faults[i]; + grp->total_faults -= p->total_numa_faults; - grp->nr_tasks--; - spin_unlock_irqrestore(&grp->lock, flags); - RCU_INIT_POINTER(p->numa_group, NULL); - put_numa_group(grp); - } + grp->nr_tasks--; + spin_unlock_irqrestore(&grp->lock, flags); + RCU_INIT_POINTER(p->numa_group, NULL); + put_numa_group(grp); + } - p->numa_faults = NULL; - kfree(numa_faults); + p->numa_faults = NULL; + kfree(numa_faults); } /* @@ -2139,88 +2139,88 @@ void task_numa_free(struct task_struct *p) */ void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags) { - struct task_struct *p = current; - bool migrated = flags & TNF_MIGRATED; - int cpu_node = task_node(current); - int local = !!(flags & TNF_FAULT_LOCAL); - struct numa_group *ng; - int priv; + struct task_struct *p = current; + bool migrated = flags & TNF_MIGRATED; + int cpu_node = task_node(current); + int local = !!(flags & TNF_FAULT_LOCAL); + struct numa_group *ng; + int priv; - if (!static_branch_likely(&sched_numa_balancing)) - return; + if (!static_branch_likely(&sched_numa_balancing)) + return; - /* for example, ksmd faulting in a user's mm */ - if (!p->mm) - return; + /* for example, ksmd faulting in a user's mm */ + if (!p->mm) + return; - /* Allocate buffer to track faults on a per-node basis */ - if (unlikely(!p->numa_faults)) { - int size = sizeof(*p->numa_faults) * - NR_NUMA_HINT_FAULT_BUCKETS * nr_node_ids; + /* Allocate buffer to track faults on a per-node basis */ + if (unlikely(!p->numa_faults)) { + int size = sizeof(*p->numa_faults) * + NR_NUMA_HINT_FAULT_BUCKETS * nr_node_ids; - p->numa_faults = kzalloc(size, GFP_KERNEL|__GFP_NOWARN); - if (!p->numa_faults) - return; + p->numa_faults = kzalloc(size, GFP_KERNEL|__GFP_NOWARN); + if (!p->numa_faults) + return; - p->total_numa_faults = 0; - memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); - } + p->total_numa_faults = 0; + memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); + } - /* - * First accesses are treated as private, otherwise consider accesses - * to be private if the accessing pid has not changed - */ - if (unlikely(last_cpupid == (-1 & LAST_CPUPID_MASK))) { - priv = 1; - } else { - priv = cpupid_match_pid(p, last_cpupid); - if (!priv && !(flags & TNF_NO_GROUP)) - task_numa_group(p, last_cpupid, flags, &priv); - } + /* + * First accesses are treated as private, otherwise consider accesses + * to be private if the accessing pid has not changed + */ + if (unlikely(last_cpupid == (-1 & LAST_CPUPID_MASK))) { + priv = 1; + } else { + priv = cpupid_match_pid(p, last_cpupid); + if (!priv && !(flags & TNF_NO_GROUP)) + task_numa_group(p, last_cpupid, flags, &priv); + } - /* - * If a workload spans multiple NUMA nodes, a shared fault that - * occurs wholly within the set of nodes that the workload is - * actively using should be counted as local. This allows the - * scan rate to slow down when a workload has settled down. - */ - ng = p->numa_group; - if (!priv && !local && ng && ng->active_nodes > 1 && - numa_is_active_node(cpu_node, ng) && - numa_is_active_node(mem_node, ng)) - local = 1; + /* + * If a workload spans multiple NUMA nodes, a shared fault that + * occurs wholly within the set of nodes that the workload is + * actively using should be counted as local. This allows the + * scan rate to slow down when a workload has settled down. + */ + ng = p->numa_group; + if (!priv && !local && ng && ng->active_nodes > 1 && + numa_is_active_node(cpu_node, ng) && + numa_is_active_node(mem_node, ng)) + local = 1; - task_numa_placement(p); + task_numa_placement(p); - /* - * Retry task to preferred node migration periodically, in case it - * case it previously failed, or the scheduler moved us. - */ - if (time_after(jiffies, p->numa_migrate_retry)) - numa_migrate_preferred(p); + /* + * Retry task to preferred node migration periodically, in case it + * case it previously failed, or the scheduler moved us. + */ + if (time_after(jiffies, p->numa_migrate_retry)) + numa_migrate_preferred(p); - if (migrated) - p->numa_pages_migrated += pages; - if (flags & TNF_MIGRATE_FAIL) - p->numa_faults_locality[2] += pages; + if (migrated) + p->numa_pages_migrated += pages; + if (flags & TNF_MIGRATE_FAIL) + p->numa_faults_locality[2] += pages; - p->numa_faults[task_faults_idx(NUMA_MEMBUF, mem_node, priv)] += pages; - p->numa_faults[task_faults_idx(NUMA_CPUBUF, cpu_node, priv)] += pages; - p->numa_faults_locality[local] += pages; + p->numa_faults[task_faults_idx(NUMA_MEMBUF, mem_node, priv)] += pages; + p->numa_faults[task_faults_idx(NUMA_CPUBUF, cpu_node, priv)] += pages; + p->numa_faults_locality[local] += pages; } static void reset_ptenuma_scan(struct task_struct *p) { - /* - * We only did a read acquisition of the mmap sem, so - * p->mm->numa_scan_seq is written to without exclusive access - * and the update is not guaranteed to be atomic. That's not - * much of an issue though, since this is just used for - * statistical sampling. Use READ_ONCE/WRITE_ONCE, which are not - * expensive, to avoid any form of compiler optimizations: - */ - WRITE_ONCE(p->mm->numa_scan_seq, READ_ONCE(p->mm->numa_scan_seq) + 1); - p->mm->numa_scan_offset = 0; + /* + * We only did a read acquisition of the mmap sem, so + * p->mm->numa_scan_seq is written to without exclusive access + * and the update is not guaranteed to be atomic. That's not + * much of an issue though, since this is just used for + * statistical sampling. Use READ_ONCE/WRITE_ONCE, which are not + * expensive, to avoid any form of compiler optimizations: + */ + WRITE_ONCE(p->mm->numa_scan_seq, READ_ONCE(p->mm->numa_scan_seq) + 1); + p->mm->numa_scan_offset = 0; } /* @@ -2229,143 +2229,143 @@ static void reset_ptenuma_scan(struct task_struct *p) */ void task_numa_work(struct callback_head *work) { - unsigned long migrate, next_scan, now = jiffies; - struct task_struct *p = current; - struct mm_struct *mm = p->mm; - u64 runtime = p->se.sum_exec_runtime; - struct vm_area_struct *vma; - unsigned long start, end; - unsigned long nr_pte_updates = 0; - long pages, virtpages; + unsigned long migrate, next_scan, now = jiffies; + struct task_struct *p = current; + struct mm_struct *mm = p->mm; + u64 runtime = p->se.sum_exec_runtime; + struct vm_area_struct *vma; + unsigned long start, end; + unsigned long nr_pte_updates = 0; + long pages, virtpages; - WARN_ON_ONCE(p != container_of(work, struct task_struct, numa_work)); + WARN_ON_ONCE(p != container_of(work, struct task_struct, numa_work)); - work->next = work; /* protect against double add */ - /* - * Who cares about NUMA placement when they're dying. - * - * NOTE: make sure not to dereference p->mm before this check, - * exit_task_work() happens _after_ exit_mm() so we could be called - * without p->mm even though we still had it when we enqueued this - * work. - */ - if (p->flags & PF_EXITING) - return; + work->next = work; /* protect against double add */ + /* + * Who cares about NUMA placement when they're dying. + * + * NOTE: make sure not to dereference p->mm before this check, + * exit_task_work() happens _after_ exit_mm() so we could be called + * without p->mm even though we still had it when we enqueued this + * work. + */ + if (p->flags & PF_EXITING) + return; - if (!mm->numa_next_scan) { - mm->numa_next_scan = now + - msecs_to_jiffies(sysctl_numa_balancing_scan_delay); + if (!mm->numa_next_scan) { + mm->numa_next_scan = now + + msecs_to_jiffies(sysctl_numa_balancing_scan_delay); + } + + /* + * Enforce maximal scan/migration frequency.. + */ + migrate = mm->numa_next_scan; + if (time_before(now, migrate)) + return; + + if (p->numa_scan_period == 0) { + p->numa_scan_period_max = task_scan_max(p); + p->numa_scan_period = task_scan_min(p); + } + + next_scan = now + msecs_to_jiffies(p->numa_scan_period); + if (cmpxchg(&mm->numa_next_scan, migrate, next_scan) != migrate) + return; + + /* + * Delay this task enough that another task of this mm will likely win + * the next time around. + */ + p->node_stamp += 2 * TICK_NSEC; + + start = mm->numa_scan_offset; + pages = sysctl_numa_balancing_scan_size; + pages <<= 20 - PAGE_SHIFT; /* MB in pages */ + virtpages = pages * 8; /* Scan up to this much virtual space */ + if (!pages) + return; + + + down_read(&mm->mmap_sem); + vma = find_vma(mm, start); + if (!vma) { + reset_ptenuma_scan(p); + start = 0; + vma = mm->mmap; + } + for (; vma; vma = vma->vm_next) { + if (!vma_migratable(vma) || !vma_policy_mof(vma) || + is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) { + continue; } /* - * Enforce maximal scan/migration frequency.. + * Shared library pages mapped by multiple processes are not + * migrated as it is expected they are cache replicated. Avoid + * hinting faults in read-only file-backed mappings or the vdso + * as migrating the pages will be of marginal benefit. */ - migrate = mm->numa_next_scan; - if (time_before(now, migrate)) - return; - - if (p->numa_scan_period == 0) { - p->numa_scan_period_max = task_scan_max(p); - p->numa_scan_period = task_scan_min(p); - } - - next_scan = now + msecs_to_jiffies(p->numa_scan_period); - if (cmpxchg(&mm->numa_next_scan, migrate, next_scan) != migrate) - return; + if (!vma->vm_mm || + (vma->vm_file && (vma->vm_flags & (VM_READ|VM_WRITE)) == (VM_READ))) + continue; /* - * Delay this task enough that another task of this mm will likely win - * the next time around. + * Skip inaccessible VMAs to avoid any confusion between + * PROT_NONE and NUMA hinting ptes */ - p->node_stamp += 2 * TICK_NSEC; + if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))) + continue; - start = mm->numa_scan_offset; - pages = sysctl_numa_balancing_scan_size; - pages <<= 20 - PAGE_SHIFT; /* MB in pages */ - virtpages = pages * 8; /* Scan up to this much virtual space */ - if (!pages) - return; + do { + start = max(start, vma->vm_start); + end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE); + end = min(end, vma->vm_end); + nr_pte_updates = change_prot_numa(vma, start, end); + /* + * Try to scan sysctl_numa_balancing_size worth of + * hpages that have at least one present PTE that + * is not already pte-numa. If the VMA contains + * areas that are unused or already full of prot_numa + * PTEs, scan up to virtpages, to skip through those + * areas faster. + */ + if (nr_pte_updates) + pages -= (end - start) >> PAGE_SHIFT; + virtpages -= (end - start) >> PAGE_SHIFT; - down_read(&mm->mmap_sem); - vma = find_vma(mm, start); - if (!vma) { - reset_ptenuma_scan(p); - start = 0; - vma = mm->mmap; - } - for (; vma; vma = vma->vm_next) { - if (!vma_migratable(vma) || !vma_policy_mof(vma) || - is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) { - continue; - } + start = end; + if (pages <= 0 || virtpages <= 0) + goto out; - /* - * Shared library pages mapped by multiple processes are not - * migrated as it is expected they are cache replicated. Avoid - * hinting faults in read-only file-backed mappings or the vdso - * as migrating the pages will be of marginal benefit. - */ - if (!vma->vm_mm || - (vma->vm_file && (vma->vm_flags & (VM_READ|VM_WRITE)) == (VM_READ))) - continue; - - /* - * Skip inaccessible VMAs to avoid any confusion between - * PROT_NONE and NUMA hinting ptes - */ - if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))) - continue; - - do { - start = max(start, vma->vm_start); - end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE); - end = min(end, vma->vm_end); - nr_pte_updates = change_prot_numa(vma, start, end); - - /* - * Try to scan sysctl_numa_balancing_size worth of - * hpages that have at least one present PTE that - * is not already pte-numa. If the VMA contains - * areas that are unused or already full of prot_numa - * PTEs, scan up to virtpages, to skip through those - * areas faster. - */ - if (nr_pte_updates) - pages -= (end - start) >> PAGE_SHIFT; - virtpages -= (end - start) >> PAGE_SHIFT; - - start = end; - if (pages <= 0 || virtpages <= 0) - goto out; - - cond_resched(); - } while (end != vma->vm_end); - } + cond_resched(); + } while (end != vma->vm_end); + } out: - /* - * It is possible to reach the end of the VMA list but the last few - * VMAs are not guaranteed to the vma_migratable. If they are not, we - * would find the !migratable VMA on the next scan but not reset the - * scanner to the start so check it now. - */ - if (vma) - mm->numa_scan_offset = start; - else - reset_ptenuma_scan(p); - up_read(&mm->mmap_sem); + /* + * It is possible to reach the end of the VMA list but the last few + * VMAs are not guaranteed to the vma_migratable. If they are not, we + * would find the !migratable VMA on the next scan but not reset the + * scanner to the start so check it now. + */ + if (vma) + mm->numa_scan_offset = start; + else + reset_ptenuma_scan(p); + up_read(&mm->mmap_sem); - /* - * Make sure tasks use at least 32x as much time to run other code - * than they used here, to limit NUMA PTE scanning overhead to 3% max. - * Usually update_task_scan_period slows down scanning enough; on an - * overloaded system we need to limit overhead on a per task basis. - */ - if (unlikely(p->se.sum_exec_runtime != runtime)) { - u64 diff = p->se.sum_exec_runtime - runtime; - p->node_stamp += 32 * diff; - } + /* + * Make sure tasks use at least 32x as much time to run other code + * than they used here, to limit NUMA PTE scanning overhead to 3% max. + * Usually update_task_scan_period slows down scanning enough; on an + * overloaded system we need to limit overhead on a per task basis. + */ + if (unlikely(p->se.sum_exec_runtime != runtime)) { + u64 diff = p->se.sum_exec_runtime - runtime; + p->node_stamp += 32 * diff; + } } /* @@ -2373,34 +2373,34 @@ out: */ void task_tick_numa(struct rq *rq, struct task_struct *curr) { - struct callback_head *work = &curr->numa_work; - u64 period, now; + struct callback_head *work = &curr->numa_work; + u64 period, now; - /* - * We don't care about NUMA placement if we don't have memory. - */ - if (!curr->mm || (curr->flags & PF_EXITING) || work->next != work) - return; + /* + * We don't care about NUMA placement if we don't have memory. + */ + if (!curr->mm || (curr->flags & PF_EXITING) || work->next != work) + return; - /* - * Using runtime rather than walltime has the dual advantage that - * we (mostly) drive the selection from busy threads and that the - * task needs to have done some actual work before we bother with - * NUMA placement. - */ - now = curr->se.sum_exec_runtime; - period = (u64)curr->numa_scan_period * NSEC_PER_MSEC; + /* + * Using runtime rather than walltime has the dual advantage that + * we (mostly) drive the selection from busy threads and that the + * task needs to have done some actual work before we bother with + * NUMA placement. + */ + now = curr->se.sum_exec_runtime; + period = (u64)curr->numa_scan_period * NSEC_PER_MSEC; - if (now > curr->node_stamp + period) { - if (!curr->node_stamp) - curr->numa_scan_period = task_scan_min(curr); - curr->node_stamp += period; + if (now > curr->node_stamp + period) { + if (!curr->node_stamp) + curr->numa_scan_period = task_scan_min(curr); + curr->node_stamp += period; - if (!time_before(jiffies, curr->mm->numa_next_scan)) { - init_task_work(work, task_numa_work); /* TODO: move this into sched_fork() */ - task_work_add(curr, work, true); - } + if (!time_before(jiffies, curr->mm->numa_next_scan)) { + init_task_work(work, task_numa_work); /* TODO: move this into sched_fork() */ + task_work_add(curr, work, true); } + } } #else static void task_tick_numa(struct rq *rq, struct task_struct *curr) @@ -2419,110 +2419,110 @@ static inline void account_numa_dequeue(struct rq *rq, struct task_struct *p) static void account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) { - update_load_add(&cfs_rq->load, se->load.weight); - if (!parent_entity(se)) - update_load_add(&rq_of(cfs_rq)->load, se->load.weight); + update_load_add(&cfs_rq->load, se->load.weight); + if (!parent_entity(se)) + update_load_add(&rq_of(cfs_rq)->load, se->load.weight); #ifdef CONFIG_SMP - if (entity_is_task(se)) { - struct rq *rq = rq_of(cfs_rq); + if (entity_is_task(se)) { + struct rq *rq = rq_of(cfs_rq); - account_numa_enqueue(rq, task_of(se)); - list_add(&se->group_node, &rq->cfs_tasks); - } + account_numa_enqueue(rq, task_of(se)); + list_add(&se->group_node, &rq->cfs_tasks); + } #endif - cfs_rq->nr_running++; + cfs_rq->nr_running++; } static void account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) { - update_load_sub(&cfs_rq->load, se->load.weight); - if (!parent_entity(se)) - update_load_sub(&rq_of(cfs_rq)->load, se->load.weight); - if (entity_is_task(se)) { - account_numa_dequeue(rq_of(cfs_rq), task_of(se)); - list_del_init(&se->group_node); - } - cfs_rq->nr_running--; + update_load_sub(&cfs_rq->load, se->load.weight); + if (!parent_entity(se)) + update_load_sub(&rq_of(cfs_rq)->load, se->load.weight); + if (entity_is_task(se)) { + account_numa_dequeue(rq_of(cfs_rq), task_of(se)); + list_del_init(&se->group_node); + } + cfs_rq->nr_running--; } #ifdef CONFIG_FAIR_GROUP_SCHED # ifdef CONFIG_SMP static inline long calc_tg_weight(struct task_group *tg, struct cfs_rq *cfs_rq) { - long tg_weight; + long tg_weight; - /* - * Use this CPU's real-time load instead of the last load contribution - * as the updating of the contribution is delayed, and we will use the - * the real-time load to calc the share. See update_tg_load_avg(). - */ - tg_weight = atomic_long_read(&tg->load_avg); - tg_weight -= cfs_rq->tg_load_avg_contrib; - tg_weight += cfs_rq->load.weight; + /* + * Use this CPU's real-time load instead of the last load contribution + * as the updating of the contribution is delayed, and we will use the + * the real-time load to calc the share. See update_tg_load_avg(). + */ + tg_weight = atomic_long_read(&tg->load_avg); + tg_weight -= cfs_rq->tg_load_avg_contrib; + tg_weight += cfs_rq->load.weight; - return tg_weight; + return tg_weight; } static long calc_cfs_shares(struct cfs_rq *cfs_rq, struct task_group *tg) { - long tg_weight, load, shares; + long tg_weight, load, shares; - tg_weight = calc_tg_weight(tg, cfs_rq); - load = cfs_rq->load.weight; + tg_weight = calc_tg_weight(tg, cfs_rq); + load = cfs_rq->load.weight; - shares = (tg->shares * load); - if (tg_weight) - shares /= tg_weight; + shares = (tg->shares * load); + if (tg_weight) + shares /= tg_weight; - if (shares < MIN_SHARES) - shares = MIN_SHARES; - if (shares > tg->shares) - shares = tg->shares; + if (shares < MIN_SHARES) + shares = MIN_SHARES; + if (shares > tg->shares) + shares = tg->shares; - return shares; + return shares; } # else /* CONFIG_SMP */ static inline long calc_cfs_shares(struct cfs_rq *cfs_rq, struct task_group *tg) { - return tg->shares; + return tg->shares; } # endif /* CONFIG_SMP */ static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, - unsigned long weight) + unsigned long weight) { - if (se->on_rq) { - /* commit outstanding execution time */ - if (cfs_rq->curr == se) - update_curr(cfs_rq); - account_entity_dequeue(cfs_rq, se); - } + if (se->on_rq) { + /* commit outstanding execution time */ + if (cfs_rq->curr == se) + update_curr(cfs_rq); + account_entity_dequeue(cfs_rq, se); + } - update_load_set(&se->load, weight); + update_load_set(&se->load, weight); - if (se->on_rq) - account_entity_enqueue(cfs_rq, se); + if (se->on_rq) + account_entity_enqueue(cfs_rq, se); } static inline int throttled_hierarchy(struct cfs_rq *cfs_rq); static void update_cfs_shares(struct cfs_rq *cfs_rq) { - struct task_group *tg; - struct sched_entity *se; - long shares; + struct task_group *tg; + struct sched_entity *se; + long shares; - tg = cfs_rq->tg; - se = tg->se[cpu_of(rq_of(cfs_rq))]; - if (!se || throttled_hierarchy(cfs_rq)) - return; + tg = cfs_rq->tg; + se = tg->se[cpu_of(rq_of(cfs_rq))]; + if (!se || throttled_hierarchy(cfs_rq)) + return; #ifndef CONFIG_SMP - if (likely(se->load.weight == tg->shares)) - return; + if (likely(se->load.weight == tg->shares)) + return; #endif - shares = calc_cfs_shares(cfs_rq, tg); + shares = calc_cfs_shares(cfs_rq, tg); - reweight_entity(cfs_rq_of(se), se, shares); + reweight_entity(cfs_rq_of(se), se, shares); } #else /* CONFIG_FAIR_GROUP_SCHED */ static inline void update_cfs_shares(struct cfs_rq *cfs_rq) @@ -2533,12 +2533,12 @@ static inline void update_cfs_shares(struct cfs_rq *cfs_rq) #ifdef CONFIG_SMP /* Precomputed fixed inverse multiplies for multiplication by y^n */ static const u32 runnable_avg_yN_inv[] = { - 0xffffffff, 0xfa83b2da, 0xf5257d14, 0xefe4b99a, 0xeac0c6e6, 0xe5b906e6, - 0xe0ccdeeb, 0xdbfbb796, 0xd744fcc9, 0xd2a81d91, 0xce248c14, 0xc9b9bd85, - 0xc5672a10, 0xc12c4cc9, 0xbd08a39e, 0xb8fbaf46, 0xb504f333, 0xb123f581, - 0xad583ee9, 0xa9a15ab4, 0xa5fed6a9, 0xa2704302, 0x9ef5325f, 0x9b8d39b9, - 0x9837f050, 0x94f4efa8, 0x91c3d373, 0x8ea4398a, 0x8b95c1e3, 0x88980e80, - 0x85aac367, 0x82cd8698, + 0xffffffff, 0xfa83b2da, 0xf5257d14, 0xefe4b99a, 0xeac0c6e6, 0xe5b906e6, + 0xe0ccdeeb, 0xdbfbb796, 0xd744fcc9, 0xd2a81d91, 0xce248c14, 0xc9b9bd85, + 0xc5672a10, 0xc12c4cc9, 0xbd08a39e, 0xb8fbaf46, 0xb504f333, 0xb123f581, + 0xad583ee9, 0xa9a15ab4, 0xa5fed6a9, 0xa2704302, 0x9ef5325f, 0x9b8d39b9, + 0x9837f050, 0x94f4efa8, 0x91c3d373, 0x8ea4398a, 0x8b95c1e3, 0x88980e80, + 0x85aac367, 0x82cd8698, }; /* @@ -2546,9 +2546,9 @@ static const u32 runnable_avg_yN_inv[] = { * over-estimates when re-combining. */ static const u32 runnable_avg_yN_sum[] = { - 0, 1002, 1982, 2941, 3880, 4798, 5697, 6576, 7437, 8279, 9103, - 9909,10698,11470,12226,12966,13690,14398,15091,15769,16433,17082, - 17718,18340,18949,19545,20128,20698,21256,21802,22336,22859,23371, + 0, 1002, 1982, 2941, 3880, 4798, 5697, 6576, 7437, 8279, 9103, + 9909,10698,11470,12226,12966,13690,14398,15091,15769,16433,17082, + 17718,18340,18949,19545,20128,20698,21256,21802,22336,22859,23371, }; /* @@ -2557,30 +2557,30 @@ static const u32 runnable_avg_yN_sum[] = { */ static __always_inline u64 decay_load(u64 val, u64 n) { - unsigned int local_n; + unsigned int local_n; - if (!n) - return val; - else if (unlikely(n > LOAD_AVG_PERIOD * 63)) - return 0; - - /* after bounds checking we can collapse to 32-bit */ - local_n = n; - - /* - * As y^PERIOD = 1/2, we can combine - * y^n = 1/2^(n/PERIOD) * y^(n%PERIOD) - * With a look-up table which covers y^n (n= LOAD_AVG_PERIOD)) { - val >>= local_n / LOAD_AVG_PERIOD; - local_n %= LOAD_AVG_PERIOD; - } - - val = mul_u64_u32_shr(val, runnable_avg_yN_inv[local_n], 32); + if (!n) return val; + else if (unlikely(n > LOAD_AVG_PERIOD * 63)) + return 0; + + /* after bounds checking we can collapse to 32-bit */ + local_n = n; + + /* + * As y^PERIOD = 1/2, we can combine + * y^n = 1/2^(n/PERIOD) * y^(n%PERIOD) + * With a look-up table which covers y^n (n= LOAD_AVG_PERIOD)) { + val >>= local_n / LOAD_AVG_PERIOD; + local_n %= LOAD_AVG_PERIOD; + } + + val = mul_u64_u32_shr(val, runnable_avg_yN_inv[local_n], 32); + return val; } /* @@ -2592,23 +2592,23 @@ static __always_inline u64 decay_load(u64 val, u64 n) */ static u32 __compute_runnable_contrib(u64 n) { - u32 contrib = 0; + u32 contrib = 0; - if (likely(n <= LOAD_AVG_PERIOD)) - return runnable_avg_yN_sum[n]; - else if (unlikely(n >= LOAD_AVG_MAX_N)) - return LOAD_AVG_MAX; + if (likely(n <= LOAD_AVG_PERIOD)) + return runnable_avg_yN_sum[n]; + else if (unlikely(n >= LOAD_AVG_MAX_N)) + return LOAD_AVG_MAX; - /* Compute \Sum k^n combining precomputed values for k^i, \Sum k^j */ - do { - contrib /= 2; /* y^LOAD_AVG_PERIOD = 1/2 */ - contrib += runnable_avg_yN_sum[LOAD_AVG_PERIOD]; + /* Compute \Sum k^n combining precomputed values for k^i, \Sum k^j */ + do { + contrib /= 2; /* y^LOAD_AVG_PERIOD = 1/2 */ + contrib += runnable_avg_yN_sum[LOAD_AVG_PERIOD]; - n -= LOAD_AVG_PERIOD; - } while (n > LOAD_AVG_PERIOD); + n -= LOAD_AVG_PERIOD; + } while (n > LOAD_AVG_PERIOD); - contrib = decay_load(contrib, n); - return contrib + runnable_avg_yN_sum[n]; + contrib = decay_load(contrib, n); + return contrib + runnable_avg_yN_sum[n]; } #if (SCHED_LOAD_SHIFT - SCHED_LOAD_RESOLUTION) != 10 || SCHED_CAPACITY_SHIFT != 10 @@ -2624,7 +2624,7 @@ static u32 __compute_runnable_contrib(u64 n) * occurred N-ms ago p_N, with p_0 corresponding to the current period, e.g. * * [<- 1024us ->|<- 1024us ->|<- 1024us ->| ... - * p0 p1 p2 + * p0 p1 p2 * (now) (~1ms ago) (~2ms ago) * * Let u_i denote the fraction of p_i that the entity was runnable. @@ -2643,111 +2643,111 @@ static u32 __compute_runnable_contrib(u64 n) * When a period "rolls over" and we have new u_0`, multiplying the previous * sum again by y is sufficient to update: * load_avg = u_0` + y*(u_0 + u_1*y + u_2*y^2 + ... ) - * = u_0 + u_1*y + u_2*y^2 + ... [re-labeling u_i --> u_{i+1}] + * = u_0 + u_1*y + u_2*y^2 + ... [re-labeling u_i --> u_{i+1}] */ static __always_inline int __update_load_avg(u64 now, int cpu, struct sched_avg *sa, - unsigned long weight, int running, struct cfs_rq *cfs_rq) + unsigned long weight, int running, struct cfs_rq *cfs_rq) { - u64 delta, scaled_delta, periods; - u32 contrib; - unsigned int delta_w, scaled_delta_w, decayed = 0; - unsigned long scale_freq, scale_cpu; + u64 delta, scaled_delta, periods; + u32 contrib; + unsigned int delta_w, scaled_delta_w, decayed = 0; + unsigned long scale_freq, scale_cpu; - delta = now - sa->last_update_time; - /* - * This should only happen when time goes backwards, which it - * unfortunately does during sched clock init when we swap over to TSC. - */ - if ((s64)delta < 0) { - sa->last_update_time = now; - return 0; - } - - /* - * Use 1024ns as the unit of measurement since it's a reasonable - * approximation of 1us and fast to compute. - */ - delta >>= 10; - if (!delta) - return 0; + delta = now - sa->last_update_time; + /* + * This should only happen when time goes backwards, which it + * unfortunately does during sched clock init when we swap over to TSC. + */ + if ((s64)delta < 0) { sa->last_update_time = now; + return 0; + } - scale_freq = arch_scale_freq_capacity(NULL, cpu); - scale_cpu = arch_scale_cpu_capacity(NULL, cpu); + /* + * Use 1024ns as the unit of measurement since it's a reasonable + * approximation of 1us and fast to compute. + */ + delta >>= 10; + if (!delta) + return 0; + sa->last_update_time = now; - /* delta_w is the amount already accumulated against our next period */ - delta_w = sa->period_contrib; - if (delta + delta_w >= 1024) { - decayed = 1; + scale_freq = arch_scale_freq_capacity(NULL, cpu); + scale_cpu = arch_scale_cpu_capacity(NULL, cpu); - /* how much left for next period will start over, we don't know yet */ - sa->period_contrib = 0; + /* delta_w is the amount already accumulated against our next period */ + delta_w = sa->period_contrib; + if (delta + delta_w >= 1024) { + decayed = 1; - /* - * Now that we know we're crossing a period boundary, figure - * out how much from delta we need to complete the current - * period and accrue it. - */ - delta_w = 1024 - delta_w; - scaled_delta_w = cap_scale(delta_w, scale_freq); - if (weight) { - sa->load_sum += weight * scaled_delta_w; - if (cfs_rq) { - cfs_rq->runnable_load_sum += - weight * scaled_delta_w; - } - } - if (running) - sa->util_sum += scaled_delta_w * scale_cpu; + /* how much left for next period will start over, we don't know yet */ + sa->period_contrib = 0; - delta -= delta_w; - - /* Figure out how many additional periods this update spans */ - periods = delta / 1024; - delta %= 1024; - - sa->load_sum = decay_load(sa->load_sum, periods + 1); - if (cfs_rq) { - cfs_rq->runnable_load_sum = - decay_load(cfs_rq->runnable_load_sum, periods + 1); - } - sa->util_sum = decay_load((u64)(sa->util_sum), periods + 1); - - /* Efficiently calculate \sum (1..n_period) 1024*y^i */ - contrib = __compute_runnable_contrib(periods); - contrib = cap_scale(contrib, scale_freq); - if (weight) { - sa->load_sum += weight * contrib; - if (cfs_rq) - cfs_rq->runnable_load_sum += weight * contrib; - } - if (running) - sa->util_sum += contrib * scale_cpu; - } - - /* Remainder of delta accrued against u_0` */ - scaled_delta = cap_scale(delta, scale_freq); + /* + * Now that we know we're crossing a period boundary, figure + * out how much from delta we need to complete the current + * period and accrue it. + */ + delta_w = 1024 - delta_w; + scaled_delta_w = cap_scale(delta_w, scale_freq); if (weight) { - sa->load_sum += weight * scaled_delta; - if (cfs_rq) - cfs_rq->runnable_load_sum += weight * scaled_delta; + sa->load_sum += weight * scaled_delta_w; + if (cfs_rq) { + cfs_rq->runnable_load_sum += + weight * scaled_delta_w; + } } if (running) - sa->util_sum += scaled_delta * scale_cpu; + sa->util_sum += scaled_delta_w * scale_cpu; - sa->period_contrib += delta; + delta -= delta_w; - if (decayed) { - sa->load_avg = div_u64(sa->load_sum, LOAD_AVG_MAX); - if (cfs_rq) { - cfs_rq->runnable_load_avg = - div_u64(cfs_rq->runnable_load_sum, LOAD_AVG_MAX); - } - sa->util_avg = sa->util_sum / LOAD_AVG_MAX; + /* Figure out how many additional periods this update spans */ + periods = delta / 1024; + delta %= 1024; + + sa->load_sum = decay_load(sa->load_sum, periods + 1); + if (cfs_rq) { + cfs_rq->runnable_load_sum = + decay_load(cfs_rq->runnable_load_sum, periods + 1); } + sa->util_sum = decay_load((u64)(sa->util_sum), periods + 1); - return decayed; + /* Efficiently calculate \sum (1..n_period) 1024*y^i */ + contrib = __compute_runnable_contrib(periods); + contrib = cap_scale(contrib, scale_freq); + if (weight) { + sa->load_sum += weight * contrib; + if (cfs_rq) + cfs_rq->runnable_load_sum += weight * contrib; + } + if (running) + sa->util_sum += contrib * scale_cpu; + } + + /* Remainder of delta accrued against u_0` */ + scaled_delta = cap_scale(delta, scale_freq); + if (weight) { + sa->load_sum += weight * scaled_delta; + if (cfs_rq) + cfs_rq->runnable_load_sum += weight * scaled_delta; + } + if (running) + sa->util_sum += scaled_delta * scale_cpu; + + sa->period_contrib += delta; + + if (decayed) { + sa->load_avg = div_u64(sa->load_sum, LOAD_AVG_MAX); + if (cfs_rq) { + cfs_rq->runnable_load_avg = + div_u64(cfs_rq->runnable_load_sum, LOAD_AVG_MAX); + } + sa->util_avg = sa->util_sum / LOAD_AVG_MAX; + } + + return decayed; } #ifdef CONFIG_FAIR_GROUP_SCHED @@ -2757,18 +2757,18 @@ __update_load_avg(u64 now, int cpu, struct sched_avg *sa, */ static inline void update_tg_load_avg(struct cfs_rq *cfs_rq, int force) { - long delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; + long delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; - /* - * No need to update load_avg for root_task_group as it is not used. - */ - if (cfs_rq->tg == &root_task_group) - return; + /* + * No need to update load_avg for root_task_group as it is not used. + */ + if (cfs_rq->tg == &root_task_group) + return; - if (force || abs(delta) > cfs_rq->tg_load_avg_contrib / 64) { - atomic_long_add(delta, &cfs_rq->tg->load_avg); - cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; - } + if (force || abs(delta) > cfs_rq->tg_load_avg_contrib / 64) { + atomic_long_add(delta, &cfs_rq->tg->load_avg); + cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; + } } /* @@ -2777,45 +2777,45 @@ static inline void update_tg_load_avg(struct cfs_rq *cfs_rq, int force) * including the state of rq->lock, should be made. */ void set_task_rq_fair(struct sched_entity *se, - struct cfs_rq *prev, struct cfs_rq *next) + struct cfs_rq *prev, struct cfs_rq *next) { - if (!sched_feat(ATTACH_AGE_LOAD)) - return; + if (!sched_feat(ATTACH_AGE_LOAD)) + return; - /* - * We are supposed to update the task to "current" time, then its up to - * date and ready to go to new CPU/cfs_rq. But we have difficulty in - * getting what current time is, so simply throw away the out-of-date - * time. This will result in the wakee task is less decayed, but giving - * the wakee more load sounds not bad. - */ - if (se->avg.last_update_time && prev) { - u64 p_last_update_time; - u64 n_last_update_time; + /* + * We are supposed to update the task to "current" time, then its up to + * date and ready to go to new CPU/cfs_rq. But we have difficulty in + * getting what current time is, so simply throw away the out-of-date + * time. This will result in the wakee task is less decayed, but giving + * the wakee more load sounds not bad. + */ + if (se->avg.last_update_time && prev) { + u64 p_last_update_time; + u64 n_last_update_time; #ifndef CONFIG_64BIT - u64 p_last_update_time_copy; - u64 n_last_update_time_copy; + u64 p_last_update_time_copy; + u64 n_last_update_time_copy; - do { - p_last_update_time_copy = prev->load_last_update_time_copy; - n_last_update_time_copy = next->load_last_update_time_copy; + do { + p_last_update_time_copy = prev->load_last_update_time_copy; + n_last_update_time_copy = next->load_last_update_time_copy; - smp_rmb(); + smp_rmb(); - p_last_update_time = prev->avg.last_update_time; - n_last_update_time = next->avg.last_update_time; + p_last_update_time = prev->avg.last_update_time; + n_last_update_time = next->avg.last_update_time; - } while (p_last_update_time != p_last_update_time_copy || - n_last_update_time != n_last_update_time_copy); + } while (p_last_update_time != p_last_update_time_copy || + n_last_update_time != n_last_update_time_copy); #else - p_last_update_time = prev->avg.last_update_time; - n_last_update_time = next->avg.last_update_time; + p_last_update_time = prev->avg.last_update_time; + n_last_update_time = next->avg.last_update_time; #endif - __update_load_avg(p_last_update_time, cpu_of(rq_of(prev)), - &se->avg, 0, 0, NULL); - se->avg.last_update_time = n_last_update_time; - } + __update_load_avg(p_last_update_time, cpu_of(rq_of(prev)), + &se->avg, 0, 0, NULL); + se->avg.last_update_time = n_last_update_time; + } } #else /* CONFIG_FAIR_GROUP_SCHED */ static inline void update_tg_load_avg(struct cfs_rq *cfs_rq, int force) {} @@ -2826,172 +2826,172 @@ static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq); /* Group cfs_rq's load_avg is used for task_h_load and update_cfs_share */ static inline int update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq) { - struct sched_avg *sa = &cfs_rq->avg; - int decayed, removed = 0; + struct sched_avg *sa = &cfs_rq->avg; + int decayed, removed = 0; - if (atomic_long_read(&cfs_rq->removed_load_avg)) { - s64 r = atomic_long_xchg(&cfs_rq->removed_load_avg, 0); - sa->load_avg = max_t(long, sa->load_avg - r, 0); - sa->load_sum = max_t(s64, sa->load_sum - r * LOAD_AVG_MAX, 0); - removed = 1; - } + if (atomic_long_read(&cfs_rq->removed_load_avg)) { + s64 r = atomic_long_xchg(&cfs_rq->removed_load_avg, 0); + sa->load_avg = max_t(long, sa->load_avg - r, 0); + sa->load_sum = max_t(s64, sa->load_sum - r * LOAD_AVG_MAX, 0); + removed = 1; + } - if (atomic_long_read(&cfs_rq->removed_util_avg)) { - long r = atomic_long_xchg(&cfs_rq->removed_util_avg, 0); - sa->util_avg = max_t(long, sa->util_avg - r, 0); - sa->util_sum = max_t(s32, sa->util_sum - r * LOAD_AVG_MAX, 0); - } + if (atomic_long_read(&cfs_rq->removed_util_avg)) { + long r = atomic_long_xchg(&cfs_rq->removed_util_avg, 0); + sa->util_avg = max_t(long, sa->util_avg - r, 0); + sa->util_sum = max_t(s32, sa->util_sum - r * LOAD_AVG_MAX, 0); + } - decayed = __update_load_avg(now, cpu_of(rq_of(cfs_rq)), sa, - scale_load_down(cfs_rq->load.weight), cfs_rq->curr != NULL, cfs_rq); + decayed = __update_load_avg(now, cpu_of(rq_of(cfs_rq)), sa, + scale_load_down(cfs_rq->load.weight), cfs_rq->curr != NULL, cfs_rq); #ifndef CONFIG_64BIT - smp_wmb(); - cfs_rq->load_last_update_time_copy = sa->last_update_time; + smp_wmb(); + cfs_rq->load_last_update_time_copy = sa->last_update_time; #endif - return decayed || removed; + return decayed || removed; } /* Update task and its cfs_rq load average */ static inline void update_load_avg(struct sched_entity *se, int update_tg) { - struct cfs_rq *cfs_rq = cfs_rq_of(se); - u64 now = cfs_rq_clock_task(cfs_rq); - struct rq *rq = rq_of(cfs_rq); - int cpu = cpu_of(rq); + struct cfs_rq *cfs_rq = cfs_rq_of(se); + u64 now = cfs_rq_clock_task(cfs_rq); + struct rq *rq = rq_of(cfs_rq); + int cpu = cpu_of(rq); + + /* + * Track task load average for carrying it to new CPU after migrated, and + * track group sched_entity load average for task_h_load calc in migration + */ + __update_load_avg(now, cpu, &se->avg, + se->on_rq * scale_load_down(se->load.weight), + cfs_rq->curr == se, NULL); + + if (update_cfs_rq_load_avg(now, cfs_rq) && update_tg) + update_tg_load_avg(cfs_rq, 0); + + if (cpu == smp_processor_id() && &rq->cfs == cfs_rq) { + unsigned long max = rq->cpu_capacity_orig; /* - * Track task load average for carrying it to new CPU after migrated, and - * track group sched_entity load average for task_h_load calc in migration + * There are a few boundary cases this might miss but it should + * get called often enough that that should (hopefully) not be + * a real problem -- added to that it only calls on the local + * CPU, so if we enqueue remotely we'll miss an update, but + * the next tick/schedule should update. + * + * It will not get called when we go idle, because the idle + * thread is a different class (!fair), nor will the utilization + * number include things like RT tasks. + * + * As is, the util number is not freq-invariant (we'd have to + * implement arch_scale_freq_capacity() for that). + * + * See cpu_util(). */ - __update_load_avg(now, cpu, &se->avg, - se->on_rq * scale_load_down(se->load.weight), - cfs_rq->curr == se, NULL); - - if (update_cfs_rq_load_avg(now, cfs_rq) && update_tg) - update_tg_load_avg(cfs_rq, 0); - - if (cpu == smp_processor_id() && &rq->cfs == cfs_rq) { - unsigned long max = rq->cpu_capacity_orig; - - /* - * There are a few boundary cases this might miss but it should - * get called often enough that that should (hopefully) not be - * a real problem -- added to that it only calls on the local - * CPU, so if we enqueue remotely we'll miss an update, but - * the next tick/schedule should update. - * - * It will not get called when we go idle, because the idle - * thread is a different class (!fair), nor will the utilization - * number include things like RT tasks. - * - * As is, the util number is not freq-invariant (we'd have to - * implement arch_scale_freq_capacity() for that). - * - * See cpu_util(). - */ - cpufreq_update_util(rq_clock(rq), - min(cfs_rq->avg.util_avg, max), max); - } + cpufreq_update_util(rq_clock(rq), + min(cfs_rq->avg.util_avg, max), max); + } } static void attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { - if (!sched_feat(ATTACH_AGE_LOAD)) - goto skip_aging; + if (!sched_feat(ATTACH_AGE_LOAD)) + goto skip_aging; + + /* + * If we got migrated (either between CPUs or between cgroups) we'll + * have aged the average right before clearing @last_update_time. + */ + if (se->avg.last_update_time) { + __update_load_avg(cfs_rq->avg.last_update_time, cpu_of(rq_of(cfs_rq)), + &se->avg, 0, 0, NULL); /* - * If we got migrated (either between CPUs or between cgroups) we'll - * have aged the average right before clearing @last_update_time. + * XXX: we could have just aged the entire load away if we've been + * absent from the fair class for too long. */ - if (se->avg.last_update_time) { - __update_load_avg(cfs_rq->avg.last_update_time, cpu_of(rq_of(cfs_rq)), - &se->avg, 0, 0, NULL); - - /* - * XXX: we could have just aged the entire load away if we've been - * absent from the fair class for too long. - */ - } + } skip_aging: - se->avg.last_update_time = cfs_rq->avg.last_update_time; - cfs_rq->avg.load_avg += se->avg.load_avg; - cfs_rq->avg.load_sum += se->avg.load_sum; - cfs_rq->avg.util_avg += se->avg.util_avg; - cfs_rq->avg.util_sum += se->avg.util_sum; + se->avg.last_update_time = cfs_rq->avg.last_update_time; + cfs_rq->avg.load_avg += se->avg.load_avg; + cfs_rq->avg.load_sum += se->avg.load_sum; + cfs_rq->avg.util_avg += se->avg.util_avg; + cfs_rq->avg.util_sum += se->avg.util_sum; } static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { - __update_load_avg(cfs_rq->avg.last_update_time, cpu_of(rq_of(cfs_rq)), - &se->avg, se->on_rq * scale_load_down(se->load.weight), - cfs_rq->curr == se, NULL); + __update_load_avg(cfs_rq->avg.last_update_time, cpu_of(rq_of(cfs_rq)), + &se->avg, se->on_rq * scale_load_down(se->load.weight), + cfs_rq->curr == se, NULL); - cfs_rq->avg.load_avg = max_t(long, cfs_rq->avg.load_avg - se->avg.load_avg, 0); - cfs_rq->avg.load_sum = max_t(s64, cfs_rq->avg.load_sum - se->avg.load_sum, 0); - cfs_rq->avg.util_avg = max_t(long, cfs_rq->avg.util_avg - se->avg.util_avg, 0); - cfs_rq->avg.util_sum = max_t(s32, cfs_rq->avg.util_sum - se->avg.util_sum, 0); + cfs_rq->avg.load_avg = max_t(long, cfs_rq->avg.load_avg - se->avg.load_avg, 0); + cfs_rq->avg.load_sum = max_t(s64, cfs_rq->avg.load_sum - se->avg.load_sum, 0); + cfs_rq->avg.util_avg = max_t(long, cfs_rq->avg.util_avg - se->avg.util_avg, 0); + cfs_rq->avg.util_sum = max_t(s32, cfs_rq->avg.util_sum - se->avg.util_sum, 0); } /* Add the load generated by se into cfs_rq's load average */ static inline void enqueue_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { - struct sched_avg *sa = &se->avg; - u64 now = cfs_rq_clock_task(cfs_rq); - int migrated, decayed; + struct sched_avg *sa = &se->avg; + u64 now = cfs_rq_clock_task(cfs_rq); + int migrated, decayed; - migrated = !sa->last_update_time; - if (!migrated) { - __update_load_avg(now, cpu_of(rq_of(cfs_rq)), sa, - se->on_rq * scale_load_down(se->load.weight), - cfs_rq->curr == se, NULL); - } + migrated = !sa->last_update_time; + if (!migrated) { + __update_load_avg(now, cpu_of(rq_of(cfs_rq)), sa, + se->on_rq * scale_load_down(se->load.weight), + cfs_rq->curr == se, NULL); + } - decayed = update_cfs_rq_load_avg(now, cfs_rq); + decayed = update_cfs_rq_load_avg(now, cfs_rq); - cfs_rq->runnable_load_avg += sa->load_avg; - cfs_rq->runnable_load_sum += sa->load_sum; + cfs_rq->runnable_load_avg += sa->load_avg; + cfs_rq->runnable_load_sum += sa->load_sum; - if (migrated) - attach_entity_load_avg(cfs_rq, se); + if (migrated) + attach_entity_load_avg(cfs_rq, se); - if (decayed || migrated) - update_tg_load_avg(cfs_rq, 0); + if (decayed || migrated) + update_tg_load_avg(cfs_rq, 0); } /* Remove the runnable load generated by se from cfs_rq's runnable load average */ static inline void dequeue_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { - update_load_avg(se, 1); + update_load_avg(se, 1); - cfs_rq->runnable_load_avg = - max_t(long, cfs_rq->runnable_load_avg - se->avg.load_avg, 0); - cfs_rq->runnable_load_sum = - max_t(s64, cfs_rq->runnable_load_sum - se->avg.load_sum, 0); + cfs_rq->runnable_load_avg = + max_t(long, cfs_rq->runnable_load_avg - se->avg.load_avg, 0); + cfs_rq->runnable_load_sum = + max_t(s64, cfs_rq->runnable_load_sum - se->avg.load_sum, 0); } #ifndef CONFIG_64BIT static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq) { - u64 last_update_time_copy; - u64 last_update_time; + u64 last_update_time_copy; + u64 last_update_time; - do { - last_update_time_copy = cfs_rq->load_last_update_time_copy; - smp_rmb(); - last_update_time = cfs_rq->avg.last_update_time; - } while (last_update_time != last_update_time_copy); + do { + last_update_time_copy = cfs_rq->load_last_update_time_copy; + smp_rmb(); + last_update_time = cfs_rq->avg.last_update_time; + } while (last_update_time != last_update_time_copy); - return last_update_time; + return last_update_time; } #else static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq) { - return cfs_rq->avg.last_update_time; + return cfs_rq->avg.last_update_time; } #endif @@ -3001,31 +3001,31 @@ static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq) */ void remove_entity_load_avg(struct sched_entity *se) { - struct cfs_rq *cfs_rq = cfs_rq_of(se); - u64 last_update_time; + struct cfs_rq *cfs_rq = cfs_rq_of(se); + u64 last_update_time; - /* - * Newly created task or never used group entity should not be removed - * from its (source) cfs_rq - */ - if (se->avg.last_update_time == 0) - return; + /* + * Newly created task or never used group entity should not be removed + * from its (source) cfs_rq + */ + if (se->avg.last_update_time == 0) + return; - last_update_time = cfs_rq_last_update_time(cfs_rq); + last_update_time = cfs_rq_last_update_time(cfs_rq); - __update_load_avg(last_update_time, cpu_of(rq_of(cfs_rq)), &se->avg, 0, 0, NULL); - atomic_long_add(se->avg.load_avg, &cfs_rq->removed_load_avg); - atomic_long_add(se->avg.util_avg, &cfs_rq->removed_util_avg); + __update_load_avg(last_update_time, cpu_of(rq_of(cfs_rq)), &se->avg, 0, 0, NULL); + atomic_long_add(se->avg.load_avg, &cfs_rq->removed_load_avg); + atomic_long_add(se->avg.util_avg, &cfs_rq->removed_util_avg); } static inline unsigned long cfs_rq_runnable_load_avg(struct cfs_rq *cfs_rq) { - return cfs_rq->runnable_load_avg; + return cfs_rq->runnable_load_avg; } static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq) { - return cfs_rq->avg.load_avg; + return cfs_rq->avg.load_avg; } static int idle_balance(struct rq *this_rq); @@ -3034,10 +3034,10 @@ static int idle_balance(struct rq *this_rq); static inline void update_load_avg(struct sched_entity *se, int not_used) { - struct cfs_rq *cfs_rq = cfs_rq_of(se); - struct rq *rq = rq_of(cfs_rq); + struct cfs_rq *cfs_rq = cfs_rq_of(se); + struct rq *rq = rq_of(cfs_rq); - cpufreq_trigger_update(rq_clock(rq)); + cpufreq_trigger_update(rq_clock(rq)); } static inline void @@ -3053,7 +3053,7 @@ detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} static inline int idle_balance(struct rq *rq) { - return 0; + return 0; } #endif /* CONFIG_SMP */ @@ -3061,108 +3061,108 @@ static inline int idle_balance(struct rq *rq) static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se) { #ifdef CONFIG_SCHEDSTATS - struct task_struct *tsk = NULL; + struct task_struct *tsk = NULL; - if (entity_is_task(se)) - tsk = task_of(se); + if (entity_is_task(se)) + tsk = task_of(se); - if (se->statistics.sleep_start) { - u64 delta = rq_clock(rq_of(cfs_rq)) - se->statistics.sleep_start; + if (se->statistics.sleep_start) { + u64 delta = rq_clock(rq_of(cfs_rq)) - se->statistics.sleep_start; - if ((s64)delta < 0) - delta = 0; + if ((s64)delta < 0) + delta = 0; - if (unlikely(delta > se->statistics.sleep_max)) - se->statistics.sleep_max = delta; + if (unlikely(delta > se->statistics.sleep_max)) + se->statistics.sleep_max = delta; - se->statistics.sleep_start = 0; - se->statistics.sum_sleep_runtime += delta; + se->statistics.sleep_start = 0; + se->statistics.sum_sleep_runtime += delta; - if (tsk) { - account_scheduler_latency(tsk, delta >> 10, 1); - trace_sched_stat_sleep(tsk, delta); - } + if (tsk) { + account_scheduler_latency(tsk, delta >> 10, 1); + trace_sched_stat_sleep(tsk, delta); } - if (se->statistics.block_start) { - u64 delta = rq_clock(rq_of(cfs_rq)) - se->statistics.block_start; + } + if (se->statistics.block_start) { + u64 delta = rq_clock(rq_of(cfs_rq)) - se->statistics.block_start; - if ((s64)delta < 0) - delta = 0; + if ((s64)delta < 0) + delta = 0; - if (unlikely(delta > se->statistics.block_max)) - se->statistics.block_max = delta; + if (unlikely(delta > se->statistics.block_max)) + se->statistics.block_max = delta; - se->statistics.block_start = 0; - se->statistics.sum_sleep_runtime += delta; + se->statistics.block_start = 0; + se->statistics.sum_sleep_runtime += delta; - if (tsk) { - if (tsk->in_iowait) { - se->statistics.iowait_sum += delta; - se->statistics.iowait_count++; - trace_sched_stat_iowait(tsk, delta); - } + if (tsk) { + if (tsk->in_iowait) { + se->statistics.iowait_sum += delta; + se->statistics.iowait_count++; + trace_sched_stat_iowait(tsk, delta); + } - trace_sched_stat_blocked(tsk, delta); + trace_sched_stat_blocked(tsk, delta); - /* - * Blocking time is in units of nanosecs, so shift by - * 20 to get a milliseconds-range estimation of the - * amount of time that the task spent sleeping: - */ - if (unlikely(prof_on == SLEEP_PROFILING)) { - profile_hits(SLEEP_PROFILING, - (void *)get_wchan(tsk), - delta >> 20); - } - account_scheduler_latency(tsk, delta >> 10, 0); - } + /* + * Blocking time is in units of nanosecs, so shift by + * 20 to get a milliseconds-range estimation of the + * amount of time that the task spent sleeping: + */ + if (unlikely(prof_on == SLEEP_PROFILING)) { + profile_hits(SLEEP_PROFILING, + (void *)get_wchan(tsk), + delta >> 20); + } + account_scheduler_latency(tsk, delta >> 10, 0); } + } #endif } static void check_spread(struct cfs_rq *cfs_rq, struct sched_entity *se) { #ifdef CONFIG_SCHED_DEBUG - s64 d = se->vruntime - cfs_rq->min_vruntime; + s64 d = se->vruntime - cfs_rq->min_vruntime; - if (d < 0) - d = -d; + if (d < 0) + d = -d; - if (d > 3*sysctl_sched_latency) - schedstat_inc(cfs_rq, nr_spread_over); + if (d > 3*sysctl_sched_latency) + schedstat_inc(cfs_rq, nr_spread_over); #endif } static void place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial) { - u64 vruntime = cfs_rq->min_vruntime; + u64 vruntime = cfs_rq->min_vruntime; + + /* + * The 'current' period is already promised to the current tasks, + * however the extra weight of the new task will slow them down a + * little, place the new task so that it fits in the slot that + * stays open at the end. + */ + if (initial && sched_feat(START_DEBIT)) + vruntime += sched_vslice(cfs_rq, se); + + /* sleeps up to a single latency don't count. */ + if (!initial) { + unsigned long thresh = sysctl_sched_latency; /* - * The 'current' period is already promised to the current tasks, - * however the extra weight of the new task will slow them down a - * little, place the new task so that it fits in the slot that - * stays open at the end. + * Halve their sleep time's effect, to allow + * for a gentler effect of sleepers: */ - if (initial && sched_feat(START_DEBIT)) - vruntime += sched_vslice(cfs_rq, se); + if (sched_feat(GENTLE_FAIR_SLEEPERS)) + thresh >>= 1; - /* sleeps up to a single latency don't count. */ - if (!initial) { - unsigned long thresh = sysctl_sched_latency; + vruntime -= thresh; + } - /* - * Halve their sleep time's effect, to allow - * for a gentler effect of sleepers: - */ - if (sched_feat(GENTLE_FAIR_SLEEPERS)) - thresh >>= 1; - - vruntime -= thresh; - } - - /* ensure we never gain time by being placed backwards. */ - se->vruntime = max_vruntime(se->vruntime, vruntime); + /* ensure we never gain time by being placed backwards. */ + se->vruntime = max_vruntime(se->vruntime, vruntime); } static void check_enqueue_throttle(struct cfs_rq *cfs_rq); @@ -3170,105 +3170,110 @@ static void check_enqueue_throttle(struct cfs_rq *cfs_rq); static inline void check_schedstat_required(void) { #ifdef CONFIG_SCHEDSTATS - if (schedstat_enabled()) - return; + if (schedstat_enabled()) + return; - /* Force schedstat enabled if a dependent tracepoint is active */ - if (trace_sched_stat_wait_enabled() || - trace_sched_stat_sleep_enabled() || - trace_sched_stat_iowait_enabled() || - trace_sched_stat_blocked_enabled() || - trace_sched_stat_runtime_enabled()) { - pr_warn_once("Scheduler tracepoints stat_sleep, stat_iowait, " - "stat_blocked and stat_runtime require the " - "kernel parameter schedstats=enabled or " - "kernel.sched_schedstats=1\n"); - } + /* Force schedstat enabled if a dependent tracepoint is active */ + if (trace_sched_stat_wait_enabled() || + trace_sched_stat_sleep_enabled() || + trace_sched_stat_iowait_enabled() || + trace_sched_stat_blocked_enabled() || + trace_sched_stat_runtime_enabled()) { + pr_warn_once("Scheduler tracepoints stat_sleep, stat_iowait, " + "stat_blocked and stat_runtime require the " + "kernel parameter schedstats=enabled or " + "kernel.sched_schedstats=1\n"); + } #endif } static void enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) { - /* - * Update the normalized vruntime before updating min_vruntime - * through calling update_curr(). - */ - if (!(flags & ENQUEUE_WAKEUP) || (flags & ENQUEUE_WAKING)) - se->vruntime += cfs_rq->min_vruntime; + /* + * Update the normalized vruntime before updating min_vruntime + * through calling update_curr(). + * + * 如果当前进程之前已经是可运行状态不是被唤醒的那么其虚拟运行时间要增加 + */ + if (!(flags & ENQUEUE_WAKEUP) || (flags & ENQUEUE_WAKING)) + se->vruntime += cfs_rq->min_vruntime; - /* - * Update run-time statistics of the 'current'. - */ - update_curr(cfs_rq); - enqueue_entity_load_avg(cfs_rq, se); - account_entity_enqueue(cfs_rq, se); - update_cfs_shares(cfs_rq); + /* + * Update run-time statistics of the 'current'. + */ + update_curr(cfs_rq); + enqueue_entity_load_avg(cfs_rq, se); + account_entity_enqueue(cfs_rq, se); + update_cfs_shares(cfs_rq); - if (flags & ENQUEUE_WAKEUP) { - place_entity(cfs_rq, se, 0); - if (schedstat_enabled()) - enqueue_sleeper(cfs_rq, se); - } + /* 如果当前进行之前在睡眠刚被唤醒 */ + if (flags & ENQUEUE_WAKEUP) + { + /* 调整进程的虚拟运行时间 */ + place_entity(cfs_rq, se, 0); + if (schedstat_enabled()) + enqueue_sleeper(cfs_rq, se); + } - check_schedstat_required(); - if (schedstat_enabled()) { - update_stats_enqueue(cfs_rq, se); - check_spread(cfs_rq, se); - } - if (se != cfs_rq->curr) - __enqueue_entity(cfs_rq, se); - se->on_rq = 1; + check_schedstat_required(); + if (schedstat_enabled()) { + update_stats_enqueue(cfs_rq, se); + check_spread(cfs_rq, se); + } + if (se != cfs_rq->curr) + __enqueue_entity(cfs_rq, se); + se->on_rq = 1; - if (cfs_rq->nr_running == 1) { - list_add_leaf_cfs_rq(cfs_rq); - check_enqueue_throttle(cfs_rq); - } + if (cfs_rq->nr_running == 1) { + list_add_leaf_cfs_rq(cfs_rq); + check_enqueue_throttle(cfs_rq); + } } static void __clear_buddies_last(struct sched_entity *se) { - for_each_sched_entity(se) { - struct cfs_rq *cfs_rq = cfs_rq_of(se); - if (cfs_rq->last != se) - break; + for_each_sched_entity(se) { + struct cfs_rq *cfs_rq = cfs_rq_of(se); + if (cfs_rq->last != se) + break; - cfs_rq->last = NULL; - } + cfs_rq->last = NULL; + } } static void __clear_buddies_next(struct sched_entity *se) { - for_each_sched_entity(se) { - struct cfs_rq *cfs_rq = cfs_rq_of(se); - if (cfs_rq->next != se) - break; + for_each_sched_entity(se) { + struct cfs_rq *cfs_rq = cfs_rq_of(se); + if (cfs_rq->next != se) + break; - cfs_rq->next = NULL; - } + cfs_rq->next = NULL; + } } static void __clear_buddies_skip(struct sched_entity *se) { - for_each_sched_entity(se) { - struct cfs_rq *cfs_rq = cfs_rq_of(se); - if (cfs_rq->skip != se) - break; + for_each_sched_entity(se) { + struct cfs_rq *cfs_rq = cfs_rq_of(se); + if (cfs_rq->skip != se) + break; - cfs_rq->skip = NULL; - } + cfs_rq->skip = NULL; + } } static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) { - if (cfs_rq->last == se) - __clear_buddies_last(se); + if (cfs_rq->last == se) + __clear_buddies_last(se); - if (cfs_rq->next == se) - __clear_buddies_next(se); + if (cfs_rq->next == se) + __clear_buddies_next(se); - if (cfs_rq->skip == se) - __clear_buddies_skip(se); + if (cfs_rq->skip == se) + __clear_buddies_skip(se); } static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq); @@ -3276,35 +3281,35 @@ static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq); static void dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) { - /* - * Update run-time statistics of the 'current'. - */ - update_curr(cfs_rq); - dequeue_entity_load_avg(cfs_rq, se); + /* + * Update run-time statistics of the 'current'. + */ + update_curr(cfs_rq); + dequeue_entity_load_avg(cfs_rq, se); - if (schedstat_enabled()) - update_stats_dequeue(cfs_rq, se, flags); + if (schedstat_enabled()) + update_stats_dequeue(cfs_rq, se, flags); - clear_buddies(cfs_rq, se); + clear_buddies(cfs_rq, se); - if (se != cfs_rq->curr) - __dequeue_entity(cfs_rq, se); - se->on_rq = 0; - account_entity_dequeue(cfs_rq, se); + if (se != cfs_rq->curr) + __dequeue_entity(cfs_rq, se); + se->on_rq = 0; + account_entity_dequeue(cfs_rq, se); - /* - * Normalize the entity after updating the min_vruntime because the - * update can refer to the ->curr item and we need to reflect this - * movement in our normalized position. - */ - if (!(flags & DEQUEUE_SLEEP)) - se->vruntime -= cfs_rq->min_vruntime; + /* + * Normalize the entity after updating the min_vruntime because the + * update can refer to the ->curr item and we need to reflect this + * movement in our normalized position. + */ + if (!(flags & DEQUEUE_SLEEP)) + se->vruntime -= cfs_rq->min_vruntime; - /* return excess runtime on last dequeue */ - return_cfs_rq_runtime(cfs_rq); + /* return excess runtime on last dequeue */ + return_cfs_rq_runtime(cfs_rq); - update_min_vruntime(cfs_rq); - update_cfs_shares(cfs_rq); + update_min_vruntime(cfs_rq); + update_cfs_shares(cfs_rq); } /* @@ -3313,70 +3318,70 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) static void check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr) { - unsigned long ideal_runtime, delta_exec; - struct sched_entity *se; - s64 delta; - - ideal_runtime = sched_slice(cfs_rq, curr); - delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime; - if (delta_exec > ideal_runtime) { - resched_curr(rq_of(cfs_rq)); - /* - * The current task ran long enough, ensure it doesn't get - * re-elected due to buddy favours. - */ - clear_buddies(cfs_rq, curr); - return; - } + unsigned long ideal_runtime, delta_exec; + struct sched_entity *se; + s64 delta; + ideal_runtime = sched_slice(cfs_rq, curr); + delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime; + if (delta_exec > ideal_runtime) { + resched_curr(rq_of(cfs_rq)); /* - * Ensure that a task that missed wakeup preemption by a - * narrow margin doesn't have to wait for a full slice. - * This also mitigates buddy induced latencies under load. + * The current task ran long enough, ensure it doesn't get + * re-elected due to buddy favours. */ - if (delta_exec < sysctl_sched_min_granularity) - return; + clear_buddies(cfs_rq, curr); + return; + } - se = __pick_first_entity(cfs_rq); - delta = curr->vruntime - se->vruntime; + /* + * Ensure that a task that missed wakeup preemption by a + * narrow margin doesn't have to wait for a full slice. + * This also mitigates buddy induced latencies under load. + */ + if (delta_exec < sysctl_sched_min_granularity) + return; - if (delta < 0) - return; + se = __pick_first_entity(cfs_rq); + delta = curr->vruntime - se->vruntime; - if (delta > ideal_runtime) - resched_curr(rq_of(cfs_rq)); + if (delta < 0) + return; + + if (delta > ideal_runtime) + resched_curr(rq_of(cfs_rq)); } static void set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { - /* 'current' is not kept within the tree. */ - if (se->on_rq) { - /* - * Any task has to be enqueued before it get to execute on - * a CPU. So account for the time it spent waiting on the - * runqueue. - */ - if (schedstat_enabled()) - update_stats_wait_end(cfs_rq, se); - __dequeue_entity(cfs_rq, se); - update_load_avg(se, 1); - } - - update_stats_curr_start(cfs_rq, se); - cfs_rq->curr = se; -#ifdef CONFIG_SCHEDSTATS + /* 'current' is not kept within the tree. */ + if (se->on_rq) { /* - * Track our maximum slice length, if the CPU's load is at - * least twice that of our own weight (i.e. dont track it - * when there are only lesser-weight tasks around): + * Any task has to be enqueued before it get to execute on + * a CPU. So account for the time it spent waiting on the + * runqueue. */ - if (schedstat_enabled() && rq_of(cfs_rq)->load.weight >= 2*se->load.weight) { - se->statistics.slice_max = max(se->statistics.slice_max, - se->sum_exec_runtime - se->prev_sum_exec_runtime); - } + if (schedstat_enabled()) + update_stats_wait_end(cfs_rq, se); + __dequeue_entity(cfs_rq, se); + update_load_avg(se, 1); + } + + update_stats_curr_start(cfs_rq, se); + cfs_rq->curr = se; +#ifdef CONFIG_SCHEDSTATS + /* + * Track our maximum slice length, if the CPU's load is at + * least twice that of our own weight (i.e. dont track it + * when there are only lesser-weight tasks around): + */ + if (schedstat_enabled() && rq_of(cfs_rq)->load.weight >= 2*se->load.weight) { + se->statistics.slice_max = max(se->statistics.slice_max, + se->sum_exec_runtime - se->prev_sum_exec_runtime); + } #endif - se->prev_sum_exec_runtime = se->sum_exec_runtime; + se->prev_sum_exec_runtime = se->sum_exec_runtime; } static int @@ -3392,116 +3397,116 @@ wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se); static struct sched_entity * pick_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *curr) { - struct sched_entity *left = __pick_first_entity(cfs_rq); - struct sched_entity *se; + struct sched_entity *left = __pick_first_entity(cfs_rq); + struct sched_entity *se; - /* - * If curr is set we have to see if its left of the leftmost entity - * still in the tree, provided there was anything in the tree at all. - */ - if (!left || (curr && entity_before(curr, left))) - left = curr; + /* + * If curr is set we have to see if its left of the leftmost entity + * still in the tree, provided there was anything in the tree at all. + */ + if (!left || (curr && entity_before(curr, left))) + left = curr; - se = left; /* ideally we run the leftmost entity */ + se = left; /* ideally we run the leftmost entity */ - /* - * Avoid running the skip buddy, if running something else can - * be done without getting too unfair. - */ - if (cfs_rq->skip == se) { - struct sched_entity *second; + /* + * Avoid running the skip buddy, if running something else can + * be done without getting too unfair. + */ + if (cfs_rq->skip == se) { + struct sched_entity *second; - if (se == curr) { - second = __pick_first_entity(cfs_rq); - } else { - second = __pick_next_entity(se); - if (!second || (curr && entity_before(curr, second))) - second = curr; - } - - if (second && wakeup_preempt_entity(second, left) < 1) - se = second; + if (se == curr) { + second = __pick_first_entity(cfs_rq); + } else { + second = __pick_next_entity(se); + if (!second || (curr && entity_before(curr, second))) + second = curr; } - /* - * Prefer last buddy, try to return the CPU to a preempted task. - */ - if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, left) < 1) - se = cfs_rq->last; + if (second && wakeup_preempt_entity(second, left) < 1) + se = second; + } - /* - * Someone really wants this to run. If it's not unfair, run it. - */ - if (cfs_rq->next && wakeup_preempt_entity(cfs_rq->next, left) < 1) - se = cfs_rq->next; + /* + * Prefer last buddy, try to return the CPU to a preempted task. + */ + if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, left) < 1) + se = cfs_rq->last; - clear_buddies(cfs_rq, se); + /* + * Someone really wants this to run. If it's not unfair, run it. + */ + if (cfs_rq->next && wakeup_preempt_entity(cfs_rq->next, left) < 1) + se = cfs_rq->next; - return se; + clear_buddies(cfs_rq, se); + + return se; } static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq); static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev) { - /* - * If still on the runqueue then deactivate_task() - * was not called and update_curr() has to be done: - */ + /* + * If still on the runqueue then deactivate_task() + * was not called and update_curr() has to be done: + */ + if (prev->on_rq) + update_curr(cfs_rq); + + /* throttle cfs_rqs exceeding runtime */ + check_cfs_rq_runtime(cfs_rq); + + if (schedstat_enabled()) { + check_spread(cfs_rq, prev); if (prev->on_rq) - update_curr(cfs_rq); + update_stats_wait_start(cfs_rq, prev); + } - /* throttle cfs_rqs exceeding runtime */ - check_cfs_rq_runtime(cfs_rq); - - if (schedstat_enabled()) { - check_spread(cfs_rq, prev); - if (prev->on_rq) - update_stats_wait_start(cfs_rq, prev); - } - - if (prev->on_rq) { - /* Put 'current' back into the tree. */ - __enqueue_entity(cfs_rq, prev); - /* in !on_rq case, update occurred at dequeue */ - update_load_avg(prev, 0); - } - cfs_rq->curr = NULL; + if (prev->on_rq) { + /* Put 'current' back into the tree. */ + __enqueue_entity(cfs_rq, prev); + /* in !on_rq case, update occurred at dequeue */ + update_load_avg(prev, 0); + } + cfs_rq->curr = NULL; } static void entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued) { - /* - * Update run-time statistics of the 'current'. - */ - update_curr(cfs_rq); + /* + * Update run-time statistics of the 'current'. + */ + update_curr(cfs_rq); - /* - * Ensure that runnable average is periodically updated. - */ - update_load_avg(curr, 1); - update_cfs_shares(cfs_rq); + /* + * Ensure that runnable average is periodically updated. + */ + update_load_avg(curr, 1); + update_cfs_shares(cfs_rq); #ifdef CONFIG_SCHED_HRTICK - /* - * queued ticks are scheduled to match the slice, so don't bother - * validating it and just reschedule. - */ - if (queued) { - resched_curr(rq_of(cfs_rq)); - return; - } - /* - * don't let the period tick interfere with the hrtick preemption - */ - if (!sched_feat(DOUBLE_TICK) && - hrtimer_active(&rq_of(cfs_rq)->hrtick_timer)) - return; + /* + * queued ticks are scheduled to match the slice, so don't bother + * validating it and just reschedule. + */ + if (queued) { + resched_curr(rq_of(cfs_rq)); + return; + } + /* + * don't let the period tick interfere with the hrtick preemption + */ + if (!sched_feat(DOUBLE_TICK) && + hrtimer_active(&rq_of(cfs_rq)->hrtick_timer)) + return; #endif - if (cfs_rq->nr_running > 1) - check_preempt_tick(cfs_rq, curr); + if (cfs_rq->nr_running > 1) + check_preempt_tick(cfs_rq, curr); } @@ -3516,22 +3521,22 @@ static struct static_key __cfs_bandwidth_used; static inline bool cfs_bandwidth_used(void) { - return static_key_false(&__cfs_bandwidth_used); + return static_key_false(&__cfs_bandwidth_used); } void cfs_bandwidth_usage_inc(void) { - static_key_slow_inc(&__cfs_bandwidth_used); + static_key_slow_inc(&__cfs_bandwidth_used); } void cfs_bandwidth_usage_dec(void) { - static_key_slow_dec(&__cfs_bandwidth_used); + static_key_slow_dec(&__cfs_bandwidth_used); } #else /* HAVE_JUMP_LABEL */ static bool cfs_bandwidth_used(void) { - return true; + return true; } void cfs_bandwidth_usage_inc(void) {} @@ -3544,12 +3549,12 @@ void cfs_bandwidth_usage_dec(void) {} */ static inline u64 default_cfs_period(void) { - return 100000000ULL; + return 100000000ULL; } static inline u64 sched_cfs_bandwidth_slice(void) { - return (u64)sysctl_sched_cfs_bandwidth_slice * NSEC_PER_USEC; + return (u64)sysctl_sched_cfs_bandwidth_slice * NSEC_PER_USEC; } /* @@ -3561,65 +3566,65 @@ static inline u64 sched_cfs_bandwidth_slice(void) */ void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b) { - u64 now; + u64 now; - if (cfs_b->quota == RUNTIME_INF) - return; + if (cfs_b->quota == RUNTIME_INF) + return; - now = sched_clock_cpu(smp_processor_id()); - cfs_b->runtime = cfs_b->quota; - cfs_b->runtime_expires = now + ktime_to_ns(cfs_b->period); + now = sched_clock_cpu(smp_processor_id()); + cfs_b->runtime = cfs_b->quota; + cfs_b->runtime_expires = now + ktime_to_ns(cfs_b->period); } static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) { - return &tg->cfs_bandwidth; + return &tg->cfs_bandwidth; } /* rq->task_clock normalized against any time this cfs_rq has spent throttled */ static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq) { - if (unlikely(cfs_rq->throttle_count)) - return cfs_rq->throttled_clock_task; + if (unlikely(cfs_rq->throttle_count)) + return cfs_rq->throttled_clock_task; - return rq_clock_task(rq_of(cfs_rq)) - cfs_rq->throttled_clock_task_time; + return rq_clock_task(rq_of(cfs_rq)) - cfs_rq->throttled_clock_task_time; } /* returns 0 on failure to allocate runtime */ static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq) { - struct task_group *tg = cfs_rq->tg; - struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg); - u64 amount = 0, min_amount, expires; + struct task_group *tg = cfs_rq->tg; + struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg); + u64 amount = 0, min_amount, expires; - /* note: this is a positive sum as runtime_remaining <= 0 */ - min_amount = sched_cfs_bandwidth_slice() - cfs_rq->runtime_remaining; + /* note: this is a positive sum as runtime_remaining <= 0 */ + min_amount = sched_cfs_bandwidth_slice() - cfs_rq->runtime_remaining; - raw_spin_lock(&cfs_b->lock); - if (cfs_b->quota == RUNTIME_INF) - amount = min_amount; - else { - start_cfs_bandwidth(cfs_b); + raw_spin_lock(&cfs_b->lock); + if (cfs_b->quota == RUNTIME_INF) + amount = min_amount; + else { + start_cfs_bandwidth(cfs_b); - if (cfs_b->runtime > 0) { - amount = min(cfs_b->runtime, min_amount); - cfs_b->runtime -= amount; - cfs_b->idle = 0; - } + if (cfs_b->runtime > 0) { + amount = min(cfs_b->runtime, min_amount); + cfs_b->runtime -= amount; + cfs_b->idle = 0; } - expires = cfs_b->runtime_expires; - raw_spin_unlock(&cfs_b->lock); + } + expires = cfs_b->runtime_expires; + raw_spin_unlock(&cfs_b->lock); - cfs_rq->runtime_remaining += amount; - /* - * we may have advanced our local expiration to account for allowed - * spread between our sched_clock and the one on which runtime was - * issued. - */ - if ((s64)(expires - cfs_rq->runtime_expires) > 0) - cfs_rq->runtime_expires = expires; + cfs_rq->runtime_remaining += amount; + /* + * we may have advanced our local expiration to account for allowed + * spread between our sched_clock and the one on which runtime was + * issued. + */ + if ((s64)(expires - cfs_rq->runtime_expires) > 0) + cfs_rq->runtime_expires = expires; - return cfs_rq->runtime_remaining > 0; + return cfs_rq->runtime_remaining > 0; } /* @@ -3628,70 +3633,70 @@ static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq) */ static void expire_cfs_rq_runtime(struct cfs_rq *cfs_rq) { - struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); + struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); - /* if the deadline is ahead of our clock, nothing to do */ - if (likely((s64)(rq_clock(rq_of(cfs_rq)) - cfs_rq->runtime_expires) < 0)) - return; + /* if the deadline is ahead of our clock, nothing to do */ + if (likely((s64)(rq_clock(rq_of(cfs_rq)) - cfs_rq->runtime_expires) < 0)) + return; - if (cfs_rq->runtime_remaining < 0) - return; + if (cfs_rq->runtime_remaining < 0) + return; - /* - * If the local deadline has passed we have to consider the - * possibility that our sched_clock is 'fast' and the global deadline - * has not truly expired. - * - * Fortunately we can check determine whether this the case by checking - * whether the global deadline has advanced. It is valid to compare - * cfs_b->runtime_expires without any locks since we only care about - * exact equality, so a partial write will still work. - */ + /* + * If the local deadline has passed we have to consider the + * possibility that our sched_clock is 'fast' and the global deadline + * has not truly expired. + * + * Fortunately we can check determine whether this the case by checking + * whether the global deadline has advanced. It is valid to compare + * cfs_b->runtime_expires without any locks since we only care about + * exact equality, so a partial write will still work. + */ - if (cfs_rq->runtime_expires != cfs_b->runtime_expires) { - /* extend local deadline, drift is bounded above by 2 ticks */ - cfs_rq->runtime_expires += TICK_NSEC; - } else { - /* global deadline is ahead, expiration has passed */ - cfs_rq->runtime_remaining = 0; - } + if (cfs_rq->runtime_expires != cfs_b->runtime_expires) { + /* extend local deadline, drift is bounded above by 2 ticks */ + cfs_rq->runtime_expires += TICK_NSEC; + } else { + /* global deadline is ahead, expiration has passed */ + cfs_rq->runtime_remaining = 0; + } } static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) { - /* dock delta_exec before expiring quota (as it could span periods) */ - cfs_rq->runtime_remaining -= delta_exec; - expire_cfs_rq_runtime(cfs_rq); + /* dock delta_exec before expiring quota (as it could span periods) */ + cfs_rq->runtime_remaining -= delta_exec; + expire_cfs_rq_runtime(cfs_rq); - if (likely(cfs_rq->runtime_remaining > 0)) - return; + if (likely(cfs_rq->runtime_remaining > 0)) + return; - /* - * if we're unable to extend our runtime we resched so that the active - * hierarchy can be throttled - */ - if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr)) - resched_curr(rq_of(cfs_rq)); + /* + * if we're unable to extend our runtime we resched so that the active + * hierarchy can be throttled + */ + if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr)) + resched_curr(rq_of(cfs_rq)); } static __always_inline void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) { - if (!cfs_bandwidth_used() || !cfs_rq->runtime_enabled) - return; + if (!cfs_bandwidth_used() || !cfs_rq->runtime_enabled) + return; - __account_cfs_rq_runtime(cfs_rq, delta_exec); + __account_cfs_rq_runtime(cfs_rq, delta_exec); } static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) { - return cfs_bandwidth_used() && cfs_rq->throttled; + return cfs_bandwidth_used() && cfs_rq->throttled; } /* check whether cfs_rq, or any parent, is throttled */ static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) { - return cfs_bandwidth_used() && cfs_rq->throttle_count; + return cfs_bandwidth_used() && cfs_rq->throttle_count; } /* @@ -3700,186 +3705,186 @@ static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) * load-balance operations. */ static inline int throttled_lb_pair(struct task_group *tg, - int src_cpu, int dest_cpu) + int src_cpu, int dest_cpu) { - struct cfs_rq *src_cfs_rq, *dest_cfs_rq; + struct cfs_rq *src_cfs_rq, *dest_cfs_rq; - src_cfs_rq = tg->cfs_rq[src_cpu]; - dest_cfs_rq = tg->cfs_rq[dest_cpu]; + src_cfs_rq = tg->cfs_rq[src_cpu]; + dest_cfs_rq = tg->cfs_rq[dest_cpu]; - return throttled_hierarchy(src_cfs_rq) || - throttled_hierarchy(dest_cfs_rq); + return throttled_hierarchy(src_cfs_rq) || + throttled_hierarchy(dest_cfs_rq); } /* updated child weight may affect parent so we have to do this bottom up */ static int tg_unthrottle_up(struct task_group *tg, void *data) { - struct rq *rq = data; - struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; + struct rq *rq = data; + struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; - cfs_rq->throttle_count--; + cfs_rq->throttle_count--; #ifdef CONFIG_SMP - if (!cfs_rq->throttle_count) { - /* adjust cfs_rq_clock_task() */ - cfs_rq->throttled_clock_task_time += rq_clock_task(rq) - - cfs_rq->throttled_clock_task; - } + if (!cfs_rq->throttle_count) { + /* adjust cfs_rq_clock_task() */ + cfs_rq->throttled_clock_task_time += rq_clock_task(rq) - + cfs_rq->throttled_clock_task; + } #endif - return 0; + return 0; } static int tg_throttle_down(struct task_group *tg, void *data) { - struct rq *rq = data; - struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; + struct rq *rq = data; + struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; - /* group is entering throttled state, stop time */ - if (!cfs_rq->throttle_count) - cfs_rq->throttled_clock_task = rq_clock_task(rq); - cfs_rq->throttle_count++; + /* group is entering throttled state, stop time */ + if (!cfs_rq->throttle_count) + cfs_rq->throttled_clock_task = rq_clock_task(rq); + cfs_rq->throttle_count++; - return 0; + return 0; } static void throttle_cfs_rq(struct cfs_rq *cfs_rq) { - struct rq *rq = rq_of(cfs_rq); - struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); - struct sched_entity *se; - long task_delta, dequeue = 1; - bool empty; + struct rq *rq = rq_of(cfs_rq); + struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); + struct sched_entity *se; + long task_delta, dequeue = 1; + bool empty; - se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))]; + se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))]; - /* freeze hierarchy runnable averages while throttled */ - rcu_read_lock(); - walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq); - rcu_read_unlock(); + /* freeze hierarchy runnable averages while throttled */ + rcu_read_lock(); + walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq); + rcu_read_unlock(); - task_delta = cfs_rq->h_nr_running; - for_each_sched_entity(se) { - struct cfs_rq *qcfs_rq = cfs_rq_of(se); - /* throttled entity or throttle-on-deactivate */ - if (!se->on_rq) - break; + task_delta = cfs_rq->h_nr_running; + for_each_sched_entity(se) { + struct cfs_rq *qcfs_rq = cfs_rq_of(se); + /* throttled entity or throttle-on-deactivate */ + if (!se->on_rq) + break; - if (dequeue) - dequeue_entity(qcfs_rq, se, DEQUEUE_SLEEP); - qcfs_rq->h_nr_running -= task_delta; + if (dequeue) + dequeue_entity(qcfs_rq, se, DEQUEUE_SLEEP); + qcfs_rq->h_nr_running -= task_delta; - if (qcfs_rq->load.weight) - dequeue = 0; - } + if (qcfs_rq->load.weight) + dequeue = 0; + } - if (!se) - sub_nr_running(rq, task_delta); + if (!se) + sub_nr_running(rq, task_delta); - cfs_rq->throttled = 1; - cfs_rq->throttled_clock = rq_clock(rq); - raw_spin_lock(&cfs_b->lock); - empty = list_empty(&cfs_b->throttled_cfs_rq); + cfs_rq->throttled = 1; + cfs_rq->throttled_clock = rq_clock(rq); + raw_spin_lock(&cfs_b->lock); + empty = list_empty(&cfs_b->throttled_cfs_rq); - /* - * Add to the _head_ of the list, so that an already-started - * distribute_cfs_runtime will not see us - */ - list_add_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq); + /* + * Add to the _head_ of the list, so that an already-started + * distribute_cfs_runtime will not see us + */ + list_add_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq); - /* - * If we're the first throttled task, make sure the bandwidth - * timer is running. - */ - if (empty) - start_cfs_bandwidth(cfs_b); + /* + * If we're the first throttled task, make sure the bandwidth + * timer is running. + */ + if (empty) + start_cfs_bandwidth(cfs_b); - raw_spin_unlock(&cfs_b->lock); + raw_spin_unlock(&cfs_b->lock); } void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) { - struct rq *rq = rq_of(cfs_rq); - struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); - struct sched_entity *se; - int enqueue = 1; - long task_delta; + struct rq *rq = rq_of(cfs_rq); + struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); + struct sched_entity *se; + int enqueue = 1; + long task_delta; - se = cfs_rq->tg->se[cpu_of(rq)]; + se = cfs_rq->tg->se[cpu_of(rq)]; - cfs_rq->throttled = 0; + cfs_rq->throttled = 0; - update_rq_clock(rq); + update_rq_clock(rq); - raw_spin_lock(&cfs_b->lock); - cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock; - list_del_rcu(&cfs_rq->throttled_list); - raw_spin_unlock(&cfs_b->lock); + raw_spin_lock(&cfs_b->lock); + cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock; + list_del_rcu(&cfs_rq->throttled_list); + raw_spin_unlock(&cfs_b->lock); - /* update hierarchical throttle state */ - walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq); + /* update hierarchical throttle state */ + walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq); - if (!cfs_rq->load.weight) - return; + if (!cfs_rq->load.weight) + return; - task_delta = cfs_rq->h_nr_running; - for_each_sched_entity(se) { - if (se->on_rq) - enqueue = 0; + task_delta = cfs_rq->h_nr_running; + for_each_sched_entity(se) { + if (se->on_rq) + enqueue = 0; - cfs_rq = cfs_rq_of(se); - if (enqueue) - enqueue_entity(cfs_rq, se, ENQUEUE_WAKEUP); - cfs_rq->h_nr_running += task_delta; + cfs_rq = cfs_rq_of(se); + if (enqueue) + enqueue_entity(cfs_rq, se, ENQUEUE_WAKEUP); + cfs_rq->h_nr_running += task_delta; - if (cfs_rq_throttled(cfs_rq)) - break; - } + if (cfs_rq_throttled(cfs_rq)) + break; + } - if (!se) - add_nr_running(rq, task_delta); + if (!se) + add_nr_running(rq, task_delta); - /* determine whether we need to wake up potentially idle cpu */ - if (rq->curr == rq->idle && rq->cfs.nr_running) - resched_curr(rq); + /* determine whether we need to wake up potentially idle cpu */ + if (rq->curr == rq->idle && rq->cfs.nr_running) + resched_curr(rq); } static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b, - u64 remaining, u64 expires) + u64 remaining, u64 expires) { - struct cfs_rq *cfs_rq; - u64 runtime; - u64 starting_runtime = remaining; + struct cfs_rq *cfs_rq; + u64 runtime; + u64 starting_runtime = remaining; - rcu_read_lock(); - list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq, - throttled_list) { - struct rq *rq = rq_of(cfs_rq); + rcu_read_lock(); + list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq, + throttled_list) { + struct rq *rq = rq_of(cfs_rq); - raw_spin_lock(&rq->lock); - if (!cfs_rq_throttled(cfs_rq)) - goto next; + raw_spin_lock(&rq->lock); + if (!cfs_rq_throttled(cfs_rq)) + goto next; - runtime = -cfs_rq->runtime_remaining + 1; - if (runtime > remaining) - runtime = remaining; - remaining -= runtime; + runtime = -cfs_rq->runtime_remaining + 1; + if (runtime > remaining) + runtime = remaining; + remaining -= runtime; - cfs_rq->runtime_remaining += runtime; - cfs_rq->runtime_expires = expires; + cfs_rq->runtime_remaining += runtime; + cfs_rq->runtime_expires = expires; - /* we check whether we're throttled above */ - if (cfs_rq->runtime_remaining > 0) - unthrottle_cfs_rq(cfs_rq); + /* we check whether we're throttled above */ + if (cfs_rq->runtime_remaining > 0) + unthrottle_cfs_rq(cfs_rq); next: - raw_spin_unlock(&rq->lock); + raw_spin_unlock(&rq->lock); - if (!remaining) - break; - } - rcu_read_unlock(); + if (!remaining) + break; + } + rcu_read_unlock(); - return starting_runtime - remaining; + return starting_runtime - remaining; } /* @@ -3890,68 +3895,68 @@ next: */ static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun) { - u64 runtime, runtime_expires; - int throttled; + u64 runtime, runtime_expires; + int throttled; - /* no need to continue the timer with no bandwidth constraint */ - if (cfs_b->quota == RUNTIME_INF) - goto out_deactivate; + /* no need to continue the timer with no bandwidth constraint */ + if (cfs_b->quota == RUNTIME_INF) + goto out_deactivate; + + throttled = !list_empty(&cfs_b->throttled_cfs_rq); + cfs_b->nr_periods += overrun; + + /* + * idle depends on !throttled (for the case of a large deficit), and if + * we're going inactive then everything else can be deferred + */ + if (cfs_b->idle && !throttled) + goto out_deactivate; + + __refill_cfs_bandwidth_runtime(cfs_b); + + if (!throttled) { + /* mark as potentially idle for the upcoming period */ + cfs_b->idle = 1; + return 0; + } + + /* account preceding periods in which throttling occurred */ + cfs_b->nr_throttled += overrun; + + runtime_expires = cfs_b->runtime_expires; + + /* + * This check is repeated as we are holding onto the new bandwidth while + * we unthrottle. This can potentially race with an unthrottled group + * trying to acquire new bandwidth from the global pool. This can result + * in us over-using our runtime if it is all used during this loop, but + * only by limited amounts in that extreme case. + */ + while (throttled && cfs_b->runtime > 0) { + runtime = cfs_b->runtime; + raw_spin_unlock(&cfs_b->lock); + /* we can't nest cfs_b->lock while distributing bandwidth */ + runtime = distribute_cfs_runtime(cfs_b, runtime, + runtime_expires); + raw_spin_lock(&cfs_b->lock); throttled = !list_empty(&cfs_b->throttled_cfs_rq); - cfs_b->nr_periods += overrun; - /* - * idle depends on !throttled (for the case of a large deficit), and if - * we're going inactive then everything else can be deferred - */ - if (cfs_b->idle && !throttled) - goto out_deactivate; + cfs_b->runtime -= min(runtime, cfs_b->runtime); + } - __refill_cfs_bandwidth_runtime(cfs_b); + /* + * While we are ensured activity in the period following an + * unthrottle, this also covers the case in which the new bandwidth is + * insufficient to cover the existing bandwidth deficit. (Forcing the + * timer to remain active while there are any throttled entities.) + */ + cfs_b->idle = 0; - if (!throttled) { - /* mark as potentially idle for the upcoming period */ - cfs_b->idle = 1; - return 0; - } - - /* account preceding periods in which throttling occurred */ - cfs_b->nr_throttled += overrun; - - runtime_expires = cfs_b->runtime_expires; - - /* - * This check is repeated as we are holding onto the new bandwidth while - * we unthrottle. This can potentially race with an unthrottled group - * trying to acquire new bandwidth from the global pool. This can result - * in us over-using our runtime if it is all used during this loop, but - * only by limited amounts in that extreme case. - */ - while (throttled && cfs_b->runtime > 0) { - runtime = cfs_b->runtime; - raw_spin_unlock(&cfs_b->lock); - /* we can't nest cfs_b->lock while distributing bandwidth */ - runtime = distribute_cfs_runtime(cfs_b, runtime, - runtime_expires); - raw_spin_lock(&cfs_b->lock); - - throttled = !list_empty(&cfs_b->throttled_cfs_rq); - - cfs_b->runtime -= min(runtime, cfs_b->runtime); - } - - /* - * While we are ensured activity in the period following an - * unthrottle, this also covers the case in which the new bandwidth is - * insufficient to cover the existing bandwidth deficit. (Forcing the - * timer to remain active while there are any throttled entities.) - */ - cfs_b->idle = 0; - - return 0; + return 0; out_deactivate: - return 1; + return 1; } /* a cfs_rq won't donate quota below this amount */ @@ -3970,68 +3975,68 @@ static const u64 cfs_bandwidth_slack_period = 5 * NSEC_PER_MSEC; */ static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire) { - struct hrtimer *refresh_timer = &cfs_b->period_timer; - u64 remaining; + struct hrtimer *refresh_timer = &cfs_b->period_timer; + u64 remaining; - /* if the call-back is running a quota refresh is already occurring */ - if (hrtimer_callback_running(refresh_timer)) - return 1; + /* if the call-back is running a quota refresh is already occurring */ + if (hrtimer_callback_running(refresh_timer)) + return 1; - /* is a quota refresh about to occur? */ - remaining = ktime_to_ns(hrtimer_expires_remaining(refresh_timer)); - if (remaining < min_expire) - return 1; + /* is a quota refresh about to occur? */ + remaining = ktime_to_ns(hrtimer_expires_remaining(refresh_timer)); + if (remaining < min_expire) + return 1; - return 0; + return 0; } static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b) { - u64 min_left = cfs_bandwidth_slack_period + min_bandwidth_expiration; + u64 min_left = cfs_bandwidth_slack_period + min_bandwidth_expiration; - /* if there's a quota refresh soon don't bother with slack */ - if (runtime_refresh_within(cfs_b, min_left)) - return; + /* if there's a quota refresh soon don't bother with slack */ + if (runtime_refresh_within(cfs_b, min_left)) + return; - hrtimer_start(&cfs_b->slack_timer, - ns_to_ktime(cfs_bandwidth_slack_period), - HRTIMER_MODE_REL); + hrtimer_start(&cfs_b->slack_timer, + ns_to_ktime(cfs_bandwidth_slack_period), + HRTIMER_MODE_REL); } /* we know any runtime found here is valid as update_curr() precedes return */ static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq) { - struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); - s64 slack_runtime = cfs_rq->runtime_remaining - min_cfs_rq_runtime; + struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); + s64 slack_runtime = cfs_rq->runtime_remaining - min_cfs_rq_runtime; - if (slack_runtime <= 0) - return; + if (slack_runtime <= 0) + return; - raw_spin_lock(&cfs_b->lock); - if (cfs_b->quota != RUNTIME_INF && - cfs_rq->runtime_expires == cfs_b->runtime_expires) { - cfs_b->runtime += slack_runtime; + raw_spin_lock(&cfs_b->lock); + if (cfs_b->quota != RUNTIME_INF && + cfs_rq->runtime_expires == cfs_b->runtime_expires) { + cfs_b->runtime += slack_runtime; - /* we are under rq->lock, defer unthrottling using a timer */ - if (cfs_b->runtime > sched_cfs_bandwidth_slice() && - !list_empty(&cfs_b->throttled_cfs_rq)) - start_cfs_slack_bandwidth(cfs_b); - } - raw_spin_unlock(&cfs_b->lock); + /* we are under rq->lock, defer unthrottling using a timer */ + if (cfs_b->runtime > sched_cfs_bandwidth_slice() && + !list_empty(&cfs_b->throttled_cfs_rq)) + start_cfs_slack_bandwidth(cfs_b); + } + raw_spin_unlock(&cfs_b->lock); - /* even if it's not valid for return we don't want to try again */ - cfs_rq->runtime_remaining -= slack_runtime; + /* even if it's not valid for return we don't want to try again */ + cfs_rq->runtime_remaining -= slack_runtime; } static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) { - if (!cfs_bandwidth_used()) - return; + if (!cfs_bandwidth_used()) + return; - if (!cfs_rq->runtime_enabled || cfs_rq->nr_running) - return; + if (!cfs_rq->runtime_enabled || cfs_rq->nr_running) + return; - __return_cfs_rq_runtime(cfs_rq); + __return_cfs_rq_runtime(cfs_rq); } /* @@ -4040,31 +4045,31 @@ static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) */ static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) { - u64 runtime = 0, slice = sched_cfs_bandwidth_slice(); - u64 expires; + u64 runtime = 0, slice = sched_cfs_bandwidth_slice(); + u64 expires; - /* confirm we're still not at a refresh boundary */ - raw_spin_lock(&cfs_b->lock); - if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) { - raw_spin_unlock(&cfs_b->lock); - return; - } - - if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice) - runtime = cfs_b->runtime; - - expires = cfs_b->runtime_expires; + /* confirm we're still not at a refresh boundary */ + raw_spin_lock(&cfs_b->lock); + if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) { raw_spin_unlock(&cfs_b->lock); + return; + } - if (!runtime) - return; + if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice) + runtime = cfs_b->runtime; - runtime = distribute_cfs_runtime(cfs_b, runtime, expires); + expires = cfs_b->runtime_expires; + raw_spin_unlock(&cfs_b->lock); - raw_spin_lock(&cfs_b->lock); - if (expires == cfs_b->runtime_expires) - cfs_b->runtime -= min(runtime, cfs_b->runtime); - raw_spin_unlock(&cfs_b->lock); + if (!runtime) + return; + + runtime = distribute_cfs_runtime(cfs_b, runtime, expires); + + raw_spin_lock(&cfs_b->lock); + if (expires == cfs_b->runtime_expires) + cfs_b->runtime -= min(runtime, cfs_b->runtime); + raw_spin_unlock(&cfs_b->lock); } /* @@ -4074,157 +4079,157 @@ static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) */ static void check_enqueue_throttle(struct cfs_rq *cfs_rq) { - if (!cfs_bandwidth_used()) - return; + if (!cfs_bandwidth_used()) + return; - /* an active group must be handled by the update_curr()->put() path */ - if (!cfs_rq->runtime_enabled || cfs_rq->curr) - return; + /* an active group must be handled by the update_curr()->put() path */ + if (!cfs_rq->runtime_enabled || cfs_rq->curr) + return; - /* ensure the group is not already throttled */ - if (cfs_rq_throttled(cfs_rq)) - return; + /* ensure the group is not already throttled */ + if (cfs_rq_throttled(cfs_rq)) + return; - /* update runtime allocation */ - account_cfs_rq_runtime(cfs_rq, 0); - if (cfs_rq->runtime_remaining <= 0) - throttle_cfs_rq(cfs_rq); + /* update runtime allocation */ + account_cfs_rq_runtime(cfs_rq, 0); + if (cfs_rq->runtime_remaining <= 0) + throttle_cfs_rq(cfs_rq); } /* conditionally throttle active cfs_rq's from put_prev_entity() */ static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) { - if (!cfs_bandwidth_used()) - return false; + if (!cfs_bandwidth_used()) + return false; - if (likely(!cfs_rq->runtime_enabled || cfs_rq->runtime_remaining > 0)) - return false; + if (likely(!cfs_rq->runtime_enabled || cfs_rq->runtime_remaining > 0)) + return false; - /* - * it's possible for a throttled entity to be forced into a running - * state (e.g. set_curr_task), in this case we're finished. - */ - if (cfs_rq_throttled(cfs_rq)) - return true; - - throttle_cfs_rq(cfs_rq); + /* + * it's possible for a throttled entity to be forced into a running + * state (e.g. set_curr_task), in this case we're finished. + */ + if (cfs_rq_throttled(cfs_rq)) return true; + + throttle_cfs_rq(cfs_rq); + return true; } static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer) { - struct cfs_bandwidth *cfs_b = - container_of(timer, struct cfs_bandwidth, slack_timer); + struct cfs_bandwidth *cfs_b = + container_of(timer, struct cfs_bandwidth, slack_timer); - do_sched_cfs_slack_timer(cfs_b); + do_sched_cfs_slack_timer(cfs_b); - return HRTIMER_NORESTART; + return HRTIMER_NORESTART; } static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer) { - struct cfs_bandwidth *cfs_b = - container_of(timer, struct cfs_bandwidth, period_timer); - int overrun; - int idle = 0; + struct cfs_bandwidth *cfs_b = + container_of(timer, struct cfs_bandwidth, period_timer); + int overrun; + int idle = 0; - raw_spin_lock(&cfs_b->lock); - for (;;) { - overrun = hrtimer_forward_now(timer, cfs_b->period); - if (!overrun) - break; + raw_spin_lock(&cfs_b->lock); + for (;;) { + overrun = hrtimer_forward_now(timer, cfs_b->period); + if (!overrun) + break; - idle = do_sched_cfs_period_timer(cfs_b, overrun); - } - if (idle) - cfs_b->period_active = 0; - raw_spin_unlock(&cfs_b->lock); + idle = do_sched_cfs_period_timer(cfs_b, overrun); + } + if (idle) + cfs_b->period_active = 0; + raw_spin_unlock(&cfs_b->lock); - return idle ? HRTIMER_NORESTART : HRTIMER_RESTART; + return idle ? HRTIMER_NORESTART : HRTIMER_RESTART; } void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) { - raw_spin_lock_init(&cfs_b->lock); - cfs_b->runtime = 0; - cfs_b->quota = RUNTIME_INF; - cfs_b->period = ns_to_ktime(default_cfs_period()); + raw_spin_lock_init(&cfs_b->lock); + cfs_b->runtime = 0; + cfs_b->quota = RUNTIME_INF; + cfs_b->period = ns_to_ktime(default_cfs_period()); - INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq); - hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED); - cfs_b->period_timer.function = sched_cfs_period_timer; - hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); - cfs_b->slack_timer.function = sched_cfs_slack_timer; + INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq); + hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED); + cfs_b->period_timer.function = sched_cfs_period_timer; + hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + cfs_b->slack_timer.function = sched_cfs_slack_timer; } static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) { - cfs_rq->runtime_enabled = 0; - INIT_LIST_HEAD(&cfs_rq->throttled_list); + cfs_rq->runtime_enabled = 0; + INIT_LIST_HEAD(&cfs_rq->throttled_list); } void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b) { - lockdep_assert_held(&cfs_b->lock); + lockdep_assert_held(&cfs_b->lock); - if (!cfs_b->period_active) { - cfs_b->period_active = 1; - hrtimer_forward_now(&cfs_b->period_timer, cfs_b->period); - hrtimer_start_expires(&cfs_b->period_timer, HRTIMER_MODE_ABS_PINNED); - } + if (!cfs_b->period_active) { + cfs_b->period_active = 1; + hrtimer_forward_now(&cfs_b->period_timer, cfs_b->period); + hrtimer_start_expires(&cfs_b->period_timer, HRTIMER_MODE_ABS_PINNED); + } } static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) { - /* init_cfs_bandwidth() was not called */ - if (!cfs_b->throttled_cfs_rq.next) - return; + /* init_cfs_bandwidth() was not called */ + if (!cfs_b->throttled_cfs_rq.next) + return; - hrtimer_cancel(&cfs_b->period_timer); - hrtimer_cancel(&cfs_b->slack_timer); + hrtimer_cancel(&cfs_b->period_timer); + hrtimer_cancel(&cfs_b->slack_timer); } static void __maybe_unused update_runtime_enabled(struct rq *rq) { - struct cfs_rq *cfs_rq; + struct cfs_rq *cfs_rq; - for_each_leaf_cfs_rq(rq, cfs_rq) { - struct cfs_bandwidth *cfs_b = &cfs_rq->tg->cfs_bandwidth; + for_each_leaf_cfs_rq(rq, cfs_rq) { + struct cfs_bandwidth *cfs_b = &cfs_rq->tg->cfs_bandwidth; - raw_spin_lock(&cfs_b->lock); - cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF; - raw_spin_unlock(&cfs_b->lock); - } + raw_spin_lock(&cfs_b->lock); + cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF; + raw_spin_unlock(&cfs_b->lock); + } } static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq) { - struct cfs_rq *cfs_rq; + struct cfs_rq *cfs_rq; - for_each_leaf_cfs_rq(rq, cfs_rq) { - if (!cfs_rq->runtime_enabled) - continue; + for_each_leaf_cfs_rq(rq, cfs_rq) { + if (!cfs_rq->runtime_enabled) + continue; - /* - * clock_task is not advancing so we just need to make sure - * there's some valid quota amount - */ - cfs_rq->runtime_remaining = 1; - /* - * Offline rq is schedulable till cpu is completely disabled - * in take_cpu_down(), so we prevent new cfs throttling here. - */ - cfs_rq->runtime_enabled = 0; + /* + * clock_task is not advancing so we just need to make sure + * there's some valid quota amount + */ + cfs_rq->runtime_remaining = 1; + /* + * Offline rq is schedulable till cpu is completely disabled + * in take_cpu_down(), so we prevent new cfs throttling here. + */ + cfs_rq->runtime_enabled = 0; - if (cfs_rq_throttled(cfs_rq)) - unthrottle_cfs_rq(cfs_rq); - } + if (cfs_rq_throttled(cfs_rq)) + unthrottle_cfs_rq(cfs_rq); + } } #else /* CONFIG_CFS_BANDWIDTH */ static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq) { - return rq_clock_task(rq_of(cfs_rq)); + return rq_clock_task(rq_of(cfs_rq)); } static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) {} @@ -4234,18 +4239,18 @@ static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) { - return 0; + return 0; } static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) { - return 0; + return 0; } static inline int throttled_lb_pair(struct task_group *tg, - int src_cpu, int dest_cpu) + int src_cpu, int dest_cpu) { - return 0; + return 0; } void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {} @@ -4256,7 +4261,7 @@ static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) { - return NULL; + return NULL; } static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {} static inline void update_runtime_enabled(struct rq *rq) {} @@ -4271,23 +4276,23 @@ static inline void unthrottle_offline_cfs_rqs(struct rq *rq) {} #ifdef CONFIG_SCHED_HRTICK static void hrtick_start_fair(struct rq *rq, struct task_struct *p) { - struct sched_entity *se = &p->se; - struct cfs_rq *cfs_rq = cfs_rq_of(se); + struct sched_entity *se = &p->se; + struct cfs_rq *cfs_rq = cfs_rq_of(se); - WARN_ON(task_rq(p) != rq); + WARN_ON(task_rq(p) != rq); - if (cfs_rq->nr_running > 1) { - u64 slice = sched_slice(cfs_rq, se); - u64 ran = se->sum_exec_runtime - se->prev_sum_exec_runtime; - s64 delta = slice - ran; + if (cfs_rq->nr_running > 1) { + u64 slice = sched_slice(cfs_rq, se); + u64 ran = se->sum_exec_runtime - se->prev_sum_exec_runtime; + s64 delta = slice - ran; - if (delta < 0) { - if (rq->curr == p) - resched_curr(rq); - return; - } - hrtick_start(rq, delta); + if (delta < 0) { + if (rq->curr == p) + resched_curr(rq); + return; } + hrtick_start(rq, delta); + } } /* @@ -4297,13 +4302,13 @@ static void hrtick_start_fair(struct rq *rq, struct task_struct *p) */ static void hrtick_update(struct rq *rq) { - struct task_struct *curr = rq->curr; + struct task_struct *curr = rq->curr; - if (!hrtick_enabled(rq) || curr->sched_class != &fair_sched_class) - return; + if (!hrtick_enabled(rq) || curr->sched_class != &fair_sched_class) + return; - if (cfs_rq_of(&curr->se)->nr_running < sched_nr_latency) - hrtick_start_fair(rq, curr); + if (cfs_rq_of(&curr->se)->nr_running < sched_nr_latency) + hrtick_start_fair(rq, curr); } #else /* !CONFIG_SCHED_HRTICK */ static inline void @@ -4324,43 +4329,43 @@ static inline void hrtick_update(struct rq *rq) static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) { - struct cfs_rq *cfs_rq; - struct sched_entity *se = &p->se; + struct cfs_rq *cfs_rq; + struct sched_entity *se = &p->se; - for_each_sched_entity(se) { - if (se->on_rq) - break; - cfs_rq = cfs_rq_of(se); - enqueue_entity(cfs_rq, se, flags); + for_each_sched_entity(se) { + if (se->on_rq) + break; + cfs_rq = cfs_rq_of(se); + enqueue_entity(cfs_rq, se, flags); - /* - * end evaluation on encountering a throttled cfs_rq - * - * note: in the case of encountering a throttled cfs_rq we will - * post the final h_nr_running increment below. - */ - if (cfs_rq_throttled(cfs_rq)) - break; - cfs_rq->h_nr_running++; + /* + * end evaluation on encountering a throttled cfs_rq + * + * note: in the case of encountering a throttled cfs_rq we will + * post the final h_nr_running increment below. + */ + if (cfs_rq_throttled(cfs_rq)) + break; + cfs_rq->h_nr_running++; - flags = ENQUEUE_WAKEUP; - } + flags = ENQUEUE_WAKEUP; + } - for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); - cfs_rq->h_nr_running++; + for_each_sched_entity(se) { + cfs_rq = cfs_rq_of(se); + cfs_rq->h_nr_running++; - if (cfs_rq_throttled(cfs_rq)) - break; + if (cfs_rq_throttled(cfs_rq)) + break; - update_load_avg(se, 1); - update_cfs_shares(cfs_rq); - } + update_load_avg(se, 1); + update_cfs_shares(cfs_rq); + } - if (!se) - add_nr_running(rq, 1); + if (!se) + add_nr_running(rq, 1); - hrtick_update(rq); + hrtick_update(rq); } static void set_next_buddy(struct sched_entity *se); @@ -4372,55 +4377,55 @@ static void set_next_buddy(struct sched_entity *se); */ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) { - struct cfs_rq *cfs_rq; - struct sched_entity *se = &p->se; - int task_sleep = flags & DEQUEUE_SLEEP; + struct cfs_rq *cfs_rq; + struct sched_entity *se = &p->se; + int task_sleep = flags & DEQUEUE_SLEEP; - for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); - dequeue_entity(cfs_rq, se, flags); + for_each_sched_entity(se) { + cfs_rq = cfs_rq_of(se); + dequeue_entity(cfs_rq, se, flags); - /* - * end evaluation on encountering a throttled cfs_rq - * - * note: in the case of encountering a throttled cfs_rq we will - * post the final h_nr_running decrement below. - */ - if (cfs_rq_throttled(cfs_rq)) - break; - cfs_rq->h_nr_running--; + /* + * end evaluation on encountering a throttled cfs_rq + * + * note: in the case of encountering a throttled cfs_rq we will + * post the final h_nr_running decrement below. + */ + if (cfs_rq_throttled(cfs_rq)) + break; + cfs_rq->h_nr_running--; - /* Don't dequeue parent if it has other entities besides us */ - if (cfs_rq->load.weight) { - /* - * Bias pick_next to pick a task from this cfs_rq, as - * p is sleeping when it is within its sched_slice. - */ - if (task_sleep && parent_entity(se)) - set_next_buddy(parent_entity(se)); + /* Don't dequeue parent if it has other entities besides us */ + if (cfs_rq->load.weight) { + /* + * Bias pick_next to pick a task from this cfs_rq, as + * p is sleeping when it is within its sched_slice. + */ + if (task_sleep && parent_entity(se)) + set_next_buddy(parent_entity(se)); - /* avoid re-evaluating load for this entity */ - se = parent_entity(se); - break; - } - flags |= DEQUEUE_SLEEP; + /* avoid re-evaluating load for this entity */ + se = parent_entity(se); + break; } + flags |= DEQUEUE_SLEEP; + } - for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); - cfs_rq->h_nr_running--; + for_each_sched_entity(se) { + cfs_rq = cfs_rq_of(se); + cfs_rq->h_nr_running--; - if (cfs_rq_throttled(cfs_rq)) - break; + if (cfs_rq_throttled(cfs_rq)) + break; - update_load_avg(se, 1); - update_cfs_shares(cfs_rq); - } + update_load_avg(se, 1); + update_cfs_shares(cfs_rq); + } - if (!se) - sub_nr_running(rq, 1); + if (!se) + sub_nr_running(rq, 1); - hrtick_update(rq); + hrtick_update(rq); } #ifdef CONFIG_SMP @@ -4451,15 +4456,15 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) * * The calculation is approximated on a 128 point scale. */ -#define DEGRADE_SHIFT 7 +#define DEGRADE_SHIFT 7 static const u8 degrade_zero_ticks[CPU_LOAD_IDX_MAX] = {0, 8, 32, 64, 128}; static const u8 degrade_factor[CPU_LOAD_IDX_MAX][DEGRADE_SHIFT + 1] = { - { 0, 0, 0, 0, 0, 0, 0, 0 }, - { 64, 32, 8, 0, 0, 0, 0, 0 }, - { 96, 72, 40, 12, 1, 0, 0, 0 }, - { 112, 98, 75, 43, 15, 1, 0, 0 }, - { 120, 112, 98, 76, 45, 16, 2, 0 } + { 0, 0, 0, 0, 0, 0, 0, 0 }, + { 64, 32, 8, 0, 0, 0, 0, 0 }, + { 96, 72, 40, 12, 1, 0, 0, 0 }, + { 112, 98, 75, 43, 15, 1, 0, 0 }, + { 120, 112, 98, 76, 45, 16, 2, 0 } }; /* @@ -4470,25 +4475,25 @@ static const u8 degrade_factor[CPU_LOAD_IDX_MAX][DEGRADE_SHIFT + 1] = { static unsigned long decay_load_missed(unsigned long load, unsigned long missed_updates, int idx) { - int j = 0; + int j = 0; - if (!missed_updates) - return load; - - if (missed_updates >= degrade_zero_ticks[idx]) - return 0; - - if (idx == 1) - return load >> missed_updates; - - while (missed_updates) { - if (missed_updates % 2) - load = (load * degrade_factor[idx][j]) >> DEGRADE_SHIFT; - - missed_updates >>= 1; - j++; - } + if (!missed_updates) return load; + + if (missed_updates >= degrade_zero_ticks[idx]) + return 0; + + if (idx == 1) + return load >> missed_updates; + + while (missed_updates) { + if (missed_updates % 2) + load = (load * degrade_factor[idx][j]) >> DEGRADE_SHIFT; + + missed_updates >>= 1; + j++; + } + return load; } /** @@ -4509,13 +4514,13 @@ decay_load_missed(unsigned long load, unsigned long missed_updates, int idx) * the @pending_updates argument. * * load[i]_n = (1 - 1/2^i) * load[i]_n-1 + (1/2^i) * load_n-1 - * = A * load[i]_n-1 + B ; A := (1 - 1/2^i), B := (1/2^i) * load - * = A * (A * load[i]_n-2 + B) + B - * = A * (A * (A * load[i]_n-3 + B) + B) + B - * = A^3 * load[i]_n-3 + (A^2 + A + 1) * B - * = A^n * load[i]_0 + (A^(n-1) + A^(n-2) + ... + 1) * B - * = A^n * load[i]_0 + ((1 - A^n) / (1 - A)) * B - * = (1 - 1/2^i)^n * (load[i]_0 - load) + load + * = A * load[i]_n-1 + B ; A := (1 - 1/2^i), B := (1/2^i) * load + * = A * (A * load[i]_n-2 + B) + B + * = A * (A * (A * load[i]_n-3 + B) + B) + B + * = A^3 * load[i]_n-3 + (A^2 + A + 1) * B + * = A^n * load[i]_0 + (A^(n-1) + A^(n-2) + ... + 1) * B + * = A^n * load[i]_0 + ((1 - A^n) / (1 - A)) * B + * = (1 - 1/2^i)^n * (load[i]_0 - load) + load * * In the above we've assumed load_n := load, which is true for NOHZ_FULL as * any change in load would have resulted in the tick being turned back on. @@ -4528,70 +4533,70 @@ decay_load_missed(unsigned long load, unsigned long missed_updates, int idx) * term. See the @active paramter. */ static void __update_cpu_load(struct rq *this_rq, unsigned long this_load, - unsigned long pending_updates, int active) + unsigned long pending_updates, int active) { - unsigned long tickless_load = active ? this_rq->cpu_load[0] : 0; - int i, scale; + unsigned long tickless_load = active ? this_rq->cpu_load[0] : 0; + int i, scale; - this_rq->nr_load_updates++; + this_rq->nr_load_updates++; - /* Update our load: */ - this_rq->cpu_load[0] = this_load; /* Fasttrack for idx 0 */ - for (i = 1, scale = 2; i < CPU_LOAD_IDX_MAX; i++, scale += scale) { - unsigned long old_load, new_load; + /* Update our load: */ + this_rq->cpu_load[0] = this_load; /* Fasttrack for idx 0 */ + for (i = 1, scale = 2; i < CPU_LOAD_IDX_MAX; i++, scale += scale) { + unsigned long old_load, new_load; - /* scale is effectively 1 << i now, and >> i divides by scale */ + /* scale is effectively 1 << i now, and >> i divides by scale */ - old_load = this_rq->cpu_load[i]; - old_load = decay_load_missed(old_load, pending_updates - 1, i); - if (tickless_load) { - old_load -= decay_load_missed(tickless_load, pending_updates - 1, i); - /* - * old_load can never be a negative value because a - * decayed tickless_load cannot be greater than the - * original tickless_load. - */ - old_load += tickless_load; - } - new_load = this_load; - /* - * Round up the averaging division if load is increasing. This - * prevents us from getting stuck on 9 if the load is 10, for - * example. - */ - if (new_load > old_load) - new_load += scale - 1; - - this_rq->cpu_load[i] = (old_load * (scale - 1) + new_load) >> i; + old_load = this_rq->cpu_load[i]; + old_load = decay_load_missed(old_load, pending_updates - 1, i); + if (tickless_load) { + old_load -= decay_load_missed(tickless_load, pending_updates - 1, i); + /* + * old_load can never be a negative value because a + * decayed tickless_load cannot be greater than the + * original tickless_load. + */ + old_load += tickless_load; } + new_load = this_load; + /* + * Round up the averaging division if load is increasing. This + * prevents us from getting stuck on 9 if the load is 10, for + * example. + */ + if (new_load > old_load) + new_load += scale - 1; - sched_avg_update(this_rq); + this_rq->cpu_load[i] = (old_load * (scale - 1) + new_load) >> i; + } + + sched_avg_update(this_rq); } /* Used instead of source_load when we know the type == 0 */ static unsigned long weighted_cpuload(const int cpu) { - return cfs_rq_runnable_load_avg(&cpu_rq(cpu)->cfs); + return cfs_rq_runnable_load_avg(&cpu_rq(cpu)->cfs); } #ifdef CONFIG_NO_HZ_COMMON static void __update_cpu_load_nohz(struct rq *this_rq, - unsigned long curr_jiffies, - unsigned long load, - int active) + unsigned long curr_jiffies, + unsigned long load, + int active) { - unsigned long pending_updates; + unsigned long pending_updates; - pending_updates = curr_jiffies - this_rq->last_load_update_tick; - if (pending_updates) { - this_rq->last_load_update_tick = curr_jiffies; - /* - * In the regular NOHZ case, we were idle, this means load 0. - * In the NOHZ_FULL case, we were non-idle, we should consider - * its weighted load. - */ - __update_cpu_load(this_rq, load, pending_updates, active); - } + pending_updates = curr_jiffies - this_rq->last_load_update_tick; + if (pending_updates) { + this_rq->last_load_update_tick = curr_jiffies; + /* + * In the regular NOHZ case, we were idle, this means load 0. + * In the NOHZ_FULL case, we were non-idle, we should consider + * its weighted load. + */ + __update_cpu_load(this_rq, load, pending_updates, active); + } } /* @@ -4613,13 +4618,13 @@ static void __update_cpu_load_nohz(struct rq *this_rq, */ static void update_cpu_load_idle(struct rq *this_rq) { - /* - * bail if there's load or we're actually up-to-date. - */ - if (weighted_cpuload(cpu_of(this_rq))) - return; + /* + * bail if there's load or we're actually up-to-date. + */ + if (weighted_cpuload(cpu_of(this_rq))) + return; - __update_cpu_load_nohz(this_rq, READ_ONCE(jiffies), 0, 0); + __update_cpu_load_nohz(this_rq, READ_ONCE(jiffies), 0, 0); } /* @@ -4627,16 +4632,16 @@ static void update_cpu_load_idle(struct rq *this_rq) */ void update_cpu_load_nohz(int active) { - struct rq *this_rq = this_rq(); - unsigned long curr_jiffies = READ_ONCE(jiffies); - unsigned long load = active ? weighted_cpuload(cpu_of(this_rq)) : 0; + struct rq *this_rq = this_rq(); + unsigned long curr_jiffies = READ_ONCE(jiffies); + unsigned long load = active ? weighted_cpuload(cpu_of(this_rq)) : 0; - if (curr_jiffies == this_rq->last_load_update_tick) - return; + if (curr_jiffies == this_rq->last_load_update_tick) + return; - raw_spin_lock(&this_rq->lock); - __update_cpu_load_nohz(this_rq, curr_jiffies, load, active); - raw_spin_unlock(&this_rq->lock); + raw_spin_lock(&this_rq->lock); + __update_cpu_load_nohz(this_rq, curr_jiffies, load, active); + raw_spin_unlock(&this_rq->lock); } #endif /* CONFIG_NO_HZ */ @@ -4645,12 +4650,12 @@ void update_cpu_load_nohz(int active) */ void update_cpu_load_active(struct rq *this_rq) { - unsigned long load = weighted_cpuload(cpu_of(this_rq)); - /* - * See the mess around update_cpu_load_idle() / update_cpu_load_nohz(). - */ - this_rq->last_load_update_tick = jiffies; - __update_cpu_load(this_rq, load, 1, 1); + unsigned long load = weighted_cpuload(cpu_of(this_rq)); + /* + * See the mess around update_cpu_load_idle() / update_cpu_load_nohz(). + */ + this_rq->last_load_update_tick = jiffies; + __update_cpu_load(this_rq, load, 1, 1); } /* @@ -4662,13 +4667,13 @@ void update_cpu_load_active(struct rq *this_rq) */ static unsigned long source_load(int cpu, int type) { - struct rq *rq = cpu_rq(cpu); - unsigned long total = weighted_cpuload(cpu); + struct rq *rq = cpu_rq(cpu); + unsigned long total = weighted_cpuload(cpu); - if (type == 0 || !sched_feat(LB_BIAS)) - return total; + if (type == 0 || !sched_feat(LB_BIAS)) + return total; - return min(rq->cpu_load[type-1], total); + return min(rq->cpu_load[type-1], total); } /* @@ -4677,75 +4682,75 @@ static unsigned long source_load(int cpu, int type) */ static unsigned long target_load(int cpu, int type) { - struct rq *rq = cpu_rq(cpu); - unsigned long total = weighted_cpuload(cpu); + struct rq *rq = cpu_rq(cpu); + unsigned long total = weighted_cpuload(cpu); - if (type == 0 || !sched_feat(LB_BIAS)) - return total; + if (type == 0 || !sched_feat(LB_BIAS)) + return total; - return max(rq->cpu_load[type-1], total); + return max(rq->cpu_load[type-1], total); } static unsigned long capacity_of(int cpu) { - return cpu_rq(cpu)->cpu_capacity; + return cpu_rq(cpu)->cpu_capacity; } static unsigned long capacity_orig_of(int cpu) { - return cpu_rq(cpu)->cpu_capacity_orig; + return cpu_rq(cpu)->cpu_capacity_orig; } static unsigned long cpu_avg_load_per_task(int cpu) { - struct rq *rq = cpu_rq(cpu); - unsigned long nr_running = READ_ONCE(rq->cfs.h_nr_running); - unsigned long load_avg = weighted_cpuload(cpu); + struct rq *rq = cpu_rq(cpu); + unsigned long nr_running = READ_ONCE(rq->cfs.h_nr_running); + unsigned long load_avg = weighted_cpuload(cpu); - if (nr_running) - return load_avg / nr_running; + if (nr_running) + return load_avg / nr_running; - return 0; + return 0; } static void record_wakee(struct task_struct *p) { - /* - * Rough decay (wiping) for cost saving, don't worry - * about the boundary, really active task won't care - * about the loss. - */ - if (time_after(jiffies, current->wakee_flip_decay_ts + HZ)) { - current->wakee_flips >>= 1; - current->wakee_flip_decay_ts = jiffies; - } + /* + * Rough decay (wiping) for cost saving, don't worry + * about the boundary, really active task won't care + * about the loss. + */ + if (time_after(jiffies, current->wakee_flip_decay_ts + HZ)) { + current->wakee_flips >>= 1; + current->wakee_flip_decay_ts = jiffies; + } - if (current->last_wakee != p) { - current->last_wakee = p; - current->wakee_flips++; - } + if (current->last_wakee != p) { + current->last_wakee = p; + current->wakee_flips++; + } } static void task_waking_fair(struct task_struct *p) { - struct sched_entity *se = &p->se; - struct cfs_rq *cfs_rq = cfs_rq_of(se); - u64 min_vruntime; + struct sched_entity *se = &p->se; + struct cfs_rq *cfs_rq = cfs_rq_of(se); + u64 min_vruntime; #ifndef CONFIG_64BIT - u64 min_vruntime_copy; + u64 min_vruntime_copy; - do { - min_vruntime_copy = cfs_rq->min_vruntime_copy; - smp_rmb(); - min_vruntime = cfs_rq->min_vruntime; - } while (min_vruntime != min_vruntime_copy); -#else + do { + min_vruntime_copy = cfs_rq->min_vruntime_copy; + smp_rmb(); min_vruntime = cfs_rq->min_vruntime; + } while (min_vruntime != min_vruntime_copy); +#else + min_vruntime = cfs_rq->min_vruntime; #endif - se->vruntime -= min_vruntime; - record_wakee(p); + se->vruntime -= min_vruntime; + record_wakee(p); } #ifdef CONFIG_FAIR_GROUP_SCHED @@ -4763,7 +4768,7 @@ static void task_waking_fair(struct task_struct *p) * Given a runqueue weight distribution (rw_i) we can compute a shares * distribution (s_i) using: * - * s_i = rw_i / \Sum rw_j (1) + * s_i = rw_i / \Sum rw_j (1) * * Suppose we have 4 CPUs and our @tg is a direct child of the root group and * has 7 equal weight tasks, distributed as below (rw_i), with the resulting @@ -4780,7 +4785,7 @@ static void task_waking_fair(struct task_struct *p) * So for a change of @wl to the local @cpu with an overall group weight change * of @wl we can compute the new shares distribution (s'_i) using: * - * s'_i = (rw_i + @wl) / (@wg + \Sum rw_j) (2) + * s'_i = (rw_i + @wl) / (@wg + \Sum rw_j) (2) * * Suppose we're interested in CPUs 0 and 1, and want to compute the load * differences in waking a task to CPU 0. The additional task changes the @@ -4791,7 +4796,7 @@ static void task_waking_fair(struct task_struct *p) * * We can then compute the difference in effective weight by using: * - * dw_i = S * (s'_i - s_i) (3) + * dw_i = S * (s'_i - s_i) (3) * * Where 'S' is the group weight as seen by its parent. * @@ -4801,64 +4806,64 @@ static void task_waking_fair(struct task_struct *p) */ static long effective_load(struct task_group *tg, int cpu, long wl, long wg) { - struct sched_entity *se = tg->se[cpu]; - - if (!tg->parent) /* the trivial, non-cgroup case */ - return wl; - - for_each_sched_entity(se) { - long w, W; - - tg = se->my_q->tg; - - /* - * W = @wg + \Sum rw_j - */ - W = wg + calc_tg_weight(tg, se->my_q); - - /* - * w = rw_i + @wl - */ - w = cfs_rq_load_avg(se->my_q) + wl; - - /* - * wl = S * s'_i; see (2) - */ - if (W > 0 && w < W) - wl = (w * (long)tg->shares) / W; - else - wl = tg->shares; - - /* - * Per the above, wl is the new se->load.weight value; since - * those are clipped to [MIN_SHARES, ...) do so now. See - * calc_cfs_shares(). - */ - if (wl < MIN_SHARES) - wl = MIN_SHARES; - - /* - * wl = dw_i = S * (s'_i - s_i); see (3) - */ - wl -= se->avg.load_avg; - - /* - * Recursively apply this logic to all parent groups to compute - * the final effective load change on the root group. Since - * only the @tg group gets extra weight, all parent groups can - * only redistribute existing shares. @wl is the shift in shares - * resulting from this level per the above. - */ - wg = 0; - } + struct sched_entity *se = tg->se[cpu]; + if (!tg->parent) /* the trivial, non-cgroup case */ return wl; + + for_each_sched_entity(se) { + long w, W; + + tg = se->my_q->tg; + + /* + * W = @wg + \Sum rw_j + */ + W = wg + calc_tg_weight(tg, se->my_q); + + /* + * w = rw_i + @wl + */ + w = cfs_rq_load_avg(se->my_q) + wl; + + /* + * wl = S * s'_i; see (2) + */ + if (W > 0 && w < W) + wl = (w * (long)tg->shares) / W; + else + wl = tg->shares; + + /* + * Per the above, wl is the new se->load.weight value; since + * those are clipped to [MIN_SHARES, ...) do so now. See + * calc_cfs_shares(). + */ + if (wl < MIN_SHARES) + wl = MIN_SHARES; + + /* + * wl = dw_i = S * (s'_i - s_i); see (3) + */ + wl -= se->avg.load_avg; + + /* + * Recursively apply this logic to all parent groups to compute + * the final effective load change on the root group. Since + * only the @tg group gets extra weight, all parent groups can + * only redistribute existing shares. @wl is the shift in shares + * resulting from this level per the above. + */ + wg = 0; + } + + return wl; } #else static long effective_load(struct task_group *tg, int cpu, long wl, long wg) { - return wl; + return wl; } #endif @@ -4877,81 +4882,81 @@ static long effective_load(struct task_group *tg, int cpu, long wl, long wg) */ static int wake_wide(struct task_struct *p) { - unsigned int master = current->wakee_flips; - unsigned int slave = p->wakee_flips; - int factor = this_cpu_read(sd_llc_size); + unsigned int master = current->wakee_flips; + unsigned int slave = p->wakee_flips; + int factor = this_cpu_read(sd_llc_size); - if (master < slave) - swap(master, slave); - if (slave < factor || master < slave * factor) - return 0; - return 1; + if (master < slave) + swap(master, slave); + if (slave < factor || master < slave * factor) + return 0; + return 1; } static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync) { - s64 this_load, load; - s64 this_eff_load, prev_eff_load; - int idx, this_cpu, prev_cpu; - struct task_group *tg; - unsigned long weight; - int balanced; + s64 this_load, load; + s64 this_eff_load, prev_eff_load; + int idx, this_cpu, prev_cpu; + struct task_group *tg; + unsigned long weight; + int balanced; - idx = sd->wake_idx; - this_cpu = smp_processor_id(); - prev_cpu = task_cpu(p); - load = source_load(prev_cpu, idx); - this_load = target_load(this_cpu, idx); + idx = sd->wake_idx; + this_cpu = smp_processor_id(); + prev_cpu = task_cpu(p); + load = source_load(prev_cpu, idx); + this_load = target_load(this_cpu, idx); - /* - * If sync wakeup then subtract the (maximum possible) - * effect of the currently running task from the load - * of the current CPU: - */ - if (sync) { - tg = task_group(current); - weight = current->se.avg.load_avg; + /* + * If sync wakeup then subtract the (maximum possible) + * effect of the currently running task from the load + * of the current CPU: + */ + if (sync) { + tg = task_group(current); + weight = current->se.avg.load_avg; - this_load += effective_load(tg, this_cpu, -weight, -weight); - load += effective_load(tg, prev_cpu, 0, -weight); - } + this_load += effective_load(tg, this_cpu, -weight, -weight); + load += effective_load(tg, prev_cpu, 0, -weight); + } - tg = task_group(p); - weight = p->se.avg.load_avg; + tg = task_group(p); + weight = p->se.avg.load_avg; - /* - * In low-load situations, where prev_cpu is idle and this_cpu is idle - * due to the sync cause above having dropped this_load to 0, we'll - * always have an imbalance, but there's really nothing you can do - * about that, so that's good too. - * - * Otherwise check if either cpus are near enough in load to allow this - * task to be woken on this_cpu. - */ - this_eff_load = 100; - this_eff_load *= capacity_of(prev_cpu); + /* + * In low-load situations, where prev_cpu is idle and this_cpu is idle + * due to the sync cause above having dropped this_load to 0, we'll + * always have an imbalance, but there's really nothing you can do + * about that, so that's good too. + * + * Otherwise check if either cpus are near enough in load to allow this + * task to be woken on this_cpu. + */ + this_eff_load = 100; + this_eff_load *= capacity_of(prev_cpu); - prev_eff_load = 100 + (sd->imbalance_pct - 100) / 2; - prev_eff_load *= capacity_of(this_cpu); + prev_eff_load = 100 + (sd->imbalance_pct - 100) / 2; + prev_eff_load *= capacity_of(this_cpu); - if (this_load > 0) { - this_eff_load *= this_load + - effective_load(tg, this_cpu, weight, weight); + if (this_load > 0) { + this_eff_load *= this_load + + effective_load(tg, this_cpu, weight, weight); - prev_eff_load *= load + effective_load(tg, prev_cpu, 0, weight); - } + prev_eff_load *= load + effective_load(tg, prev_cpu, 0, weight); + } - balanced = this_eff_load <= prev_eff_load; + balanced = this_eff_load <= prev_eff_load; - schedstat_inc(p, se.statistics.nr_wakeups_affine_attempts); + schedstat_inc(p, se.statistics.nr_wakeups_affine_attempts); - if (!balanced) - return 0; + if (!balanced) + return 0; - schedstat_inc(sd, ttwu_move_affine); - schedstat_inc(p, se.statistics.nr_wakeups_affine); + schedstat_inc(sd, ttwu_move_affine); + schedstat_inc(p, se.statistics.nr_wakeups_affine); - return 1; + return 1; } /* @@ -4960,56 +4965,56 @@ static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync) */ static struct sched_group * find_idlest_group(struct sched_domain *sd, struct task_struct *p, - int this_cpu, int sd_flag) + int this_cpu, int sd_flag) { - struct sched_group *idlest = NULL, *group = sd->groups; - unsigned long min_load = ULONG_MAX, this_load = 0; - int load_idx = sd->forkexec_idx; - int imbalance = 100 + (sd->imbalance_pct-100)/2; + struct sched_group *idlest = NULL, *group = sd->groups; + unsigned long min_load = ULONG_MAX, this_load = 0; + int load_idx = sd->forkexec_idx; + int imbalance = 100 + (sd->imbalance_pct-100)/2; - if (sd_flag & SD_BALANCE_WAKE) - load_idx = sd->wake_idx; + if (sd_flag & SD_BALANCE_WAKE) + load_idx = sd->wake_idx; - do { - unsigned long load, avg_load; - int local_group; - int i; + do { + unsigned long load, avg_load; + int local_group; + int i; - /* Skip over this group if it has no CPUs allowed */ - if (!cpumask_intersects(sched_group_cpus(group), - tsk_cpus_allowed(p))) - continue; + /* Skip over this group if it has no CPUs allowed */ + if (!cpumask_intersects(sched_group_cpus(group), + tsk_cpus_allowed(p))) + continue; - local_group = cpumask_test_cpu(this_cpu, - sched_group_cpus(group)); + local_group = cpumask_test_cpu(this_cpu, + sched_group_cpus(group)); - /* Tally up the load of all CPUs in the group */ - avg_load = 0; + /* Tally up the load of all CPUs in the group */ + avg_load = 0; - for_each_cpu(i, sched_group_cpus(group)) { - /* Bias balancing toward cpus of our domain */ - if (local_group) - load = source_load(i, load_idx); - else - load = target_load(i, load_idx); + for_each_cpu(i, sched_group_cpus(group)) { + /* Bias balancing toward cpus of our domain */ + if (local_group) + load = source_load(i, load_idx); + else + load = target_load(i, load_idx); - avg_load += load; - } + avg_load += load; + } - /* Adjust by relative CPU capacity of the group */ - avg_load = (avg_load * SCHED_CAPACITY_SCALE) / group->sgc->capacity; + /* Adjust by relative CPU capacity of the group */ + avg_load = (avg_load * SCHED_CAPACITY_SCALE) / group->sgc->capacity; - if (local_group) { - this_load = avg_load; - } else if (avg_load < min_load) { - min_load = avg_load; - idlest = group; - } - } while (group = group->next, group != sd->groups); + if (local_group) { + this_load = avg_load; + } else if (avg_load < min_load) { + min_load = avg_load; + idlest = group; + } + } while (group = group->next, group != sd->groups); - if (!idlest || 100*this_load < imbalance*min_load) - return NULL; - return idlest; + if (!idlest || 100*this_load < imbalance*min_load) + return NULL; + return idlest; } /* @@ -5018,47 +5023,47 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, static int find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu) { - unsigned long load, min_load = ULONG_MAX; - unsigned int min_exit_latency = UINT_MAX; - u64 latest_idle_timestamp = 0; - int least_loaded_cpu = this_cpu; - int shallowest_idle_cpu = -1; - int i; + unsigned long load, min_load = ULONG_MAX; + unsigned int min_exit_latency = UINT_MAX; + u64 latest_idle_timestamp = 0; + int least_loaded_cpu = this_cpu; + int shallowest_idle_cpu = -1; + int i; - /* Traverse only the allowed CPUs */ - for_each_cpu_and(i, sched_group_cpus(group), tsk_cpus_allowed(p)) { - if (idle_cpu(i)) { - struct rq *rq = cpu_rq(i); - struct cpuidle_state *idle = idle_get_state(rq); - if (idle && idle->exit_latency < min_exit_latency) { - /* - * We give priority to a CPU whose idle state - * has the smallest exit latency irrespective - * of any idle timestamp. - */ - min_exit_latency = idle->exit_latency; - latest_idle_timestamp = rq->idle_stamp; - shallowest_idle_cpu = i; - } else if ((!idle || idle->exit_latency == min_exit_latency) && - rq->idle_stamp > latest_idle_timestamp) { - /* - * If equal or no active idle state, then - * the most recently idled CPU might have - * a warmer cache. - */ - latest_idle_timestamp = rq->idle_stamp; - shallowest_idle_cpu = i; - } - } else if (shallowest_idle_cpu == -1) { - load = weighted_cpuload(i); - if (load < min_load || (load == min_load && i == this_cpu)) { - min_load = load; - least_loaded_cpu = i; - } - } + /* Traverse only the allowed CPUs */ + for_each_cpu_and(i, sched_group_cpus(group), tsk_cpus_allowed(p)) { + if (idle_cpu(i)) { + struct rq *rq = cpu_rq(i); + struct cpuidle_state *idle = idle_get_state(rq); + if (idle && idle->exit_latency < min_exit_latency) { + /* + * We give priority to a CPU whose idle state + * has the smallest exit latency irrespective + * of any idle timestamp. + */ + min_exit_latency = idle->exit_latency; + latest_idle_timestamp = rq->idle_stamp; + shallowest_idle_cpu = i; + } else if ((!idle || idle->exit_latency == min_exit_latency) && + rq->idle_stamp > latest_idle_timestamp) { + /* + * If equal or no active idle state, then + * the most recently idled CPU might have + * a warmer cache. + */ + latest_idle_timestamp = rq->idle_stamp; + shallowest_idle_cpu = i; + } + } else if (shallowest_idle_cpu == -1) { + load = weighted_cpuload(i); + if (load < min_load || (load == min_load && i == this_cpu)) { + min_load = load; + least_loaded_cpu = i; + } } + } - return shallowest_idle_cpu != -1 ? shallowest_idle_cpu : least_loaded_cpu; + return shallowest_idle_cpu != -1 ? shallowest_idle_cpu : least_loaded_cpu; } /* @@ -5066,61 +5071,61 @@ find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu) */ static int select_idle_sibling(struct task_struct *p, int target) { - struct sched_domain *sd; - struct sched_group *sg; - int i = task_cpu(p); + struct sched_domain *sd; + struct sched_group *sg; + int i = task_cpu(p); - if (idle_cpu(target)) - return target; - - /* - * If the prevous cpu is cache affine and idle, don't be stupid. - */ - if (i != target && cpus_share_cache(i, target) && idle_cpu(i)) - return i; - - /* - * Otherwise, iterate the domains and find an eligible idle cpu. - * - * A completely idle sched group at higher domains is more - * desirable than an idle group at a lower level, because lower - * domains have smaller groups and usually share hardware - * resources which causes tasks to contend on them, e.g. x86 - * hyperthread siblings in the lowest domain (SMT) can contend - * on the shared cpu pipeline. - * - * However, while we prefer idle groups at higher domains - * finding an idle cpu at the lowest domain is still better than - * returning 'target', which we've already established, isn't - * idle. - */ - sd = rcu_dereference(per_cpu(sd_llc, target)); - for_each_lower_domain(sd) { - sg = sd->groups; - do { - if (!cpumask_intersects(sched_group_cpus(sg), - tsk_cpus_allowed(p))) - goto next; - - /* Ensure the entire group is idle */ - for_each_cpu(i, sched_group_cpus(sg)) { - if (i == target || !idle_cpu(i)) - goto next; - } - - /* - * It doesn't matter which cpu we pick, the - * whole group is idle. - */ - target = cpumask_first_and(sched_group_cpus(sg), - tsk_cpus_allowed(p)); - goto done; -next: - sg = sg->next; - } while (sg != sd->groups); - } -done: + if (idle_cpu(target)) return target; + + /* + * If the prevous cpu is cache affine and idle, don't be stupid. + */ + if (i != target && cpus_share_cache(i, target) && idle_cpu(i)) + return i; + + /* + * Otherwise, iterate the domains and find an eligible idle cpu. + * + * A completely idle sched group at higher domains is more + * desirable than an idle group at a lower level, because lower + * domains have smaller groups and usually share hardware + * resources which causes tasks to contend on them, e.g. x86 + * hyperthread siblings in the lowest domain (SMT) can contend + * on the shared cpu pipeline. + * + * However, while we prefer idle groups at higher domains + * finding an idle cpu at the lowest domain is still better than + * returning 'target', which we've already established, isn't + * idle. + */ + sd = rcu_dereference(per_cpu(sd_llc, target)); + for_each_lower_domain(sd) { + sg = sd->groups; + do { + if (!cpumask_intersects(sched_group_cpus(sg), + tsk_cpus_allowed(p))) + goto next; + + /* Ensure the entire group is idle */ + for_each_cpu(i, sched_group_cpus(sg)) { + if (i == target || !idle_cpu(i)) + goto next; + } + + /* + * It doesn't matter which cpu we pick, the + * whole group is idle. + */ + target = cpumask_first_and(sched_group_cpus(sg), + tsk_cpus_allowed(p)); + goto done; +next: + sg = sg->next; + } while (sg != sd->groups); + } +done: + return target; } /* @@ -5151,10 +5156,10 @@ done: */ static int cpu_util(int cpu) { - unsigned long util = cpu_rq(cpu)->cfs.avg.util_avg; - unsigned long capacity = capacity_orig_of(cpu); + unsigned long util = cpu_rq(cpu)->cfs.avg.util_avg; + unsigned long capacity = capacity_orig_of(cpu); - return (util >= capacity) ? capacity : util; + return (util >= capacity) ? capacity : util; } /* @@ -5172,83 +5177,83 @@ static int cpu_util(int cpu) static int select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_flags) { - struct sched_domain *tmp, *affine_sd = NULL, *sd = NULL; - int cpu = smp_processor_id(); - int new_cpu = prev_cpu; - int want_affine = 0; - int sync = wake_flags & WF_SYNC; + struct sched_domain *tmp, *affine_sd = NULL, *sd = NULL; + int cpu = smp_processor_id(); + int new_cpu = prev_cpu; + int want_affine = 0; + int sync = wake_flags & WF_SYNC; - if (sd_flag & SD_BALANCE_WAKE) - want_affine = !wake_wide(p) && cpumask_test_cpu(cpu, tsk_cpus_allowed(p)); + if (sd_flag & SD_BALANCE_WAKE) + want_affine = !wake_wide(p) && cpumask_test_cpu(cpu, tsk_cpus_allowed(p)); - rcu_read_lock(); + rcu_read_lock(); + for_each_domain(cpu, tmp) { + if (!(tmp->flags & SD_LOAD_BALANCE)) + break; + + /* + * If both cpu and prev_cpu are part of this domain, + * cpu is a valid SD_WAKE_AFFINE target. + */ + if (want_affine && (tmp->flags & SD_WAKE_AFFINE) && + cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) { + affine_sd = tmp; + break; + } + + if (tmp->flags & sd_flag) + sd = tmp; + else if (!want_affine) + break; + } + + if (affine_sd) { + sd = NULL; /* Prefer wake_affine over balance flags */ + if (cpu != prev_cpu && wake_affine(affine_sd, p, sync)) + new_cpu = cpu; + } + + if (!sd) { + if (sd_flag & SD_BALANCE_WAKE) /* XXX always ? */ + new_cpu = select_idle_sibling(p, new_cpu); + + } else while (sd) { + struct sched_group *group; + int weight; + + if (!(sd->flags & sd_flag)) { + sd = sd->child; + continue; + } + + group = find_idlest_group(sd, p, cpu, sd_flag); + if (!group) { + sd = sd->child; + continue; + } + + new_cpu = find_idlest_cpu(group, p, cpu); + if (new_cpu == -1 || new_cpu == cpu) { + /* Now try balancing at a lower domain level of cpu */ + sd = sd->child; + continue; + } + + /* Now try balancing at a lower domain level of new_cpu */ + cpu = new_cpu; + weight = sd->span_weight; + sd = NULL; for_each_domain(cpu, tmp) { - if (!(tmp->flags & SD_LOAD_BALANCE)) - break; - - /* - * If both cpu and prev_cpu are part of this domain, - * cpu is a valid SD_WAKE_AFFINE target. - */ - if (want_affine && (tmp->flags & SD_WAKE_AFFINE) && - cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) { - affine_sd = tmp; - break; - } - - if (tmp->flags & sd_flag) - sd = tmp; - else if (!want_affine) - break; + if (weight <= tmp->span_weight) + break; + if (tmp->flags & sd_flag) + sd = tmp; } + /* while loop will break here if sd == NULL */ + } + rcu_read_unlock(); - if (affine_sd) { - sd = NULL; /* Prefer wake_affine over balance flags */ - if (cpu != prev_cpu && wake_affine(affine_sd, p, sync)) - new_cpu = cpu; - } - - if (!sd) { - if (sd_flag & SD_BALANCE_WAKE) /* XXX always ? */ - new_cpu = select_idle_sibling(p, new_cpu); - - } else while (sd) { - struct sched_group *group; - int weight; - - if (!(sd->flags & sd_flag)) { - sd = sd->child; - continue; - } - - group = find_idlest_group(sd, p, cpu, sd_flag); - if (!group) { - sd = sd->child; - continue; - } - - new_cpu = find_idlest_cpu(group, p, cpu); - if (new_cpu == -1 || new_cpu == cpu) { - /* Now try balancing at a lower domain level of cpu */ - sd = sd->child; - continue; - } - - /* Now try balancing at a lower domain level of new_cpu */ - cpu = new_cpu; - weight = sd->span_weight; - sd = NULL; - for_each_domain(cpu, tmp) { - if (weight <= tmp->span_weight) - break; - if (tmp->flags & sd_flag) - sd = tmp; - } - /* while loop will break here if sd == NULL */ - } - rcu_read_unlock(); - - return new_cpu; + return new_cpu; } /* @@ -5258,56 +5263,56 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_f */ static void migrate_task_rq_fair(struct task_struct *p) { - /* - * We are supposed to update the task to "current" time, then its up to date - * and ready to go to new CPU/cfs_rq. But we have difficulty in getting - * what current time is, so simply throw away the out-of-date time. This - * will result in the wakee task is less decayed, but giving the wakee more - * load sounds not bad. - */ - remove_entity_load_avg(&p->se); + /* + * We are supposed to update the task to "current" time, then its up to date + * and ready to go to new CPU/cfs_rq. But we have difficulty in getting + * what current time is, so simply throw away the out-of-date time. This + * will result in the wakee task is less decayed, but giving the wakee more + * load sounds not bad. + */ + remove_entity_load_avg(&p->se); - /* Tell new CPU we are migrated */ - p->se.avg.last_update_time = 0; + /* Tell new CPU we are migrated */ + p->se.avg.last_update_time = 0; - /* We have migrated, no longer consider this task hot */ - p->se.exec_start = 0; + /* We have migrated, no longer consider this task hot */ + p->se.exec_start = 0; } static void task_dead_fair(struct task_struct *p) { - remove_entity_load_avg(&p->se); + remove_entity_load_avg(&p->se); } #endif /* CONFIG_SMP */ static unsigned long wakeup_gran(struct sched_entity *curr, struct sched_entity *se) { - unsigned long gran = sysctl_sched_wakeup_granularity; + unsigned long gran = sysctl_sched_wakeup_granularity; - /* - * Since its curr running now, convert the gran from real-time - * to virtual-time in his units. - * - * By using 'se' instead of 'curr' we penalize light tasks, so - * they get preempted easier. That is, if 'se' < 'curr' then - * the resulting gran will be larger, therefore penalizing the - * lighter, if otoh 'se' > 'curr' then the resulting gran will - * be smaller, again penalizing the lighter task. - * - * This is especially important for buddies when the leftmost - * task is higher priority than the buddy. - */ - return calc_delta_fair(gran, se); + /* + * Since its curr running now, convert the gran from real-time + * to virtual-time in his units. + * + * By using 'se' instead of 'curr' we penalize light tasks, so + * they get preempted easier. That is, if 'se' < 'curr' then + * the resulting gran will be larger, therefore penalizing the + * lighter, if otoh 'se' > 'curr' then the resulting gran will + * be smaller, again penalizing the lighter task. + * + * This is especially important for buddies when the leftmost + * task is higher priority than the buddy. + */ + return calc_delta_fair(gran, se); } /* * Should 'se' preempt 'curr'. * - * |s1 - * |s2 + * |s1 + * |s2 * |s3 - * g + * g * |<--->|c * * w(c, s1) = -1 @@ -5318,40 +5323,40 @@ wakeup_gran(struct sched_entity *curr, struct sched_entity *se) static int wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se) { - s64 gran, vdiff = curr->vruntime - se->vruntime; + s64 gran, vdiff = curr->vruntime - se->vruntime; - if (vdiff <= 0) - return -1; + if (vdiff <= 0) + return -1; - gran = wakeup_gran(curr, se); - if (vdiff > gran) - return 1; + gran = wakeup_gran(curr, se); + if (vdiff > gran) + return 1; - return 0; + return 0; } static void set_last_buddy(struct sched_entity *se) { - if (entity_is_task(se) && unlikely(task_of(se)->policy == SCHED_IDLE)) - return; + if (entity_is_task(se) && unlikely(task_of(se)->policy == SCHED_IDLE)) + return; - for_each_sched_entity(se) - cfs_rq_of(se)->last = se; + for_each_sched_entity(se) + cfs_rq_of(se)->last = se; } static void set_next_buddy(struct sched_entity *se) { - if (entity_is_task(se) && unlikely(task_of(se)->policy == SCHED_IDLE)) - return; + if (entity_is_task(se) && unlikely(task_of(se)->policy == SCHED_IDLE)) + return; - for_each_sched_entity(se) - cfs_rq_of(se)->next = se; + for_each_sched_entity(se) + cfs_rq_of(se)->next = se; } static void set_skip_buddy(struct sched_entity *se) { - for_each_sched_entity(se) - cfs_rq_of(se)->skip = se; + for_each_sched_entity(se) + cfs_rq_of(se)->skip = se; } /* @@ -5359,216 +5364,216 @@ static void set_skip_buddy(struct sched_entity *se) */ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_flags) { - struct task_struct *curr = rq->curr; - struct sched_entity *se = &curr->se, *pse = &p->se; - struct cfs_rq *cfs_rq = task_cfs_rq(curr); - int scale = cfs_rq->nr_running >= sched_nr_latency; - int next_buddy_marked = 0; - - if (unlikely(se == pse)) - return; - - /* - * This is possible from callers such as attach_tasks(), in which we - * unconditionally check_prempt_curr() after an enqueue (which may have - * lead to a throttle). This both saves work and prevents false - * next-buddy nomination below. - */ - if (unlikely(throttled_hierarchy(cfs_rq_of(pse)))) - return; - - if (sched_feat(NEXT_BUDDY) && scale && !(wake_flags & WF_FORK)) { - set_next_buddy(pse); - next_buddy_marked = 1; - } - - /* - * We can come here with TIF_NEED_RESCHED already set from new task - * wake up path. - * - * Note: this also catches the edge-case of curr being in a throttled - * group (e.g. via set_curr_task), since update_curr() (in the - * enqueue of curr) will have resulted in resched being set. This - * prevents us from potentially nominating it as a false LAST_BUDDY - * below. - */ - if (test_tsk_need_resched(curr)) - return; - - /* Idle tasks are by definition preempted by non-idle tasks. */ - if (unlikely(curr->policy == SCHED_IDLE) && - likely(p->policy != SCHED_IDLE)) - goto preempt; - - /* - * Batch and idle tasks do not preempt non-idle tasks (their preemption - * is driven by the tick): - */ - if (unlikely(p->policy != SCHED_NORMAL) || !sched_feat(WAKEUP_PREEMPTION)) - return; - - find_matching_se(&se, &pse); - update_curr(cfs_rq_of(se)); - BUG_ON(!pse); - if (wakeup_preempt_entity(se, pse) == 1) { - /* - * Bias pick_next to pick the sched entity that is - * triggering this preemption. - */ - if (!next_buddy_marked) - set_next_buddy(pse); - goto preempt; - } + struct task_struct *curr = rq->curr; + struct sched_entity *se = &curr->se, *pse = &p->se; + struct cfs_rq *cfs_rq = task_cfs_rq(curr); + int scale = cfs_rq->nr_running >= sched_nr_latency; + int next_buddy_marked = 0; + if (unlikely(se == pse)) return; -preempt: - resched_curr(rq); - /* - * Only set the backward buddy when the current task is still - * on the rq. This can happen when a wakeup gets interleaved - * with schedule on the ->pre_schedule() or idle_balance() - * point, either of which can * drop the rq lock. - * - * Also, during early boot the idle thread is in the fair class, - * for obvious reasons its a bad idea to schedule back to it. - */ - if (unlikely(!se->on_rq || curr == rq->idle)) - return; + /* + * This is possible from callers such as attach_tasks(), in which we + * unconditionally check_prempt_curr() after an enqueue (which may have + * lead to a throttle). This both saves work and prevents false + * next-buddy nomination below. + */ + if (unlikely(throttled_hierarchy(cfs_rq_of(pse)))) + return; - if (sched_feat(LAST_BUDDY) && scale && entity_is_task(se)) - set_last_buddy(se); + if (sched_feat(NEXT_BUDDY) && scale && !(wake_flags & WF_FORK)) { + set_next_buddy(pse); + next_buddy_marked = 1; + } + + /* + * We can come here with TIF_NEED_RESCHED already set from new task + * wake up path. + * + * Note: this also catches the edge-case of curr being in a throttled + * group (e.g. via set_curr_task), since update_curr() (in the + * enqueue of curr) will have resulted in resched being set. This + * prevents us from potentially nominating it as a false LAST_BUDDY + * below. + */ + if (test_tsk_need_resched(curr)) + return; + + /* Idle tasks are by definition preempted by non-idle tasks. */ + if (unlikely(curr->policy == SCHED_IDLE) && + likely(p->policy != SCHED_IDLE)) + goto preempt; + + /* + * Batch and idle tasks do not preempt non-idle tasks (their preemption + * is driven by the tick): + */ + if (unlikely(p->policy != SCHED_NORMAL) || !sched_feat(WAKEUP_PREEMPTION)) + return; + + find_matching_se(&se, &pse); + update_curr(cfs_rq_of(se)); + BUG_ON(!pse); + if (wakeup_preempt_entity(se, pse) == 1) { + /* + * Bias pick_next to pick the sched entity that is + * triggering this preemption. + */ + if (!next_buddy_marked) + set_next_buddy(pse); + goto preempt; + } + + return; + +preempt: + resched_curr(rq); + /* + * Only set the backward buddy when the current task is still + * on the rq. This can happen when a wakeup gets interleaved + * with schedule on the ->pre_schedule() or idle_balance() + * point, either of which can * drop the rq lock. + * + * Also, during early boot the idle thread is in the fair class, + * for obvious reasons its a bad idea to schedule back to it. + */ + if (unlikely(!se->on_rq || curr == rq->idle)) + return; + + if (sched_feat(LAST_BUDDY) && scale && entity_is_task(se)) + set_last_buddy(se); } static struct task_struct * pick_next_task_fair(struct rq *rq, struct task_struct *prev) { - struct cfs_rq *cfs_rq = &rq->cfs; - struct sched_entity *se; - struct task_struct *p; - int new_tasks; + struct cfs_rq *cfs_rq = &rq->cfs; + struct sched_entity *se; + struct task_struct *p; + int new_tasks; again: #ifdef CONFIG_FAIR_GROUP_SCHED - if (!cfs_rq->nr_running) - goto idle; + if (!cfs_rq->nr_running) + goto idle; - if (prev->sched_class != &fair_sched_class) + if (prev->sched_class != &fair_sched_class) + goto simple; + + /* + * Because of the set_next_buddy() in dequeue_task_fair() it is rather + * likely that a next task is from the same cgroup as the current. + * + * Therefore attempt to avoid putting and setting the entire cgroup + * hierarchy, only change the part that actually changes. + */ + + do { + struct sched_entity *curr = cfs_rq->curr; + + /* + * Since we got here without doing put_prev_entity() we also + * have to consider cfs_rq->curr. If it is still a runnable + * entity, update_curr() will update its vruntime, otherwise + * forget we've ever seen it. + */ + if (curr) { + if (curr->on_rq) + update_curr(cfs_rq); + else + curr = NULL; + + /* + * This call to check_cfs_rq_runtime() will do the + * throttle and dequeue its entity in the parent(s). + * Therefore the 'simple' nr_running test will indeed + * be correct. + */ + if (unlikely(check_cfs_rq_runtime(cfs_rq))) goto simple; - - /* - * Because of the set_next_buddy() in dequeue_task_fair() it is rather - * likely that a next task is from the same cgroup as the current. - * - * Therefore attempt to avoid putting and setting the entire cgroup - * hierarchy, only change the part that actually changes. - */ - - do { - struct sched_entity *curr = cfs_rq->curr; - - /* - * Since we got here without doing put_prev_entity() we also - * have to consider cfs_rq->curr. If it is still a runnable - * entity, update_curr() will update its vruntime, otherwise - * forget we've ever seen it. - */ - if (curr) { - if (curr->on_rq) - update_curr(cfs_rq); - else - curr = NULL; - - /* - * This call to check_cfs_rq_runtime() will do the - * throttle and dequeue its entity in the parent(s). - * Therefore the 'simple' nr_running test will indeed - * be correct. - */ - if (unlikely(check_cfs_rq_runtime(cfs_rq))) - goto simple; - } - - se = pick_next_entity(cfs_rq, curr); - cfs_rq = group_cfs_rq(se); - } while (cfs_rq); - - p = task_of(se); - - /* - * Since we haven't yet done put_prev_entity and if the selected task - * is a different task than we started out with, try and touch the - * least amount of cfs_rqs. - */ - if (prev != p) { - struct sched_entity *pse = &prev->se; - - while (!(cfs_rq = is_same_group(se, pse))) { - int se_depth = se->depth; - int pse_depth = pse->depth; - - if (se_depth <= pse_depth) { - put_prev_entity(cfs_rq_of(pse), pse); - pse = parent_entity(pse); - } - if (se_depth >= pse_depth) { - set_next_entity(cfs_rq_of(se), se); - se = parent_entity(se); - } - } - - put_prev_entity(cfs_rq, pse); - set_next_entity(cfs_rq, se); } - if (hrtick_enabled(rq)) - hrtick_start_fair(rq, p); + se = pick_next_entity(cfs_rq, curr); + cfs_rq = group_cfs_rq(se); + } while (cfs_rq); - return p; + p = task_of(se); + + /* + * Since we haven't yet done put_prev_entity and if the selected task + * is a different task than we started out with, try and touch the + * least amount of cfs_rqs. + */ + if (prev != p) { + struct sched_entity *pse = &prev->se; + + while (!(cfs_rq = is_same_group(se, pse))) { + int se_depth = se->depth; + int pse_depth = pse->depth; + + if (se_depth <= pse_depth) { + put_prev_entity(cfs_rq_of(pse), pse); + pse = parent_entity(pse); + } + if (se_depth >= pse_depth) { + set_next_entity(cfs_rq_of(se), se); + se = parent_entity(se); + } + } + + put_prev_entity(cfs_rq, pse); + set_next_entity(cfs_rq, se); + } + + if (hrtick_enabled(rq)) + hrtick_start_fair(rq, p); + + return p; simple: - cfs_rq = &rq->cfs; + cfs_rq = &rq->cfs; #endif - if (!cfs_rq->nr_running) - goto idle; + if (!cfs_rq->nr_running) + goto idle; - put_prev_task(rq, prev); + put_prev_task(rq, prev); - do { - se = pick_next_entity(cfs_rq, NULL); - set_next_entity(cfs_rq, se); - cfs_rq = group_cfs_rq(se); - } while (cfs_rq); + do { + se = pick_next_entity(cfs_rq, NULL); + set_next_entity(cfs_rq, se); + cfs_rq = group_cfs_rq(se); + } while (cfs_rq); - p = task_of(se); + p = task_of(se); - if (hrtick_enabled(rq)) - hrtick_start_fair(rq, p); + if (hrtick_enabled(rq)) + hrtick_start_fair(rq, p); - return p; + return p; idle: - /* - * This is OK, because current is on_cpu, which avoids it being picked - * for load-balance and preemption/IRQs are still disabled avoiding - * further scheduler activity on it and we're being very careful to - * re-start the picking loop. - */ - lockdep_unpin_lock(&rq->lock); - new_tasks = idle_balance(rq); - lockdep_pin_lock(&rq->lock); - /* - * Because idle_balance() releases (and re-acquires) rq->lock, it is - * possible for any higher priority task to appear. In that case we - * must re-start the pick_next_entity() loop. - */ - if (new_tasks < 0) - return RETRY_TASK; + /* + * This is OK, because current is on_cpu, which avoids it being picked + * for load-balance and preemption/IRQs are still disabled avoiding + * further scheduler activity on it and we're being very careful to + * re-start the picking loop. + */ + lockdep_unpin_lock(&rq->lock); + new_tasks = idle_balance(rq); + lockdep_pin_lock(&rq->lock); + /* + * Because idle_balance() releases (and re-acquires) rq->lock, it is + * possible for any higher priority task to appear. In that case we + * must re-start the pick_next_entity() loop. + */ + if (new_tasks < 0) + return RETRY_TASK; - if (new_tasks > 0) - goto again; + if (new_tasks > 0) + goto again; - return NULL; + return NULL; } /* @@ -5576,13 +5581,13 @@ idle: */ static void put_prev_task_fair(struct rq *rq, struct task_struct *prev) { - struct sched_entity *se = &prev->se; - struct cfs_rq *cfs_rq; + struct sched_entity *se = &prev->se; + struct cfs_rq *cfs_rq; - for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); - put_prev_entity(cfs_rq, se); - } + for_each_sched_entity(se) { + cfs_rq = cfs_rq_of(se); + put_prev_entity(cfs_rq, se); + } } /* @@ -5592,49 +5597,49 @@ static void put_prev_task_fair(struct rq *rq, struct task_struct *prev) */ static void yield_task_fair(struct rq *rq) { - struct task_struct *curr = rq->curr; - struct cfs_rq *cfs_rq = task_cfs_rq(curr); - struct sched_entity *se = &curr->se; + struct task_struct *curr = rq->curr; + struct cfs_rq *cfs_rq = task_cfs_rq(curr); + struct sched_entity *se = &curr->se; + /* + * Are we the only task in the tree? + */ + if (unlikely(rq->nr_running == 1)) + return; + + clear_buddies(cfs_rq, se); + + if (curr->policy != SCHED_BATCH) { + update_rq_clock(rq); /* - * Are we the only task in the tree? + * Update run-time statistics of the 'current'. */ - if (unlikely(rq->nr_running == 1)) - return; + update_curr(cfs_rq); + /* + * Tell update_rq_clock() that we've just updated, + * so we don't do microscopic update in schedule() + * and double the fastpath cost. + */ + rq_clock_skip_update(rq, true); + } - clear_buddies(cfs_rq, se); - - if (curr->policy != SCHED_BATCH) { - update_rq_clock(rq); - /* - * Update run-time statistics of the 'current'. - */ - update_curr(cfs_rq); - /* - * Tell update_rq_clock() that we've just updated, - * so we don't do microscopic update in schedule() - * and double the fastpath cost. - */ - rq_clock_skip_update(rq, true); - } - - set_skip_buddy(se); + set_skip_buddy(se); } static bool yield_to_task_fair(struct rq *rq, struct task_struct *p, bool preempt) { - struct sched_entity *se = &p->se; + struct sched_entity *se = &p->se; - /* throttled hierarchies are not runnable */ - if (!se->on_rq || throttled_hierarchy(cfs_rq_of(se))) - return false; + /* throttled hierarchies are not runnable */ + if (!se->on_rq || throttled_hierarchy(cfs_rq_of(se))) + return false; - /* Tell the scheduler that we'd really like pse to run next. */ - set_next_buddy(se); + /* Tell the scheduler that we'd really like pse to run next. */ + set_next_buddy(se); - yield_task_fair(rq); + yield_task_fair(rq); - return true; + return true; } #ifdef CONFIG_SMP @@ -5647,12 +5652,12 @@ static bool yield_to_task_fair(struct rq *rq, struct task_struct *p, bool preemp * per-cpu scheduler provides, namely provide a proportional amount of compute * time to each task. This is expressed in the following equation: * - * W_i,n/P_i == W_j,n/P_j for all i,j (1) + * W_i,n/P_i == W_j,n/P_j for all i,j (1) * * Where W_i,n is the n-th weight average for cpu i. The instantaneous weight * W_i,0 is defined as: * - * W_i,0 = \Sum_j w_i,j (2) + * W_i,0 = \Sum_j w_i,j (2) * * Where w_i,j is the weight of the j-th runnable task on cpu i. This weight * is derived from the nice value as per prio_to_weight[]. @@ -5660,7 +5665,7 @@ static bool yield_to_task_fair(struct rq *rq, struct task_struct *p, bool preemp * The weight average is an exponential decay average of the instantaneous * weight: * - * W'_i,n = (2^n - 1) / 2^n * W_i,n + 1 / 2^n * W_i,0 (3) + * W'_i,n = (2^n - 1) / 2^n * W_i,n + 1 / 2^n * W_i,0 (3) * * C_i is the compute capacity of cpu i, typically it is the * fraction of 'recent' time available for SCHED_OTHER task execution. But it @@ -5693,12 +5698,12 @@ static bool yield_to_task_fair(struct rq *rq, struct task_struct *p, bool preemp * This yields: * * log_2 n 1 n - * \Sum { --- * --- * 2^i } = O(n) (5) + * \Sum { --- * --- * 2^i } = O(n) (5) * i = 0 2^i 2^i - * `- size of each group - * | | `- number of cpus doing load-balance - * | `- freq - * `- sum over all levels + * `- size of each group + * | | `- number of cpus doing load-balance + * | `- freq + * `- sum over all levels * * Coupled with a limit on how many tasks we can migrate every balance pass, * this makes (5) the runtime complexity of the balancer. @@ -5708,19 +5713,19 @@ static bool yield_to_task_fair(struct rq *rq, struct task_struct *p, bool preemp * * The adjacency matrix of the resulting graph is given by: * - * log_2 n + * log_2 n * A_i,j = \Union (i % 2^k == 0) && i / 2^(k+1) == j / 2^(k+1) (6) - * k = 0 + * k = 0 * * And you'll find that: * - * A^(log_2 n)_i,j != 0 for all i,j (7) + * A^(log_2 n)_i,j != 0 for all i,j (7) * * Showing there's indeed a path between every cpu in at most O(log n) steps. * The task movement gives a factor of O(m), giving a convergence complexity * of: * - * O(nm log n), n := nr_cpus, m := nr_tasks (8) + * O(nm log n), n := nr_cpus, m := nr_tasks (8) * * * WORK CONSERVING @@ -5739,13 +5744,13 @@ static bool yield_to_task_fair(struct rq *rq, struct task_struct *p, bool preemp * * Cgroups make a horror show out of (2), instead of a simple sum we get: * - * s_k,i - * W_i,0 = \Sum_j \Prod_k w_k * ----- (9) - * S_k + * s_k,i + * W_i,0 = \Sum_j \Prod_k w_k * ----- (9) + * S_k * * Where * - * s_k,i = \Sum_j w_i,j,k and S_k = \Sum_i s_k,i (10) + * s_k,i = \Sum_j w_i,j,k and S_k = \Sum_i s_k,i (10) * * w_i,j,k is the weight of the j-th runnable task in the k-th cgroup on cpu i. * @@ -5766,29 +5771,29 @@ enum fbq_type { regular, remote, all }; #define LBF_SOME_PINNED 0x08 struct lb_env { - struct sched_domain *sd; + struct sched_domain *sd; - struct rq *src_rq; - int src_cpu; + struct rq *src_rq; + int src_cpu; - int dst_cpu; - struct rq *dst_rq; + int dst_cpu; + struct rq *dst_rq; - struct cpumask *dst_grpmask; - int new_dst_cpu; - enum cpu_idle_type idle; - long imbalance; - /* The set of CPUs under consideration for load-balancing */ - struct cpumask *cpus; + struct cpumask *dst_grpmask; + int new_dst_cpu; + enum cpu_idle_type idle; + long imbalance; + /* The set of CPUs under consideration for load-balancing */ + struct cpumask *cpus; - unsigned int flags; + unsigned int flags; - unsigned int loop; - unsigned int loop_break; - unsigned int loop_max; + unsigned int loop; + unsigned int loop_break; + unsigned int loop_max; - enum fbq_type fbq_type; - struct list_head tasks; + enum fbq_type fbq_type; + struct list_head tasks; }; /* @@ -5796,32 +5801,32 @@ struct lb_env { */ static int task_hot(struct task_struct *p, struct lb_env *env) { - s64 delta; + s64 delta; - lockdep_assert_held(&env->src_rq->lock); + lockdep_assert_held(&env->src_rq->lock); - if (p->sched_class != &fair_sched_class) - return 0; + if (p->sched_class != &fair_sched_class) + return 0; - if (unlikely(p->policy == SCHED_IDLE)) - return 0; + if (unlikely(p->policy == SCHED_IDLE)) + return 0; - /* - * Buddy candidates are cache hot: - */ - if (sched_feat(CACHE_HOT_BUDDY) && env->dst_rq->nr_running && - (&p->se == cfs_rq_of(&p->se)->next || - &p->se == cfs_rq_of(&p->se)->last)) - return 1; + /* + * Buddy candidates are cache hot: + */ + if (sched_feat(CACHE_HOT_BUDDY) && env->dst_rq->nr_running && + (&p->se == cfs_rq_of(&p->se)->next || + &p->se == cfs_rq_of(&p->se)->last)) + return 1; - if (sysctl_sched_migration_cost == -1) - return 1; - if (sysctl_sched_migration_cost == 0) - return 0; + if (sysctl_sched_migration_cost == -1) + return 1; + if (sysctl_sched_migration_cost == 0) + return 0; - delta = rq_clock_task(env->src_rq) - p->se.exec_start; + delta = rq_clock_task(env->src_rq) - p->se.exec_start; - return delta < (s64)sysctl_sched_migration_cost; + return delta < (s64)sysctl_sched_migration_cost; } #ifdef CONFIG_NUMA_BALANCING @@ -5832,50 +5837,50 @@ static int task_hot(struct task_struct *p, struct lb_env *env) */ static int migrate_degrades_locality(struct task_struct *p, struct lb_env *env) { - struct numa_group *numa_group = rcu_dereference(p->numa_group); - unsigned long src_faults, dst_faults; - int src_nid, dst_nid; + struct numa_group *numa_group = rcu_dereference(p->numa_group); + unsigned long src_faults, dst_faults; + int src_nid, dst_nid; - if (!static_branch_likely(&sched_numa_balancing)) - return -1; + if (!static_branch_likely(&sched_numa_balancing)) + return -1; - if (!p->numa_faults || !(env->sd->flags & SD_NUMA)) - return -1; + if (!p->numa_faults || !(env->sd->flags & SD_NUMA)) + return -1; - src_nid = cpu_to_node(env->src_cpu); - dst_nid = cpu_to_node(env->dst_cpu); + src_nid = cpu_to_node(env->src_cpu); + dst_nid = cpu_to_node(env->dst_cpu); - if (src_nid == dst_nid) - return -1; + if (src_nid == dst_nid) + return -1; - /* Migrating away from the preferred node is always bad. */ - if (src_nid == p->numa_preferred_nid) { - if (env->src_rq->nr_running > env->src_rq->nr_preferred_running) - return 1; - else - return -1; - } + /* Migrating away from the preferred node is always bad. */ + if (src_nid == p->numa_preferred_nid) { + if (env->src_rq->nr_running > env->src_rq->nr_preferred_running) + return 1; + else + return -1; + } - /* Encourage migration to the preferred node. */ - if (dst_nid == p->numa_preferred_nid) - return 0; + /* Encourage migration to the preferred node. */ + if (dst_nid == p->numa_preferred_nid) + return 0; - if (numa_group) { - src_faults = group_faults(p, src_nid); - dst_faults = group_faults(p, dst_nid); - } else { - src_faults = task_faults(p, src_nid); - dst_faults = task_faults(p, dst_nid); - } + if (numa_group) { + src_faults = group_faults(p, src_nid); + dst_faults = group_faults(p, dst_nid); + } else { + src_faults = task_faults(p, src_nid); + dst_faults = task_faults(p, dst_nid); + } - return dst_faults < src_faults; + return dst_faults < src_faults; } #else static inline int migrate_degrades_locality(struct task_struct *p, - struct lb_env *env) + struct lb_env *env) { - return -1; + return -1; } #endif @@ -5885,79 +5890,79 @@ static inline int migrate_degrades_locality(struct task_struct *p, static int can_migrate_task(struct task_struct *p, struct lb_env *env) { - int tsk_cache_hot; + int tsk_cache_hot; - lockdep_assert_held(&env->src_rq->lock); + lockdep_assert_held(&env->src_rq->lock); - /* - * We do not migrate tasks that are: - * 1) throttled_lb_pair, or - * 2) cannot be migrated to this CPU due to cpus_allowed, or - * 3) running (obviously), or - * 4) are cache-hot on their current CPU. - */ - if (throttled_lb_pair(task_group(p), env->src_cpu, env->dst_cpu)) - return 0; - - if (!cpumask_test_cpu(env->dst_cpu, tsk_cpus_allowed(p))) { - int cpu; - - schedstat_inc(p, se.statistics.nr_failed_migrations_affine); - - env->flags |= LBF_SOME_PINNED; - - /* - * Remember if this task can be migrated to any other cpu in - * our sched_group. We may want to revisit it if we couldn't - * meet load balance goals by pulling other tasks on src_cpu. - * - * Also avoid computing new_dst_cpu if we have already computed - * one in current iteration. - */ - if (!env->dst_grpmask || (env->flags & LBF_DST_PINNED)) - return 0; - - /* Prevent to re-select dst_cpu via env's cpus */ - for_each_cpu_and(cpu, env->dst_grpmask, env->cpus) { - if (cpumask_test_cpu(cpu, tsk_cpus_allowed(p))) { - env->flags |= LBF_DST_PINNED; - env->new_dst_cpu = cpu; - break; - } - } - - return 0; - } - - /* Record that we found atleast one task that could run on dst_cpu */ - env->flags &= ~LBF_ALL_PINNED; - - if (task_running(env->src_rq, p)) { - schedstat_inc(p, se.statistics.nr_failed_migrations_running); - return 0; - } - - /* - * Aggressive migration if: - * 1) destination numa is preferred - * 2) task is cache cold, or - * 3) too many balance attempts have failed. - */ - tsk_cache_hot = migrate_degrades_locality(p, env); - if (tsk_cache_hot == -1) - tsk_cache_hot = task_hot(p, env); - - if (tsk_cache_hot <= 0 || - env->sd->nr_balance_failed > env->sd->cache_nice_tries) { - if (tsk_cache_hot == 1) { - schedstat_inc(env->sd, lb_hot_gained[env->idle]); - schedstat_inc(p, se.statistics.nr_forced_migrations); - } - return 1; - } - - schedstat_inc(p, se.statistics.nr_failed_migrations_hot); + /* + * We do not migrate tasks that are: + * 1) throttled_lb_pair, or + * 2) cannot be migrated to this CPU due to cpus_allowed, or + * 3) running (obviously), or + * 4) are cache-hot on their current CPU. + */ + if (throttled_lb_pair(task_group(p), env->src_cpu, env->dst_cpu)) return 0; + + if (!cpumask_test_cpu(env->dst_cpu, tsk_cpus_allowed(p))) { + int cpu; + + schedstat_inc(p, se.statistics.nr_failed_migrations_affine); + + env->flags |= LBF_SOME_PINNED; + + /* + * Remember if this task can be migrated to any other cpu in + * our sched_group. We may want to revisit it if we couldn't + * meet load balance goals by pulling other tasks on src_cpu. + * + * Also avoid computing new_dst_cpu if we have already computed + * one in current iteration. + */ + if (!env->dst_grpmask || (env->flags & LBF_DST_PINNED)) + return 0; + + /* Prevent to re-select dst_cpu via env's cpus */ + for_each_cpu_and(cpu, env->dst_grpmask, env->cpus) { + if (cpumask_test_cpu(cpu, tsk_cpus_allowed(p))) { + env->flags |= LBF_DST_PINNED; + env->new_dst_cpu = cpu; + break; + } + } + + return 0; + } + + /* Record that we found atleast one task that could run on dst_cpu */ + env->flags &= ~LBF_ALL_PINNED; + + if (task_running(env->src_rq, p)) { + schedstat_inc(p, se.statistics.nr_failed_migrations_running); + return 0; + } + + /* + * Aggressive migration if: + * 1) destination numa is preferred + * 2) task is cache cold, or + * 3) too many balance attempts have failed. + */ + tsk_cache_hot = migrate_degrades_locality(p, env); + if (tsk_cache_hot == -1) + tsk_cache_hot = task_hot(p, env); + + if (tsk_cache_hot <= 0 || + env->sd->nr_balance_failed > env->sd->cache_nice_tries) { + if (tsk_cache_hot == 1) { + schedstat_inc(env->sd, lb_hot_gained[env->idle]); + schedstat_inc(p, se.statistics.nr_forced_migrations); + } + return 1; + } + + schedstat_inc(p, se.statistics.nr_failed_migrations_hot); + return 0; } /* @@ -5965,11 +5970,11 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) */ static void detach_task(struct task_struct *p, struct lb_env *env) { - lockdep_assert_held(&env->src_rq->lock); + lockdep_assert_held(&env->src_rq->lock); - p->on_rq = TASK_ON_RQ_MIGRATING; - deactivate_task(env->src_rq, p, 0); - set_task_cpu(p, env->dst_cpu); + p->on_rq = TASK_ON_RQ_MIGRATING; + deactivate_task(env->src_rq, p, 0); + set_task_cpu(p, env->dst_cpu); } /* @@ -5980,26 +5985,26 @@ static void detach_task(struct task_struct *p, struct lb_env *env) */ static struct task_struct *detach_one_task(struct lb_env *env) { - struct task_struct *p, *n; + struct task_struct *p, *n; - lockdep_assert_held(&env->src_rq->lock); + lockdep_assert_held(&env->src_rq->lock); - list_for_each_entry_safe(p, n, &env->src_rq->cfs_tasks, se.group_node) { - if (!can_migrate_task(p, env)) - continue; + list_for_each_entry_safe(p, n, &env->src_rq->cfs_tasks, se.group_node) { + if (!can_migrate_task(p, env)) + continue; - detach_task(p, env); + detach_task(p, env); - /* - * Right now, this is only the second place where - * lb_gained[env->idle] is updated (other is detach_tasks) - * so we can safely collect stats here rather than - * inside detach_tasks(). - */ - schedstat_inc(env->sd, lb_gained[env->idle]); - return p; - } - return NULL; + /* + * Right now, this is only the second place where + * lb_gained[env->idle] is updated (other is detach_tasks) + * so we can safely collect stats here rather than + * inside detach_tasks(). + */ + schedstat_inc(env->sd, lb_gained[env->idle]); + return p; + } + return NULL; } static const unsigned int sched_nr_migrate_break = 32; @@ -6012,85 +6017,85 @@ static const unsigned int sched_nr_migrate_break = 32; */ static int detach_tasks(struct lb_env *env) { - struct list_head *tasks = &env->src_rq->cfs_tasks; - struct task_struct *p; - unsigned long load; - int detached = 0; + struct list_head *tasks = &env->src_rq->cfs_tasks; + struct task_struct *p; + unsigned long load; + int detached = 0; - lockdep_assert_held(&env->src_rq->lock); + lockdep_assert_held(&env->src_rq->lock); - if (env->imbalance <= 0) - return 0; + if (env->imbalance <= 0) + return 0; - while (!list_empty(tasks)) { - /* - * We don't want to steal all, otherwise we may be treated likewise, - * which could at worst lead to a livelock crash. - */ - if (env->idle != CPU_NOT_IDLE && env->src_rq->nr_running <= 1) - break; + while (!list_empty(tasks)) { + /* + * We don't want to steal all, otherwise we may be treated likewise, + * which could at worst lead to a livelock crash. + */ + if (env->idle != CPU_NOT_IDLE && env->src_rq->nr_running <= 1) + break; - p = list_first_entry(tasks, struct task_struct, se.group_node); + p = list_first_entry(tasks, struct task_struct, se.group_node); - env->loop++; - /* We've more or less seen every task there is, call it quits */ - if (env->loop > env->loop_max) - break; + env->loop++; + /* We've more or less seen every task there is, call it quits */ + if (env->loop > env->loop_max) + break; - /* take a breather every nr_migrate tasks */ - if (env->loop > env->loop_break) { - env->loop_break += sched_nr_migrate_break; - env->flags |= LBF_NEED_BREAK; - break; - } - - if (!can_migrate_task(p, env)) - goto next; - - load = task_h_load(p); - - if (sched_feat(LB_MIN) && load < 16 && !env->sd->nr_balance_failed) - goto next; - - if ((load / 2) > env->imbalance) - goto next; - - detach_task(p, env); - list_add(&p->se.group_node, &env->tasks); - - detached++; - env->imbalance -= load; - -#ifdef CONFIG_PREEMPT - /* - * NEWIDLE balancing is a source of latency, so preemptible - * kernels will stop after the first task is detached to minimize - * the critical section. - */ - if (env->idle == CPU_NEWLY_IDLE) - break; -#endif - - /* - * We only want to steal up to the prescribed amount of - * weighted load. - */ - if (env->imbalance <= 0) - break; - - continue; -next: - list_move_tail(&p->se.group_node, tasks); + /* take a breather every nr_migrate tasks */ + if (env->loop > env->loop_break) { + env->loop_break += sched_nr_migrate_break; + env->flags |= LBF_NEED_BREAK; + break; } - /* - * Right now, this is one of only two places we collect this stat - * so we can safely collect detach_one_task() stats here rather - * than inside detach_one_task(). - */ - schedstat_add(env->sd, lb_gained[env->idle], detached); + if (!can_migrate_task(p, env)) + goto next; - return detached; + load = task_h_load(p); + + if (sched_feat(LB_MIN) && load < 16 && !env->sd->nr_balance_failed) + goto next; + + if ((load / 2) > env->imbalance) + goto next; + + detach_task(p, env); + list_add(&p->se.group_node, &env->tasks); + + detached++; + env->imbalance -= load; + +#ifdef CONFIG_PREEMPT + /* + * NEWIDLE balancing is a source of latency, so preemptible + * kernels will stop after the first task is detached to minimize + * the critical section. + */ + if (env->idle == CPU_NEWLY_IDLE) + break; +#endif + + /* + * We only want to steal up to the prescribed amount of + * weighted load. + */ + if (env->imbalance <= 0) + break; + + continue; +next: + list_move_tail(&p->se.group_node, tasks); + } + + /* + * Right now, this is one of only two places we collect this stat + * so we can safely collect detach_one_task() stats here rather + * than inside detach_one_task(). + */ + schedstat_add(env->sd, lb_gained[env->idle], detached); + + return detached; } /* @@ -6098,12 +6103,12 @@ next: */ static void attach_task(struct rq *rq, struct task_struct *p) { - lockdep_assert_held(&rq->lock); + lockdep_assert_held(&rq->lock); - BUG_ON(task_rq(p) != rq); - activate_task(rq, p, 0); - p->on_rq = TASK_ON_RQ_QUEUED; - check_preempt_curr(rq, p, 0); + BUG_ON(task_rq(p) != rq); + activate_task(rq, p, 0); + p->on_rq = TASK_ON_RQ_QUEUED; + check_preempt_curr(rq, p, 0); } /* @@ -6112,9 +6117,9 @@ static void attach_task(struct rq *rq, struct task_struct *p) */ static void attach_one_task(struct rq *rq, struct task_struct *p) { - raw_spin_lock(&rq->lock); - attach_task(rq, p); - raw_spin_unlock(&rq->lock); + raw_spin_lock(&rq->lock); + attach_task(rq, p); + raw_spin_unlock(&rq->lock); } /* @@ -6123,44 +6128,44 @@ static void attach_one_task(struct rq *rq, struct task_struct *p) */ static void attach_tasks(struct lb_env *env) { - struct list_head *tasks = &env->tasks; - struct task_struct *p; + struct list_head *tasks = &env->tasks; + struct task_struct *p; - raw_spin_lock(&env->dst_rq->lock); + raw_spin_lock(&env->dst_rq->lock); - while (!list_empty(tasks)) { - p = list_first_entry(tasks, struct task_struct, se.group_node); - list_del_init(&p->se.group_node); + while (!list_empty(tasks)) { + p = list_first_entry(tasks, struct task_struct, se.group_node); + list_del_init(&p->se.group_node); - attach_task(env->dst_rq, p); - } + attach_task(env->dst_rq, p); + } - raw_spin_unlock(&env->dst_rq->lock); + raw_spin_unlock(&env->dst_rq->lock); } #ifdef CONFIG_FAIR_GROUP_SCHED static void update_blocked_averages(int cpu) { - struct rq *rq = cpu_rq(cpu); - struct cfs_rq *cfs_rq; - unsigned long flags; + struct rq *rq = cpu_rq(cpu); + struct cfs_rq *cfs_rq; + unsigned long flags; - raw_spin_lock_irqsave(&rq->lock, flags); - update_rq_clock(rq); + raw_spin_lock_irqsave(&rq->lock, flags); + update_rq_clock(rq); - /* - * Iterates the task_group tree in a bottom up fashion, see - * list_add_leaf_cfs_rq() for details. - */ - for_each_leaf_cfs_rq(rq, cfs_rq) { - /* throttled entities do not contribute to load */ - if (throttled_hierarchy(cfs_rq)) - continue; + /* + * Iterates the task_group tree in a bottom up fashion, see + * list_add_leaf_cfs_rq() for details. + */ + for_each_leaf_cfs_rq(rq, cfs_rq) { + /* throttled entities do not contribute to load */ + if (throttled_hierarchy(cfs_rq)) + continue; - if (update_cfs_rq_load_avg(cfs_rq_clock_task(cfs_rq), cfs_rq)) - update_tg_load_avg(cfs_rq, 0); - } - raw_spin_unlock_irqrestore(&rq->lock, flags); + if (update_cfs_rq_load_avg(cfs_rq_clock_task(cfs_rq), cfs_rq)) + update_tg_load_avg(cfs_rq, 0); + } + raw_spin_unlock_irqrestore(&rq->lock, flags); } /* @@ -6170,127 +6175,127 @@ static void update_blocked_averages(int cpu) */ static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq) { - struct rq *rq = rq_of(cfs_rq); - struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)]; - unsigned long now = jiffies; - unsigned long load; + struct rq *rq = rq_of(cfs_rq); + struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)]; + unsigned long now = jiffies; + unsigned long load; + if (cfs_rq->last_h_load_update == now) + return; + + cfs_rq->h_load_next = NULL; + for_each_sched_entity(se) { + cfs_rq = cfs_rq_of(se); + cfs_rq->h_load_next = se; if (cfs_rq->last_h_load_update == now) - return; + break; + } - cfs_rq->h_load_next = NULL; - for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); - cfs_rq->h_load_next = se; - if (cfs_rq->last_h_load_update == now) - break; - } + if (!se) { + cfs_rq->h_load = cfs_rq_load_avg(cfs_rq); + cfs_rq->last_h_load_update = now; + } - if (!se) { - cfs_rq->h_load = cfs_rq_load_avg(cfs_rq); - cfs_rq->last_h_load_update = now; - } - - while ((se = cfs_rq->h_load_next) != NULL) { - load = cfs_rq->h_load; - load = div64_ul(load * se->avg.load_avg, - cfs_rq_load_avg(cfs_rq) + 1); - cfs_rq = group_cfs_rq(se); - cfs_rq->h_load = load; - cfs_rq->last_h_load_update = now; - } + while ((se = cfs_rq->h_load_next) != NULL) { + load = cfs_rq->h_load; + load = div64_ul(load * se->avg.load_avg, + cfs_rq_load_avg(cfs_rq) + 1); + cfs_rq = group_cfs_rq(se); + cfs_rq->h_load = load; + cfs_rq->last_h_load_update = now; + } } static unsigned long task_h_load(struct task_struct *p) { - struct cfs_rq *cfs_rq = task_cfs_rq(p); + struct cfs_rq *cfs_rq = task_cfs_rq(p); - update_cfs_rq_h_load(cfs_rq); - return div64_ul(p->se.avg.load_avg * cfs_rq->h_load, - cfs_rq_load_avg(cfs_rq) + 1); + update_cfs_rq_h_load(cfs_rq); + return div64_ul(p->se.avg.load_avg * cfs_rq->h_load, + cfs_rq_load_avg(cfs_rq) + 1); } #else static inline void update_blocked_averages(int cpu) { - struct rq *rq = cpu_rq(cpu); - struct cfs_rq *cfs_rq = &rq->cfs; - unsigned long flags; + struct rq *rq = cpu_rq(cpu); + struct cfs_rq *cfs_rq = &rq->cfs; + unsigned long flags; - raw_spin_lock_irqsave(&rq->lock, flags); - update_rq_clock(rq); - update_cfs_rq_load_avg(cfs_rq_clock_task(cfs_rq), cfs_rq); - raw_spin_unlock_irqrestore(&rq->lock, flags); + raw_spin_lock_irqsave(&rq->lock, flags); + update_rq_clock(rq); + update_cfs_rq_load_avg(cfs_rq_clock_task(cfs_rq), cfs_rq); + raw_spin_unlock_irqrestore(&rq->lock, flags); } static unsigned long task_h_load(struct task_struct *p) { - return p->se.avg.load_avg; + return p->se.avg.load_avg; } #endif /********** Helpers for find_busiest_group ************************/ enum group_type { - group_other = 0, - group_imbalanced, - group_overloaded, + group_other = 0, + group_imbalanced, + group_overloaded, }; /* * sg_lb_stats - stats of a sched_group required for load_balancing */ struct sg_lb_stats { - unsigned long avg_load; /*Avg load across the CPUs of the group */ - unsigned long group_load; /* Total load over the CPUs of the group */ - unsigned long sum_weighted_load; /* Weighted load of group's tasks */ - unsigned long load_per_task; - unsigned long group_capacity; - unsigned long group_util; /* Total utilization of the group */ - unsigned int sum_nr_running; /* Nr tasks running in the group */ - unsigned int idle_cpus; - unsigned int group_weight; - enum group_type group_type; - int group_no_capacity; + unsigned long avg_load; /*Avg load across the CPUs of the group */ + unsigned long group_load; /* Total load over the CPUs of the group */ + unsigned long sum_weighted_load; /* Weighted load of group's tasks */ + unsigned long load_per_task; + unsigned long group_capacity; + unsigned long group_util; /* Total utilization of the group */ + unsigned int sum_nr_running; /* Nr tasks running in the group */ + unsigned int idle_cpus; + unsigned int group_weight; + enum group_type group_type; + int group_no_capacity; #ifdef CONFIG_NUMA_BALANCING - unsigned int nr_numa_running; - unsigned int nr_preferred_running; + unsigned int nr_numa_running; + unsigned int nr_preferred_running; #endif }; /* * sd_lb_stats - Structure to store the statistics of a sched_domain - * during load balancing. + * during load balancing. */ struct sd_lb_stats { - struct sched_group *busiest; /* Busiest group in this sd */ - struct sched_group *local; /* Local group in this sd */ - unsigned long total_load; /* Total load of all groups in sd */ - unsigned long total_capacity; /* Total capacity of all groups in sd */ - unsigned long avg_load; /* Average load across all groups in sd */ + struct sched_group *busiest; /* Busiest group in this sd */ + struct sched_group *local; /* Local group in this sd */ + unsigned long total_load; /* Total load of all groups in sd */ + unsigned long total_capacity; /* Total capacity of all groups in sd */ + unsigned long avg_load; /* Average load across all groups in sd */ - struct sg_lb_stats busiest_stat;/* Statistics of the busiest group */ - struct sg_lb_stats local_stat; /* Statistics of the local group */ + struct sg_lb_stats busiest_stat;/* Statistics of the busiest group */ + struct sg_lb_stats local_stat; /* Statistics of the local group */ }; static inline void init_sd_lb_stats(struct sd_lb_stats *sds) { - /* - * Skimp on the clearing to avoid duplicate work. We can avoid clearing - * local_stat because update_sg_lb_stats() does a full clear/assignment. - * We must however clear busiest_stat::avg_load because - * update_sd_pick_busiest() reads this before assignment. - */ - *sds = (struct sd_lb_stats){ - .busiest = NULL, - .local = NULL, - .total_load = 0UL, - .total_capacity = 0UL, - .busiest_stat = { - .avg_load = 0UL, - .sum_nr_running = 0, - .group_type = group_other, - }, - }; + /* + * Skimp on the clearing to avoid duplicate work. We can avoid clearing + * local_stat because update_sg_lb_stats() does a full clear/assignment. + * We must however clear busiest_stat::avg_load because + * update_sd_pick_busiest() reads this before assignment. + */ + *sds = (struct sd_lb_stats){ + .busiest = NULL, + .local = NULL, + .total_load = 0UL, + .total_capacity = 0UL, + .busiest_stat = { + .avg_load = 0UL, + .sum_nr_running = 0, + .group_type = group_other, + }, + }; } /** @@ -6301,131 +6306,131 @@ static inline void init_sd_lb_stats(struct sd_lb_stats *sds) * Return: The load index. */ static inline int get_sd_load_idx(struct sched_domain *sd, - enum cpu_idle_type idle) + enum cpu_idle_type idle) { - int load_idx; + int load_idx; - switch (idle) { - case CPU_NOT_IDLE: - load_idx = sd->busy_idx; - break; + switch (idle) { + case CPU_NOT_IDLE: + load_idx = sd->busy_idx; + break; - case CPU_NEWLY_IDLE: - load_idx = sd->newidle_idx; - break; - default: - load_idx = sd->idle_idx; - break; - } + case CPU_NEWLY_IDLE: + load_idx = sd->newidle_idx; + break; + default: + load_idx = sd->idle_idx; + break; + } - return load_idx; + return load_idx; } static unsigned long scale_rt_capacity(int cpu) { - struct rq *rq = cpu_rq(cpu); - u64 total, used, age_stamp, avg; - s64 delta; + struct rq *rq = cpu_rq(cpu); + u64 total, used, age_stamp, avg; + s64 delta; - /* - * Since we're reading these variables without serialization make sure - * we read them once before doing sanity checks on them. - */ - age_stamp = READ_ONCE(rq->age_stamp); - avg = READ_ONCE(rq->rt_avg); - delta = __rq_clock_broken(rq) - age_stamp; + /* + * Since we're reading these variables without serialization make sure + * we read them once before doing sanity checks on them. + */ + age_stamp = READ_ONCE(rq->age_stamp); + avg = READ_ONCE(rq->rt_avg); + delta = __rq_clock_broken(rq) - age_stamp; - if (unlikely(delta < 0)) - delta = 0; + if (unlikely(delta < 0)) + delta = 0; - total = sched_avg_period() + delta; + total = sched_avg_period() + delta; - used = div_u64(avg, total); + used = div_u64(avg, total); - if (likely(used < SCHED_CAPACITY_SCALE)) - return SCHED_CAPACITY_SCALE - used; + if (likely(used < SCHED_CAPACITY_SCALE)) + return SCHED_CAPACITY_SCALE - used; - return 1; + return 1; } static void update_cpu_capacity(struct sched_domain *sd, int cpu) { - unsigned long capacity = arch_scale_cpu_capacity(sd, cpu); - struct sched_group *sdg = sd->groups; + unsigned long capacity = arch_scale_cpu_capacity(sd, cpu); + struct sched_group *sdg = sd->groups; - cpu_rq(cpu)->cpu_capacity_orig = capacity; + cpu_rq(cpu)->cpu_capacity_orig = capacity; - capacity *= scale_rt_capacity(cpu); - capacity >>= SCHED_CAPACITY_SHIFT; + capacity *= scale_rt_capacity(cpu); + capacity >>= SCHED_CAPACITY_SHIFT; - if (!capacity) - capacity = 1; + if (!capacity) + capacity = 1; - cpu_rq(cpu)->cpu_capacity = capacity; - sdg->sgc->capacity = capacity; + cpu_rq(cpu)->cpu_capacity = capacity; + sdg->sgc->capacity = capacity; } void update_group_capacity(struct sched_domain *sd, int cpu) { - struct sched_domain *child = sd->child; - struct sched_group *group, *sdg = sd->groups; - unsigned long capacity; - unsigned long interval; + struct sched_domain *child = sd->child; + struct sched_group *group, *sdg = sd->groups; + unsigned long capacity; + unsigned long interval; - interval = msecs_to_jiffies(sd->balance_interval); - interval = clamp(interval, 1UL, max_load_balance_interval); - sdg->sgc->next_update = jiffies + interval; + interval = msecs_to_jiffies(sd->balance_interval); + interval = clamp(interval, 1UL, max_load_balance_interval); + sdg->sgc->next_update = jiffies + interval; - if (!child) { - update_cpu_capacity(sd, cpu); - return; + if (!child) { + update_cpu_capacity(sd, cpu); + return; + } + + capacity = 0; + + if (child->flags & SD_OVERLAP) { + /* + * SD_OVERLAP domains cannot assume that child groups + * span the current group. + */ + + for_each_cpu(cpu, sched_group_cpus(sdg)) { + struct sched_group_capacity *sgc; + struct rq *rq = cpu_rq(cpu); + + /* + * build_sched_domains() -> init_sched_groups_capacity() + * gets here before we've attached the domains to the + * runqueues. + * + * Use capacity_of(), which is set irrespective of domains + * in update_cpu_capacity(). + * + * This avoids capacity from being 0 and + * causing divide-by-zero issues on boot. + */ + if (unlikely(!rq->sd)) { + capacity += capacity_of(cpu); + continue; + } + + sgc = rq->sd->groups->sgc; + capacity += sgc->capacity; } + } else { + /* + * !SD_OVERLAP domains can assume that child groups + * span the current group. + */ - capacity = 0; + group = child->groups; + do { + capacity += group->sgc->capacity; + group = group->next; + } while (group != child->groups); + } - if (child->flags & SD_OVERLAP) { - /* - * SD_OVERLAP domains cannot assume that child groups - * span the current group. - */ - - for_each_cpu(cpu, sched_group_cpus(sdg)) { - struct sched_group_capacity *sgc; - struct rq *rq = cpu_rq(cpu); - - /* - * build_sched_domains() -> init_sched_groups_capacity() - * gets here before we've attached the domains to the - * runqueues. - * - * Use capacity_of(), which is set irrespective of domains - * in update_cpu_capacity(). - * - * This avoids capacity from being 0 and - * causing divide-by-zero issues on boot. - */ - if (unlikely(!rq->sd)) { - capacity += capacity_of(cpu); - continue; - } - - sgc = rq->sd->groups->sgc; - capacity += sgc->capacity; - } - } else { - /* - * !SD_OVERLAP domains can assume that child groups - * span the current group. - */ - - group = child->groups; - do { - capacity += group->sgc->capacity; - group = group->next; - } while (group != child->groups); - } - - sdg->sgc->capacity = capacity; + sdg->sgc->capacity = capacity; } /* @@ -6436,8 +6441,8 @@ void update_group_capacity(struct sched_domain *sd, int cpu) static inline int check_cpu_capacity(struct rq *rq, struct sched_domain *sd) { - return ((rq->cpu_capacity * sd->imbalance_pct) < - (rq->cpu_capacity_orig * 100)); + return ((rq->cpu_capacity * sd->imbalance_pct) < + (rq->cpu_capacity_orig * 100)); } /* @@ -6449,7 +6454,7 @@ check_cpu_capacity(struct rq *rq, struct sched_domain *sd) * Something like: * * { 0 1 2 3 } { 4 5 6 7 } - * * * * * + * * * * * * * If we were to balance group-wise we'd place two tasks in the first group and * two tasks in the second group. Clearly this is undesired as it will overload @@ -6471,7 +6476,7 @@ check_cpu_capacity(struct rq *rq, struct sched_domain *sd) static inline int sg_imbalanced(struct sched_group *group) { - return group->sgc->imbalance; + return group->sgc->imbalance; } /* @@ -6489,14 +6494,14 @@ static inline int sg_imbalanced(struct sched_group *group) static inline bool group_has_capacity(struct lb_env *env, struct sg_lb_stats *sgs) { - if (sgs->sum_nr_running < sgs->group_weight) - return true; + if (sgs->sum_nr_running < sgs->group_weight) + return true; - if ((sgs->group_capacity * 100) > - (sgs->group_util * env->sd->imbalance_pct)) - return true; + if ((sgs->group_capacity * 100) > + (sgs->group_util * env->sd->imbalance_pct)) + return true; - return false; + return false; } /* @@ -6510,27 +6515,27 @@ group_has_capacity(struct lb_env *env, struct sg_lb_stats *sgs) static inline bool group_is_overloaded(struct lb_env *env, struct sg_lb_stats *sgs) { - if (sgs->sum_nr_running <= sgs->group_weight) - return false; - - if ((sgs->group_capacity * 100) < - (sgs->group_util * env->sd->imbalance_pct)) - return true; - + if (sgs->sum_nr_running <= sgs->group_weight) return false; + + if ((sgs->group_capacity * 100) < + (sgs->group_util * env->sd->imbalance_pct)) + return true; + + return false; } static inline enum group_type group_classify(struct sched_group *group, - struct sg_lb_stats *sgs) + struct sg_lb_stats *sgs) { - if (sgs->group_no_capacity) - return group_overloaded; + if (sgs->group_no_capacity) + return group_overloaded; - if (sg_imbalanced(group)) - return group_imbalanced; + if (sg_imbalanced(group)) + return group_imbalanced; - return group_other; + return group_other; } /** @@ -6543,55 +6548,55 @@ group_type group_classify(struct sched_group *group, * @overload: Indicate more than one runnable task for any CPU. */ static inline void update_sg_lb_stats(struct lb_env *env, - struct sched_group *group, int load_idx, - int local_group, struct sg_lb_stats *sgs, - bool *overload) + struct sched_group *group, int load_idx, + int local_group, struct sg_lb_stats *sgs, + bool *overload) { - unsigned long load; - int i, nr_running; + unsigned long load; + int i, nr_running; - memset(sgs, 0, sizeof(*sgs)); + memset(sgs, 0, sizeof(*sgs)); - for_each_cpu_and(i, sched_group_cpus(group), env->cpus) { - struct rq *rq = cpu_rq(i); + for_each_cpu_and(i, sched_group_cpus(group), env->cpus) { + struct rq *rq = cpu_rq(i); - /* Bias balancing toward cpus of our domain */ - if (local_group) - load = target_load(i, load_idx); - else - load = source_load(i, load_idx); + /* Bias balancing toward cpus of our domain */ + if (local_group) + load = target_load(i, load_idx); + else + load = source_load(i, load_idx); - sgs->group_load += load; - sgs->group_util += cpu_util(i); - sgs->sum_nr_running += rq->cfs.h_nr_running; + sgs->group_load += load; + sgs->group_util += cpu_util(i); + sgs->sum_nr_running += rq->cfs.h_nr_running; - nr_running = rq->nr_running; - if (nr_running > 1) - *overload = true; + nr_running = rq->nr_running; + if (nr_running > 1) + *overload = true; #ifdef CONFIG_NUMA_BALANCING - sgs->nr_numa_running += rq->nr_numa_running; - sgs->nr_preferred_running += rq->nr_preferred_running; + sgs->nr_numa_running += rq->nr_numa_running; + sgs->nr_preferred_running += rq->nr_preferred_running; #endif - sgs->sum_weighted_load += weighted_cpuload(i); - /* - * No need to call idle_cpu() if nr_running is not 0 - */ - if (!nr_running && idle_cpu(i)) - sgs->idle_cpus++; - } + sgs->sum_weighted_load += weighted_cpuload(i); + /* + * No need to call idle_cpu() if nr_running is not 0 + */ + if (!nr_running && idle_cpu(i)) + sgs->idle_cpus++; + } - /* Adjust by relative CPU capacity of the group */ - sgs->group_capacity = group->sgc->capacity; - sgs->avg_load = (sgs->group_load*SCHED_CAPACITY_SCALE) / sgs->group_capacity; + /* Adjust by relative CPU capacity of the group */ + sgs->group_capacity = group->sgc->capacity; + sgs->avg_load = (sgs->group_load*SCHED_CAPACITY_SCALE) / sgs->group_capacity; - if (sgs->sum_nr_running) - sgs->load_per_task = sgs->sum_weighted_load / sgs->sum_nr_running; + if (sgs->sum_nr_running) + sgs->load_per_task = sgs->sum_weighted_load / sgs->sum_nr_running; - sgs->group_weight = group->group_weight; + sgs->group_weight = group->group_weight; - sgs->group_no_capacity = group_is_overloaded(env, sgs); - sgs->group_type = group_classify(group, sgs); + sgs->group_no_capacity = group_is_overloaded(env, sgs); + sgs->group_type = group_classify(group, sgs); } /** @@ -6608,68 +6613,68 @@ static inline void update_sg_lb_stats(struct lb_env *env, * busiest group. %false otherwise. */ static bool update_sd_pick_busiest(struct lb_env *env, - struct sd_lb_stats *sds, - struct sched_group *sg, - struct sg_lb_stats *sgs) + struct sd_lb_stats *sds, + struct sched_group *sg, + struct sg_lb_stats *sgs) { - struct sg_lb_stats *busiest = &sds->busiest_stat; + struct sg_lb_stats *busiest = &sds->busiest_stat; - if (sgs->group_type > busiest->group_type) - return true; - - if (sgs->group_type < busiest->group_type) - return false; - - if (sgs->avg_load <= busiest->avg_load) - return false; - - /* This is the busiest node in its class. */ - if (!(env->sd->flags & SD_ASYM_PACKING)) - return true; - - /* - * ASYM_PACKING needs to move all the work to the lowest - * numbered CPUs in the group, therefore mark all groups - * higher than ourself as busy. - */ - if (sgs->sum_nr_running && env->dst_cpu < group_first_cpu(sg)) { - if (!sds->busiest) - return true; - - if (group_first_cpu(sds->busiest) > group_first_cpu(sg)) - return true; - } + if (sgs->group_type > busiest->group_type) + return true; + if (sgs->group_type < busiest->group_type) return false; + + if (sgs->avg_load <= busiest->avg_load) + return false; + + /* This is the busiest node in its class. */ + if (!(env->sd->flags & SD_ASYM_PACKING)) + return true; + + /* + * ASYM_PACKING needs to move all the work to the lowest + * numbered CPUs in the group, therefore mark all groups + * higher than ourself as busy. + */ + if (sgs->sum_nr_running && env->dst_cpu < group_first_cpu(sg)) { + if (!sds->busiest) + return true; + + if (group_first_cpu(sds->busiest) > group_first_cpu(sg)) + return true; + } + + return false; } #ifdef CONFIG_NUMA_BALANCING static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs) { - if (sgs->sum_nr_running > sgs->nr_numa_running) - return regular; - if (sgs->sum_nr_running > sgs->nr_preferred_running) - return remote; - return all; + if (sgs->sum_nr_running > sgs->nr_numa_running) + return regular; + if (sgs->sum_nr_running > sgs->nr_preferred_running) + return remote; + return all; } static inline enum fbq_type fbq_classify_rq(struct rq *rq) { - if (rq->nr_running > rq->nr_numa_running) - return regular; - if (rq->nr_running > rq->nr_preferred_running) - return remote; - return all; + if (rq->nr_running > rq->nr_numa_running) + return regular; + if (rq->nr_running > rq->nr_preferred_running) + return remote; + return all; } #else static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs) { - return all; + return all; } static inline enum fbq_type fbq_classify_rq(struct rq *rq) { - return regular; + return regular; } #endif /* CONFIG_NUMA_BALANCING */ @@ -6680,81 +6685,81 @@ static inline enum fbq_type fbq_classify_rq(struct rq *rq) */ static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sds) { - struct sched_domain *child = env->sd->child; - struct sched_group *sg = env->sd->groups; - struct sg_lb_stats tmp_sgs; - int load_idx, prefer_sibling = 0; - bool overload = false; + struct sched_domain *child = env->sd->child; + struct sched_group *sg = env->sd->groups; + struct sg_lb_stats tmp_sgs; + int load_idx, prefer_sibling = 0; + bool overload = false; - if (child && child->flags & SD_PREFER_SIBLING) - prefer_sibling = 1; + if (child && child->flags & SD_PREFER_SIBLING) + prefer_sibling = 1; - load_idx = get_sd_load_idx(env->sd, env->idle); + load_idx = get_sd_load_idx(env->sd, env->idle); - do { - struct sg_lb_stats *sgs = &tmp_sgs; - int local_group; + do { + struct sg_lb_stats *sgs = &tmp_sgs; + int local_group; - local_group = cpumask_test_cpu(env->dst_cpu, sched_group_cpus(sg)); - if (local_group) { - sds->local = sg; - sgs = &sds->local_stat; + local_group = cpumask_test_cpu(env->dst_cpu, sched_group_cpus(sg)); + if (local_group) { + sds->local = sg; + sgs = &sds->local_stat; - if (env->idle != CPU_NEWLY_IDLE || - time_after_eq(jiffies, sg->sgc->next_update)) - update_group_capacity(env->sd, env->dst_cpu); - } + if (env->idle != CPU_NEWLY_IDLE || + time_after_eq(jiffies, sg->sgc->next_update)) + update_group_capacity(env->sd, env->dst_cpu); + } - update_sg_lb_stats(env, sg, load_idx, local_group, sgs, - &overload); + update_sg_lb_stats(env, sg, load_idx, local_group, sgs, + &overload); - if (local_group) - goto next_group; + if (local_group) + goto next_group; - /* - * In case the child domain prefers tasks go to siblings - * first, lower the sg capacity so that we'll try - * and move all the excess tasks away. We lower the capacity - * of a group only if the local group has the capacity to fit - * these excess tasks. The extra check prevents the case where - * you always pull from the heaviest group when it is already - * under-utilized (possible with a large weight task outweighs - * the tasks on the system). - */ - if (prefer_sibling && sds->local && - group_has_capacity(env, &sds->local_stat) && - (sgs->sum_nr_running > 1)) { - sgs->group_no_capacity = 1; - sgs->group_type = group_classify(sg, sgs); - } + /* + * In case the child domain prefers tasks go to siblings + * first, lower the sg capacity so that we'll try + * and move all the excess tasks away. We lower the capacity + * of a group only if the local group has the capacity to fit + * these excess tasks. The extra check prevents the case where + * you always pull from the heaviest group when it is already + * under-utilized (possible with a large weight task outweighs + * the tasks on the system). + */ + if (prefer_sibling && sds->local && + group_has_capacity(env, &sds->local_stat) && + (sgs->sum_nr_running > 1)) { + sgs->group_no_capacity = 1; + sgs->group_type = group_classify(sg, sgs); + } - if (update_sd_pick_busiest(env, sds, sg, sgs)) { - sds->busiest = sg; - sds->busiest_stat = *sgs; - } + if (update_sd_pick_busiest(env, sds, sg, sgs)) { + sds->busiest = sg; + sds->busiest_stat = *sgs; + } next_group: - /* Now, start updating sd_lb_stats */ - sds->total_load += sgs->group_load; - sds->total_capacity += sgs->group_capacity; + /* Now, start updating sd_lb_stats */ + sds->total_load += sgs->group_load; + sds->total_capacity += sgs->group_capacity; - sg = sg->next; - } while (sg != env->sd->groups); + sg = sg->next; + } while (sg != env->sd->groups); - if (env->sd->flags & SD_NUMA) - env->fbq_type = fbq_classify_group(&sds->busiest_stat); + if (env->sd->flags & SD_NUMA) + env->fbq_type = fbq_classify_group(&sds->busiest_stat); - if (!env->sd->parent) { - /* update overload indicator if we are at root domain */ - if (env->dst_rq->rd->overload != overload) - env->dst_rq->rd->overload = overload; - } + if (!env->sd->parent) { + /* update overload indicator if we are at root domain */ + if (env->dst_rq->rd->overload != overload) + env->dst_rq->rd->overload = overload; + } } /** * check_asym_packing - Check to see if the group is packed into the - * sched doman. + * sched doman. * * This is primarily intended to used at the sibling level. Some * cores like POWER7 prefer to use lower numbered SMT threads. In the @@ -6777,166 +6782,166 @@ next_group: */ static int check_asym_packing(struct lb_env *env, struct sd_lb_stats *sds) { - int busiest_cpu; + int busiest_cpu; - if (!(env->sd->flags & SD_ASYM_PACKING)) - return 0; + if (!(env->sd->flags & SD_ASYM_PACKING)) + return 0; - if (!sds->busiest) - return 0; + if (!sds->busiest) + return 0; - busiest_cpu = group_first_cpu(sds->busiest); - if (env->dst_cpu > busiest_cpu) - return 0; + busiest_cpu = group_first_cpu(sds->busiest); + if (env->dst_cpu > busiest_cpu) + return 0; - env->imbalance = DIV_ROUND_CLOSEST( - sds->busiest_stat.avg_load * sds->busiest_stat.group_capacity, - SCHED_CAPACITY_SCALE); + env->imbalance = DIV_ROUND_CLOSEST( + sds->busiest_stat.avg_load * sds->busiest_stat.group_capacity, + SCHED_CAPACITY_SCALE); - return 1; + return 1; } /** * fix_small_imbalance - Calculate the minor imbalance that exists - * amongst the groups of a sched_domain, during - * load balancing. + * amongst the groups of a sched_domain, during + * load balancing. * @env: The load balancing environment. * @sds: Statistics of the sched_domain whose imbalance is to be calculated. */ static inline void fix_small_imbalance(struct lb_env *env, struct sd_lb_stats *sds) { - unsigned long tmp, capa_now = 0, capa_move = 0; - unsigned int imbn = 2; - unsigned long scaled_busy_load_per_task; - struct sg_lb_stats *local, *busiest; + unsigned long tmp, capa_now = 0, capa_move = 0; + unsigned int imbn = 2; + unsigned long scaled_busy_load_per_task; + struct sg_lb_stats *local, *busiest; - local = &sds->local_stat; - busiest = &sds->busiest_stat; + local = &sds->local_stat; + busiest = &sds->busiest_stat; - if (!local->sum_nr_running) - local->load_per_task = cpu_avg_load_per_task(env->dst_cpu); - else if (busiest->load_per_task > local->load_per_task) - imbn = 1; + if (!local->sum_nr_running) + local->load_per_task = cpu_avg_load_per_task(env->dst_cpu); + else if (busiest->load_per_task > local->load_per_task) + imbn = 1; - scaled_busy_load_per_task = - (busiest->load_per_task * SCHED_CAPACITY_SCALE) / - busiest->group_capacity; + scaled_busy_load_per_task = + (busiest->load_per_task * SCHED_CAPACITY_SCALE) / + busiest->group_capacity; - if (busiest->avg_load + scaled_busy_load_per_task >= - local->avg_load + (scaled_busy_load_per_task * imbn)) { - env->imbalance = busiest->load_per_task; - return; - } + if (busiest->avg_load + scaled_busy_load_per_task >= + local->avg_load + (scaled_busy_load_per_task * imbn)) { + env->imbalance = busiest->load_per_task; + return; + } - /* - * OK, we don't have enough imbalance to justify moving tasks, - * however we may be able to increase total CPU capacity used by - * moving them. - */ + /* + * OK, we don't have enough imbalance to justify moving tasks, + * however we may be able to increase total CPU capacity used by + * moving them. + */ - capa_now += busiest->group_capacity * - min(busiest->load_per_task, busiest->avg_load); - capa_now += local->group_capacity * - min(local->load_per_task, local->avg_load); - capa_now /= SCHED_CAPACITY_SCALE; + capa_now += busiest->group_capacity * + min(busiest->load_per_task, busiest->avg_load); + capa_now += local->group_capacity * + min(local->load_per_task, local->avg_load); + capa_now /= SCHED_CAPACITY_SCALE; - /* Amount of load we'd subtract */ - if (busiest->avg_load > scaled_busy_load_per_task) { - capa_move += busiest->group_capacity * - min(busiest->load_per_task, - busiest->avg_load - scaled_busy_load_per_task); - } + /* Amount of load we'd subtract */ + if (busiest->avg_load > scaled_busy_load_per_task) { + capa_move += busiest->group_capacity * + min(busiest->load_per_task, + busiest->avg_load - scaled_busy_load_per_task); + } - /* Amount of load we'd add */ - if (busiest->avg_load * busiest->group_capacity < - busiest->load_per_task * SCHED_CAPACITY_SCALE) { - tmp = (busiest->avg_load * busiest->group_capacity) / - local->group_capacity; - } else { - tmp = (busiest->load_per_task * SCHED_CAPACITY_SCALE) / - local->group_capacity; - } - capa_move += local->group_capacity * - min(local->load_per_task, local->avg_load + tmp); - capa_move /= SCHED_CAPACITY_SCALE; + /* Amount of load we'd add */ + if (busiest->avg_load * busiest->group_capacity < + busiest->load_per_task * SCHED_CAPACITY_SCALE) { + tmp = (busiest->avg_load * busiest->group_capacity) / + local->group_capacity; + } else { + tmp = (busiest->load_per_task * SCHED_CAPACITY_SCALE) / + local->group_capacity; + } + capa_move += local->group_capacity * + min(local->load_per_task, local->avg_load + tmp); + capa_move /= SCHED_CAPACITY_SCALE; - /* Move if we gain throughput */ - if (capa_move > capa_now) - env->imbalance = busiest->load_per_task; + /* Move if we gain throughput */ + if (capa_move > capa_now) + env->imbalance = busiest->load_per_task; } /** * calculate_imbalance - Calculate the amount of imbalance present within the - * groups of a given sched_domain during load balance. + * groups of a given sched_domain during load balance. * @env: load balance environment * @sds: statistics of the sched_domain whose imbalance is to be calculated. */ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *sds) { - unsigned long max_pull, load_above_capacity = ~0UL; - struct sg_lb_stats *local, *busiest; + unsigned long max_pull, load_above_capacity = ~0UL; + struct sg_lb_stats *local, *busiest; - local = &sds->local_stat; - busiest = &sds->busiest_stat; - - if (busiest->group_type == group_imbalanced) { - /* - * In the group_imb case we cannot rely on group-wide averages - * to ensure cpu-load equilibrium, look at wider averages. XXX - */ - busiest->load_per_task = - min(busiest->load_per_task, sds->avg_load); - } + local = &sds->local_stat; + busiest = &sds->busiest_stat; + if (busiest->group_type == group_imbalanced) { /* - * In the presence of smp nice balancing, certain scenarios can have - * max load less than avg load(as we skip the groups at or below - * its cpu_capacity, while calculating max_load..) + * In the group_imb case we cannot rely on group-wide averages + * to ensure cpu-load equilibrium, look at wider averages. XXX */ - if (busiest->avg_load <= sds->avg_load || - local->avg_load >= sds->avg_load) { - env->imbalance = 0; - return fix_small_imbalance(env, sds); - } + busiest->load_per_task = + min(busiest->load_per_task, sds->avg_load); + } - /* - * If there aren't any idle cpus, avoid creating some. - */ - if (busiest->group_type == group_overloaded && - local->group_type == group_overloaded) { - load_above_capacity = busiest->sum_nr_running * - SCHED_LOAD_SCALE; - if (load_above_capacity > busiest->group_capacity) - load_above_capacity -= busiest->group_capacity; - else - load_above_capacity = ~0UL; - } + /* + * In the presence of smp nice balancing, certain scenarios can have + * max load less than avg load(as we skip the groups at or below + * its cpu_capacity, while calculating max_load..) + */ + if (busiest->avg_load <= sds->avg_load || + local->avg_load >= sds->avg_load) { + env->imbalance = 0; + return fix_small_imbalance(env, sds); + } - /* - * We're trying to get all the cpus to the average_load, so we don't - * want to push ourselves above the average load, nor do we wish to - * reduce the max loaded cpu below the average load. At the same time, - * we also don't want to reduce the group load below the group capacity - * (so that we can implement power-savings policies etc). Thus we look - * for the minimum possible imbalance. - */ - max_pull = min(busiest->avg_load - sds->avg_load, load_above_capacity); + /* + * If there aren't any idle cpus, avoid creating some. + */ + if (busiest->group_type == group_overloaded && + local->group_type == group_overloaded) { + load_above_capacity = busiest->sum_nr_running * + SCHED_LOAD_SCALE; + if (load_above_capacity > busiest->group_capacity) + load_above_capacity -= busiest->group_capacity; + else + load_above_capacity = ~0UL; + } - /* How much load to actually move to equalise the imbalance */ - env->imbalance = min( - max_pull * busiest->group_capacity, - (sds->avg_load - local->avg_load) * local->group_capacity - ) / SCHED_CAPACITY_SCALE; + /* + * We're trying to get all the cpus to the average_load, so we don't + * want to push ourselves above the average load, nor do we wish to + * reduce the max loaded cpu below the average load. At the same time, + * we also don't want to reduce the group load below the group capacity + * (so that we can implement power-savings policies etc). Thus we look + * for the minimum possible imbalance. + */ + max_pull = min(busiest->avg_load - sds->avg_load, load_above_capacity); - /* - * if *imbalance is less than the average load per runnable task - * there is no guarantee that any tasks will be moved so we'll have - * a think about bumping its value to force at least one task to be - * moved - */ - if (env->imbalance < busiest->load_per_task) - return fix_small_imbalance(env, sds); + /* How much load to actually move to equalise the imbalance */ + env->imbalance = min( + max_pull * busiest->group_capacity, + (sds->avg_load - local->avg_load) * local->group_capacity + ) / SCHED_CAPACITY_SCALE; + + /* + * if *imbalance is less than the average load per runnable task + * there is no guarantee that any tasks will be moved so we'll have + * a think about bumping its value to force at least one task to be + * moved + */ + if (env->imbalance < busiest->load_per_task) + return fix_small_imbalance(env, sds); } /******* find_busiest_group() helpers end here *********************/ @@ -6954,166 +6959,166 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s * @env: The load balancing environment. * * Return: - The busiest group if imbalance exists. - * - If no imbalance and user has opted for power-savings balance, - * return the least loaded group whose CPUs can be - * put to idle by rebalancing its tasks onto our group. + * - If no imbalance and user has opted for power-savings balance, + * return the least loaded group whose CPUs can be + * put to idle by rebalancing its tasks onto our group. */ static struct sched_group *find_busiest_group(struct lb_env *env) { - struct sg_lb_stats *local, *busiest; - struct sd_lb_stats sds; + struct sg_lb_stats *local, *busiest; + struct sd_lb_stats sds; - init_sd_lb_stats(&sds); + init_sd_lb_stats(&sds); - /* - * Compute the various statistics relavent for load balancing at - * this level. - */ - update_sd_lb_stats(env, &sds); - local = &sds.local_stat; - busiest = &sds.busiest_stat; + /* + * Compute the various statistics relavent for load balancing at + * this level. + */ + update_sd_lb_stats(env, &sds); + local = &sds.local_stat; + busiest = &sds.busiest_stat; - /* ASYM feature bypasses nice load balance check */ - if ((env->idle == CPU_IDLE || env->idle == CPU_NEWLY_IDLE) && - check_asym_packing(env, &sds)) - return sds.busiest; - - /* There is no busy sibling group to pull tasks from */ - if (!sds.busiest || busiest->sum_nr_running == 0) - goto out_balanced; - - sds.avg_load = (SCHED_CAPACITY_SCALE * sds.total_load) - / sds.total_capacity; - - /* - * If the busiest group is imbalanced the below checks don't - * work because they assume all things are equal, which typically - * isn't true due to cpus_allowed constraints and the like. - */ - if (busiest->group_type == group_imbalanced) - goto force_balance; - - /* SD_BALANCE_NEWIDLE trumps SMP nice when underutilized */ - if (env->idle == CPU_NEWLY_IDLE && group_has_capacity(env, local) && - busiest->group_no_capacity) - goto force_balance; - - /* - * If the local group is busier than the selected busiest group - * don't try and pull any tasks. - */ - if (local->avg_load >= busiest->avg_load) - goto out_balanced; - - /* - * Don't pull any tasks if this group is already above the domain - * average load. - */ - if (local->avg_load >= sds.avg_load) - goto out_balanced; - - if (env->idle == CPU_IDLE) { - /* - * This cpu is idle. If the busiest group is not overloaded - * and there is no imbalance between this and busiest group - * wrt idle cpus, it is balanced. The imbalance becomes - * significant if the diff is greater than 1 otherwise we - * might end up to just move the imbalance on another group - */ - if ((busiest->group_type != group_overloaded) && - (local->idle_cpus <= (busiest->idle_cpus + 1))) - goto out_balanced; - } else { - /* - * In the CPU_NEWLY_IDLE, CPU_NOT_IDLE cases, use - * imbalance_pct to be conservative. - */ - if (100 * busiest->avg_load <= - env->sd->imbalance_pct * local->avg_load) - goto out_balanced; - } - -force_balance: - /* Looks like there is an imbalance. Compute it */ - calculate_imbalance(env, &sds); + /* ASYM feature bypasses nice load balance check */ + if ((env->idle == CPU_IDLE || env->idle == CPU_NEWLY_IDLE) && + check_asym_packing(env, &sds)) return sds.busiest; + /* There is no busy sibling group to pull tasks from */ + if (!sds.busiest || busiest->sum_nr_running == 0) + goto out_balanced; + + sds.avg_load = (SCHED_CAPACITY_SCALE * sds.total_load) + / sds.total_capacity; + + /* + * If the busiest group is imbalanced the below checks don't + * work because they assume all things are equal, which typically + * isn't true due to cpus_allowed constraints and the like. + */ + if (busiest->group_type == group_imbalanced) + goto force_balance; + + /* SD_BALANCE_NEWIDLE trumps SMP nice when underutilized */ + if (env->idle == CPU_NEWLY_IDLE && group_has_capacity(env, local) && + busiest->group_no_capacity) + goto force_balance; + + /* + * If the local group is busier than the selected busiest group + * don't try and pull any tasks. + */ + if (local->avg_load >= busiest->avg_load) + goto out_balanced; + + /* + * Don't pull any tasks if this group is already above the domain + * average load. + */ + if (local->avg_load >= sds.avg_load) + goto out_balanced; + + if (env->idle == CPU_IDLE) { + /* + * This cpu is idle. If the busiest group is not overloaded + * and there is no imbalance between this and busiest group + * wrt idle cpus, it is balanced. The imbalance becomes + * significant if the diff is greater than 1 otherwise we + * might end up to just move the imbalance on another group + */ + if ((busiest->group_type != group_overloaded) && + (local->idle_cpus <= (busiest->idle_cpus + 1))) + goto out_balanced; + } else { + /* + * In the CPU_NEWLY_IDLE, CPU_NOT_IDLE cases, use + * imbalance_pct to be conservative. + */ + if (100 * busiest->avg_load <= + env->sd->imbalance_pct * local->avg_load) + goto out_balanced; + } + +force_balance: + /* Looks like there is an imbalance. Compute it */ + calculate_imbalance(env, &sds); + return sds.busiest; + out_balanced: - env->imbalance = 0; - return NULL; + env->imbalance = 0; + return NULL; } /* * find_busiest_queue - find the busiest runqueue among the cpus in group. */ static struct rq *find_busiest_queue(struct lb_env *env, - struct sched_group *group) + struct sched_group *group) { - struct rq *busiest = NULL, *rq; - unsigned long busiest_load = 0, busiest_capacity = 1; - int i; + struct rq *busiest = NULL, *rq; + unsigned long busiest_load = 0, busiest_capacity = 1; + int i; - for_each_cpu_and(i, sched_group_cpus(group), env->cpus) { - unsigned long capacity, wl; - enum fbq_type rt; + for_each_cpu_and(i, sched_group_cpus(group), env->cpus) { + unsigned long capacity, wl; + enum fbq_type rt; - rq = cpu_rq(i); - rt = fbq_classify_rq(rq); + rq = cpu_rq(i); + rt = fbq_classify_rq(rq); - /* - * We classify groups/runqueues into three groups: - * - regular: there are !numa tasks - * - remote: there are numa tasks that run on the 'wrong' node - * - all: there is no distinction - * - * In order to avoid migrating ideally placed numa tasks, - * ignore those when there's better options. - * - * If we ignore the actual busiest queue to migrate another - * task, the next balance pass can still reduce the busiest - * queue by moving tasks around inside the node. - * - * If we cannot move enough load due to this classification - * the next pass will adjust the group classification and - * allow migration of more tasks. - * - * Both cases only affect the total convergence complexity. - */ - if (rt > env->fbq_type) - continue; + /* + * We classify groups/runqueues into three groups: + * - regular: there are !numa tasks + * - remote: there are numa tasks that run on the 'wrong' node + * - all: there is no distinction + * + * In order to avoid migrating ideally placed numa tasks, + * ignore those when there's better options. + * + * If we ignore the actual busiest queue to migrate another + * task, the next balance pass can still reduce the busiest + * queue by moving tasks around inside the node. + * + * If we cannot move enough load due to this classification + * the next pass will adjust the group classification and + * allow migration of more tasks. + * + * Both cases only affect the total convergence complexity. + */ + if (rt > env->fbq_type) + continue; - capacity = capacity_of(i); + capacity = capacity_of(i); - wl = weighted_cpuload(i); + wl = weighted_cpuload(i); - /* - * When comparing with imbalance, use weighted_cpuload() - * which is not scaled with the cpu capacity. - */ + /* + * When comparing with imbalance, use weighted_cpuload() + * which is not scaled with the cpu capacity. + */ - if (rq->nr_running == 1 && wl > env->imbalance && - !check_cpu_capacity(rq, env->sd)) - continue; + if (rq->nr_running == 1 && wl > env->imbalance && + !check_cpu_capacity(rq, env->sd)) + continue; - /* - * For the load comparisons with the other cpu's, consider - * the weighted_cpuload() scaled with the cpu capacity, so - * that the load can be moved away from the cpu that is - * potentially running at a lower capacity. - * - * Thus we're looking for max(wl_i / capacity_i), crosswise - * multiplication to rid ourselves of the division works out - * to: wl_i * capacity_j > wl_j * capacity_i; where j is - * our previous maximum. - */ - if (wl * busiest_capacity > busiest_load * capacity) { - busiest_load = wl; - busiest_capacity = capacity; - busiest = rq; - } + /* + * For the load comparisons with the other cpu's, consider + * the weighted_cpuload() scaled with the cpu capacity, so + * that the load can be moved away from the cpu that is + * potentially running at a lower capacity. + * + * Thus we're looking for max(wl_i / capacity_i), crosswise + * multiplication to rid ourselves of the division works out + * to: wl_i * capacity_j > wl_j * capacity_i; where j is + * our previous maximum. + */ + if (wl * busiest_capacity > busiest_load * capacity) { + busiest_load = wl; + busiest_capacity = capacity; + busiest = rq; } + } - return busiest; + return busiest; } /* @@ -7127,69 +7132,69 @@ DEFINE_PER_CPU(cpumask_var_t, load_balance_mask); static int need_active_balance(struct lb_env *env) { - struct sched_domain *sd = env->sd; + struct sched_domain *sd = env->sd; - if (env->idle == CPU_NEWLY_IDLE) { - - /* - * ASYM_PACKING needs to force migrate tasks from busy but - * higher numbered CPUs in order to pack all tasks in the - * lowest numbered CPUs. - */ - if ((sd->flags & SD_ASYM_PACKING) && env->src_cpu > env->dst_cpu) - return 1; - } + if (env->idle == CPU_NEWLY_IDLE) { /* - * The dst_cpu is idle and the src_cpu CPU has only 1 CFS task. - * It's worth migrating the task if the src_cpu's capacity is reduced - * because of other sched_class or IRQs if more capacity stays - * available on dst_cpu. + * ASYM_PACKING needs to force migrate tasks from busy but + * higher numbered CPUs in order to pack all tasks in the + * lowest numbered CPUs. */ - if ((env->idle != CPU_NOT_IDLE) && - (env->src_rq->cfs.h_nr_running == 1)) { - if ((check_cpu_capacity(env->src_rq, sd)) && - (capacity_of(env->src_cpu)*sd->imbalance_pct < capacity_of(env->dst_cpu)*100)) - return 1; - } + if ((sd->flags & SD_ASYM_PACKING) && env->src_cpu > env->dst_cpu) + return 1; + } - return unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2); + /* + * The dst_cpu is idle and the src_cpu CPU has only 1 CFS task. + * It's worth migrating the task if the src_cpu's capacity is reduced + * because of other sched_class or IRQs if more capacity stays + * available on dst_cpu. + */ + if ((env->idle != CPU_NOT_IDLE) && + (env->src_rq->cfs.h_nr_running == 1)) { + if ((check_cpu_capacity(env->src_rq, sd)) && + (capacity_of(env->src_cpu)*sd->imbalance_pct < capacity_of(env->dst_cpu)*100)) + return 1; + } + + return unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2); } static int active_load_balance_cpu_stop(void *data); static int should_we_balance(struct lb_env *env) { - struct sched_group *sg = env->sd->groups; - struct cpumask *sg_cpus, *sg_mask; - int cpu, balance_cpu = -1; + struct sched_group *sg = env->sd->groups; + struct cpumask *sg_cpus, *sg_mask; + int cpu, balance_cpu = -1; - /* - * In the newly idle case, we will allow all the cpu's - * to do the newly idle load balance. - */ - if (env->idle == CPU_NEWLY_IDLE) - return 1; + /* + * In the newly idle case, we will allow all the cpu's + * to do the newly idle load balance. + */ + if (env->idle == CPU_NEWLY_IDLE) + return 1; - sg_cpus = sched_group_cpus(sg); - sg_mask = sched_group_mask(sg); - /* Try to find first idle cpu */ - for_each_cpu_and(cpu, sg_cpus, env->cpus) { - if (!cpumask_test_cpu(cpu, sg_mask) || !idle_cpu(cpu)) - continue; + sg_cpus = sched_group_cpus(sg); + sg_mask = sched_group_mask(sg); + /* Try to find first idle cpu */ + for_each_cpu_and(cpu, sg_cpus, env->cpus) { + if (!cpumask_test_cpu(cpu, sg_mask) || !idle_cpu(cpu)) + continue; - balance_cpu = cpu; - break; - } + balance_cpu = cpu; + break; + } - if (balance_cpu == -1) - balance_cpu = group_balance_cpu(sg); + if (balance_cpu == -1) + balance_cpu = group_balance_cpu(sg); - /* - * First idle cpu or the first cpu(busiest) in this sched group - * is eligible for doing load balancing at this and above domains. - */ - return balance_cpu == env->dst_cpu; + /* + * First idle cpu or the first cpu(busiest) in this sched group + * is eligible for doing load balancing at this and above domains. + */ + return balance_cpu == env->dst_cpu; } /* @@ -7197,293 +7202,293 @@ static int should_we_balance(struct lb_env *env) * tasks if there is an imbalance. */ static int load_balance(int this_cpu, struct rq *this_rq, - struct sched_domain *sd, enum cpu_idle_type idle, - int *continue_balancing) + struct sched_domain *sd, enum cpu_idle_type idle, + int *continue_balancing) { - int ld_moved, cur_ld_moved, active_balance = 0; - struct sched_domain *sd_parent = sd->parent; - struct sched_group *group; - struct rq *busiest; - unsigned long flags; - struct cpumask *cpus = this_cpu_cpumask_var_ptr(load_balance_mask); + int ld_moved, cur_ld_moved, active_balance = 0; + struct sched_domain *sd_parent = sd->parent; + struct sched_group *group; + struct rq *busiest; + unsigned long flags; + struct cpumask *cpus = this_cpu_cpumask_var_ptr(load_balance_mask); - struct lb_env env = { - .sd = sd, - .dst_cpu = this_cpu, - .dst_rq = this_rq, - .dst_grpmask = sched_group_cpus(sd->groups), - .idle = idle, - .loop_break = sched_nr_migrate_break, - .cpus = cpus, - .fbq_type = all, - .tasks = LIST_HEAD_INIT(env.tasks), - }; + struct lb_env env = { + .sd = sd, + .dst_cpu = this_cpu, + .dst_rq = this_rq, + .dst_grpmask = sched_group_cpus(sd->groups), + .idle = idle, + .loop_break = sched_nr_migrate_break, + .cpus = cpus, + .fbq_type = all, + .tasks = LIST_HEAD_INIT(env.tasks), + }; - /* - * For NEWLY_IDLE load_balancing, we don't need to consider - * other cpus in our group - */ - if (idle == CPU_NEWLY_IDLE) - env.dst_grpmask = NULL; + /* + * For NEWLY_IDLE load_balancing, we don't need to consider + * other cpus in our group + */ + if (idle == CPU_NEWLY_IDLE) + env.dst_grpmask = NULL; - cpumask_copy(cpus, cpu_active_mask); + cpumask_copy(cpus, cpu_active_mask); - schedstat_inc(sd, lb_count[idle]); + schedstat_inc(sd, lb_count[idle]); redo: - if (!should_we_balance(&env)) { - *continue_balancing = 0; - goto out_balanced; - } + if (!should_we_balance(&env)) { + *continue_balancing = 0; + goto out_balanced; + } - group = find_busiest_group(&env); - if (!group) { - schedstat_inc(sd, lb_nobusyg[idle]); - goto out_balanced; - } + group = find_busiest_group(&env); + if (!group) { + schedstat_inc(sd, lb_nobusyg[idle]); + goto out_balanced; + } - busiest = find_busiest_queue(&env, group); - if (!busiest) { - schedstat_inc(sd, lb_nobusyq[idle]); - goto out_balanced; - } + busiest = find_busiest_queue(&env, group); + if (!busiest) { + schedstat_inc(sd, lb_nobusyq[idle]); + goto out_balanced; + } - BUG_ON(busiest == env.dst_rq); + BUG_ON(busiest == env.dst_rq); - schedstat_add(sd, lb_imbalance[idle], env.imbalance); + schedstat_add(sd, lb_imbalance[idle], env.imbalance); - env.src_cpu = busiest->cpu; - env.src_rq = busiest; + env.src_cpu = busiest->cpu; + env.src_rq = busiest; - ld_moved = 0; - if (busiest->nr_running > 1) { - /* - * Attempt to move tasks. If find_busiest_group has found - * an imbalance but busiest->nr_running <= 1, the group is - * still unbalanced. ld_moved simply stays zero, so it is - * correctly treated as an imbalance. - */ - env.flags |= LBF_ALL_PINNED; - env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running); + ld_moved = 0; + if (busiest->nr_running > 1) { + /* + * Attempt to move tasks. If find_busiest_group has found + * an imbalance but busiest->nr_running <= 1, the group is + * still unbalanced. ld_moved simply stays zero, so it is + * correctly treated as an imbalance. + */ + env.flags |= LBF_ALL_PINNED; + env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running); more_balance: - raw_spin_lock_irqsave(&busiest->lock, flags); + raw_spin_lock_irqsave(&busiest->lock, flags); - /* - * cur_ld_moved - load moved in current iteration - * ld_moved - cumulative load moved across iterations - */ - cur_ld_moved = detach_tasks(&env); - - /* - * We've detached some tasks from busiest_rq. Every - * task is masked "TASK_ON_RQ_MIGRATING", so we can safely - * unlock busiest->lock, and we are able to be sure - * that nobody can manipulate the tasks in parallel. - * See task_rq_lock() family for the details. - */ - - raw_spin_unlock(&busiest->lock); - - if (cur_ld_moved) { - attach_tasks(&env); - ld_moved += cur_ld_moved; - } - - local_irq_restore(flags); - - if (env.flags & LBF_NEED_BREAK) { - env.flags &= ~LBF_NEED_BREAK; - goto more_balance; - } - - /* - * Revisit (affine) tasks on src_cpu that couldn't be moved to - * us and move them to an alternate dst_cpu in our sched_group - * where they can run. The upper limit on how many times we - * iterate on same src_cpu is dependent on number of cpus in our - * sched_group. - * - * This changes load balance semantics a bit on who can move - * load to a given_cpu. In addition to the given_cpu itself - * (or a ilb_cpu acting on its behalf where given_cpu is - * nohz-idle), we now have balance_cpu in a position to move - * load to given_cpu. In rare situations, this may cause - * conflicts (balance_cpu and given_cpu/ilb_cpu deciding - * _independently_ and at _same_ time to move some load to - * given_cpu) causing exceess load to be moved to given_cpu. - * This however should not happen so much in practice and - * moreover subsequent load balance cycles should correct the - * excess load moved. - */ - if ((env.flags & LBF_DST_PINNED) && env.imbalance > 0) { - - /* Prevent to re-select dst_cpu via env's cpus */ - cpumask_clear_cpu(env.dst_cpu, env.cpus); - - env.dst_rq = cpu_rq(env.new_dst_cpu); - env.dst_cpu = env.new_dst_cpu; - env.flags &= ~LBF_DST_PINNED; - env.loop = 0; - env.loop_break = sched_nr_migrate_break; - - /* - * Go back to "more_balance" rather than "redo" since we - * need to continue with same src_cpu. - */ - goto more_balance; - } - - /* - * We failed to reach balance because of affinity. - */ - if (sd_parent) { - int *group_imbalance = &sd_parent->groups->sgc->imbalance; - - if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0) - *group_imbalance = 1; - } - - /* All tasks on this runqueue were pinned by CPU affinity */ - if (unlikely(env.flags & LBF_ALL_PINNED)) { - cpumask_clear_cpu(cpu_of(busiest), cpus); - if (!cpumask_empty(cpus)) { - env.loop = 0; - env.loop_break = sched_nr_migrate_break; - goto redo; - } - goto out_all_pinned; - } - } - - if (!ld_moved) { - schedstat_inc(sd, lb_failed[idle]); - /* - * Increment the failure counter only on periodic balance. - * We do not want newidle balance, which can be very - * frequent, pollute the failure counter causing - * excessive cache_hot migrations and active balances. - */ - if (idle != CPU_NEWLY_IDLE) - sd->nr_balance_failed++; - - if (need_active_balance(&env)) { - raw_spin_lock_irqsave(&busiest->lock, flags); - - /* don't kick the active_load_balance_cpu_stop, - * if the curr task on busiest cpu can't be - * moved to this_cpu - */ - if (!cpumask_test_cpu(this_cpu, - tsk_cpus_allowed(busiest->curr))) { - raw_spin_unlock_irqrestore(&busiest->lock, - flags); - env.flags |= LBF_ALL_PINNED; - goto out_one_pinned; - } - - /* - * ->active_balance synchronizes accesses to - * ->active_balance_work. Once set, it's cleared - * only after active load balance is finished. - */ - if (!busiest->active_balance) { - busiest->active_balance = 1; - busiest->push_cpu = this_cpu; - active_balance = 1; - } - raw_spin_unlock_irqrestore(&busiest->lock, flags); - - if (active_balance) { - stop_one_cpu_nowait(cpu_of(busiest), - active_load_balance_cpu_stop, busiest, - &busiest->active_balance_work); - } - - /* - * We've kicked active balancing, reset the failure - * counter. - */ - sd->nr_balance_failed = sd->cache_nice_tries+1; - } - } else - sd->nr_balance_failed = 0; - - if (likely(!active_balance)) { - /* We were unbalanced, so reset the balancing interval */ - sd->balance_interval = sd->min_interval; - } else { - /* - * If we've begun active balancing, start to back off. This - * case may not be covered by the all_pinned logic if there - * is only 1 task on the busy runqueue (because we don't call - * detach_tasks). - */ - if (sd->balance_interval < sd->max_interval) - sd->balance_interval *= 2; - } - - goto out; - -out_balanced: /* - * We reach balance although we may have faced some affinity - * constraints. Clear the imbalance flag if it was set. + * cur_ld_moved - load moved in current iteration + * ld_moved - cumulative load moved across iterations + */ + cur_ld_moved = detach_tasks(&env); + + /* + * We've detached some tasks from busiest_rq. Every + * task is masked "TASK_ON_RQ_MIGRATING", so we can safely + * unlock busiest->lock, and we are able to be sure + * that nobody can manipulate the tasks in parallel. + * See task_rq_lock() family for the details. + */ + + raw_spin_unlock(&busiest->lock); + + if (cur_ld_moved) { + attach_tasks(&env); + ld_moved += cur_ld_moved; + } + + local_irq_restore(flags); + + if (env.flags & LBF_NEED_BREAK) { + env.flags &= ~LBF_NEED_BREAK; + goto more_balance; + } + + /* + * Revisit (affine) tasks on src_cpu that couldn't be moved to + * us and move them to an alternate dst_cpu in our sched_group + * where they can run. The upper limit on how many times we + * iterate on same src_cpu is dependent on number of cpus in our + * sched_group. + * + * This changes load balance semantics a bit on who can move + * load to a given_cpu. In addition to the given_cpu itself + * (or a ilb_cpu acting on its behalf where given_cpu is + * nohz-idle), we now have balance_cpu in a position to move + * load to given_cpu. In rare situations, this may cause + * conflicts (balance_cpu and given_cpu/ilb_cpu deciding + * _independently_ and at _same_ time to move some load to + * given_cpu) causing exceess load to be moved to given_cpu. + * This however should not happen so much in practice and + * moreover subsequent load balance cycles should correct the + * excess load moved. + */ + if ((env.flags & LBF_DST_PINNED) && env.imbalance > 0) { + + /* Prevent to re-select dst_cpu via env's cpus */ + cpumask_clear_cpu(env.dst_cpu, env.cpus); + + env.dst_rq = cpu_rq(env.new_dst_cpu); + env.dst_cpu = env.new_dst_cpu; + env.flags &= ~LBF_DST_PINNED; + env.loop = 0; + env.loop_break = sched_nr_migrate_break; + + /* + * Go back to "more_balance" rather than "redo" since we + * need to continue with same src_cpu. + */ + goto more_balance; + } + + /* + * We failed to reach balance because of affinity. */ if (sd_parent) { - int *group_imbalance = &sd_parent->groups->sgc->imbalance; + int *group_imbalance = &sd_parent->groups->sgc->imbalance; - if (*group_imbalance) - *group_imbalance = 0; + if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0) + *group_imbalance = 1; } -out_all_pinned: - /* - * We reach balance because all tasks are pinned at this level so - * we can't migrate them. Let the imbalance flag set so parent level - * can try to migrate them. - */ - schedstat_inc(sd, lb_balanced[idle]); + /* All tasks on this runqueue were pinned by CPU affinity */ + if (unlikely(env.flags & LBF_ALL_PINNED)) { + cpumask_clear_cpu(cpu_of(busiest), cpus); + if (!cpumask_empty(cpus)) { + env.loop = 0; + env.loop_break = sched_nr_migrate_break; + goto redo; + } + goto out_all_pinned; + } + } + if (!ld_moved) { + schedstat_inc(sd, lb_failed[idle]); + /* + * Increment the failure counter only on periodic balance. + * We do not want newidle balance, which can be very + * frequent, pollute the failure counter causing + * excessive cache_hot migrations and active balances. + */ + if (idle != CPU_NEWLY_IDLE) + sd->nr_balance_failed++; + + if (need_active_balance(&env)) { + raw_spin_lock_irqsave(&busiest->lock, flags); + + /* don't kick the active_load_balance_cpu_stop, + * if the curr task on busiest cpu can't be + * moved to this_cpu + */ + if (!cpumask_test_cpu(this_cpu, + tsk_cpus_allowed(busiest->curr))) { + raw_spin_unlock_irqrestore(&busiest->lock, + flags); + env.flags |= LBF_ALL_PINNED; + goto out_one_pinned; + } + + /* + * ->active_balance synchronizes accesses to + * ->active_balance_work. Once set, it's cleared + * only after active load balance is finished. + */ + if (!busiest->active_balance) { + busiest->active_balance = 1; + busiest->push_cpu = this_cpu; + active_balance = 1; + } + raw_spin_unlock_irqrestore(&busiest->lock, flags); + + if (active_balance) { + stop_one_cpu_nowait(cpu_of(busiest), + active_load_balance_cpu_stop, busiest, + &busiest->active_balance_work); + } + + /* + * We've kicked active balancing, reset the failure + * counter. + */ + sd->nr_balance_failed = sd->cache_nice_tries+1; + } + } else sd->nr_balance_failed = 0; -out_one_pinned: - /* tune up the balancing interval */ - if (((env.flags & LBF_ALL_PINNED) && - sd->balance_interval < MAX_PINNED_INTERVAL) || - (sd->balance_interval < sd->max_interval)) - sd->balance_interval *= 2; + if (likely(!active_balance)) { + /* We were unbalanced, so reset the balancing interval */ + sd->balance_interval = sd->min_interval; + } else { + /* + * If we've begun active balancing, start to back off. This + * case may not be covered by the all_pinned logic if there + * is only 1 task on the busy runqueue (because we don't call + * detach_tasks). + */ + if (sd->balance_interval < sd->max_interval) + sd->balance_interval *= 2; + } - ld_moved = 0; + goto out; + +out_balanced: + /* + * We reach balance although we may have faced some affinity + * constraints. Clear the imbalance flag if it was set. + */ + if (sd_parent) { + int *group_imbalance = &sd_parent->groups->sgc->imbalance; + + if (*group_imbalance) + *group_imbalance = 0; + } + +out_all_pinned: + /* + * We reach balance because all tasks are pinned at this level so + * we can't migrate them. Let the imbalance flag set so parent level + * can try to migrate them. + */ + schedstat_inc(sd, lb_balanced[idle]); + + sd->nr_balance_failed = 0; + +out_one_pinned: + /* tune up the balancing interval */ + if (((env.flags & LBF_ALL_PINNED) && + sd->balance_interval < MAX_PINNED_INTERVAL) || + (sd->balance_interval < sd->max_interval)) + sd->balance_interval *= 2; + + ld_moved = 0; out: - return ld_moved; + return ld_moved; } static inline unsigned long get_sd_balance_interval(struct sched_domain *sd, int cpu_busy) { - unsigned long interval = sd->balance_interval; + unsigned long interval = sd->balance_interval; - if (cpu_busy) - interval *= sd->busy_factor; + if (cpu_busy) + interval *= sd->busy_factor; - /* scale ms to jiffies */ - interval = msecs_to_jiffies(interval); - interval = clamp(interval, 1UL, max_load_balance_interval); + /* scale ms to jiffies */ + interval = msecs_to_jiffies(interval); + interval = clamp(interval, 1UL, max_load_balance_interval); - return interval; + return interval; } static inline void update_next_balance(struct sched_domain *sd, int cpu_busy, unsigned long *next_balance) { - unsigned long interval, next; + unsigned long interval, next; - interval = get_sd_balance_interval(sd, cpu_busy); - next = sd->last_balance + interval; + interval = get_sd_balance_interval(sd, cpu_busy); + next = sd->last_balance + interval; - if (time_after(*next_balance, next)) - *next_balance = next; + if (time_after(*next_balance, next)) + *next_balance = next; } /* @@ -7492,96 +7497,96 @@ update_next_balance(struct sched_domain *sd, int cpu_busy, unsigned long *next_b */ static int idle_balance(struct rq *this_rq) { - unsigned long next_balance = jiffies + HZ; - int this_cpu = this_rq->cpu; - struct sched_domain *sd; - int pulled_task = 0; - u64 curr_cost = 0; + unsigned long next_balance = jiffies + HZ; + int this_cpu = this_rq->cpu; + struct sched_domain *sd; + int pulled_task = 0; + u64 curr_cost = 0; - /* - * We must set idle_stamp _before_ calling idle_balance(), such that we - * measure the duration of idle_balance() as idle time. - */ - this_rq->idle_stamp = rq_clock(this_rq); + /* + * We must set idle_stamp _before_ calling idle_balance(), such that we + * measure the duration of idle_balance() as idle time. + */ + this_rq->idle_stamp = rq_clock(this_rq); - if (this_rq->avg_idle < sysctl_sched_migration_cost || - !this_rq->rd->overload) { - rcu_read_lock(); - sd = rcu_dereference_check_sched_domain(this_rq->sd); - if (sd) - update_next_balance(sd, 0, &next_balance); - rcu_read_unlock(); - - goto out; - } - - raw_spin_unlock(&this_rq->lock); - - update_blocked_averages(this_cpu); + if (this_rq->avg_idle < sysctl_sched_migration_cost || + !this_rq->rd->overload) { rcu_read_lock(); - for_each_domain(this_cpu, sd) { - int continue_balancing = 1; - u64 t0, domain_cost; - - if (!(sd->flags & SD_LOAD_BALANCE)) - continue; - - if (this_rq->avg_idle < curr_cost + sd->max_newidle_lb_cost) { - update_next_balance(sd, 0, &next_balance); - break; - } - - if (sd->flags & SD_BALANCE_NEWIDLE) { - t0 = sched_clock_cpu(this_cpu); - - pulled_task = load_balance(this_cpu, this_rq, - sd, CPU_NEWLY_IDLE, - &continue_balancing); - - domain_cost = sched_clock_cpu(this_cpu) - t0; - if (domain_cost > sd->max_newidle_lb_cost) - sd->max_newidle_lb_cost = domain_cost; - - curr_cost += domain_cost; - } - - update_next_balance(sd, 0, &next_balance); - - /* - * Stop searching for tasks to pull if there are - * now runnable tasks on this rq. - */ - if (pulled_task || this_rq->nr_running > 0) - break; - } + sd = rcu_dereference_check_sched_domain(this_rq->sd); + if (sd) + update_next_balance(sd, 0, &next_balance); rcu_read_unlock(); - raw_spin_lock(&this_rq->lock); + goto out; + } - if (curr_cost > this_rq->max_idle_balance_cost) - this_rq->max_idle_balance_cost = curr_cost; + raw_spin_unlock(&this_rq->lock); + + update_blocked_averages(this_cpu); + rcu_read_lock(); + for_each_domain(this_cpu, sd) { + int continue_balancing = 1; + u64 t0, domain_cost; + + if (!(sd->flags & SD_LOAD_BALANCE)) + continue; + + if (this_rq->avg_idle < curr_cost + sd->max_newidle_lb_cost) { + update_next_balance(sd, 0, &next_balance); + break; + } + + if (sd->flags & SD_BALANCE_NEWIDLE) { + t0 = sched_clock_cpu(this_cpu); + + pulled_task = load_balance(this_cpu, this_rq, + sd, CPU_NEWLY_IDLE, + &continue_balancing); + + domain_cost = sched_clock_cpu(this_cpu) - t0; + if (domain_cost > sd->max_newidle_lb_cost) + sd->max_newidle_lb_cost = domain_cost; + + curr_cost += domain_cost; + } + + update_next_balance(sd, 0, &next_balance); /* - * While browsing the domains, we released the rq lock, a task could - * have been enqueued in the meantime. Since we're not going idle, - * pretend we pulled a task. + * Stop searching for tasks to pull if there are + * now runnable tasks on this rq. */ - if (this_rq->cfs.h_nr_running && !pulled_task) - pulled_task = 1; + if (pulled_task || this_rq->nr_running > 0) + break; + } + rcu_read_unlock(); + + raw_spin_lock(&this_rq->lock); + + if (curr_cost > this_rq->max_idle_balance_cost) + this_rq->max_idle_balance_cost = curr_cost; + + /* + * While browsing the domains, we released the rq lock, a task could + * have been enqueued in the meantime. Since we're not going idle, + * pretend we pulled a task. + */ + if (this_rq->cfs.h_nr_running && !pulled_task) + pulled_task = 1; out: - /* Move the next balance forward */ - if (time_after(this_rq->next_balance, next_balance)) - this_rq->next_balance = next_balance; + /* Move the next balance forward */ + if (time_after(this_rq->next_balance, next_balance)) + this_rq->next_balance = next_balance; - /* Is there a task of a high priority class? */ - if (this_rq->nr_running != this_rq->cfs.h_nr_running) - pulled_task = -1; + /* Is there a task of a high priority class? */ + if (this_rq->nr_running != this_rq->cfs.h_nr_running) + pulled_task = -1; - if (pulled_task) - this_rq->idle_stamp = 0; + if (pulled_task) + this_rq->idle_stamp = 0; - return pulled_task; + return pulled_task; } /* @@ -7592,73 +7597,73 @@ out: */ static int active_load_balance_cpu_stop(void *data) { - struct rq *busiest_rq = data; - int busiest_cpu = cpu_of(busiest_rq); - int target_cpu = busiest_rq->push_cpu; - struct rq *target_rq = cpu_rq(target_cpu); - struct sched_domain *sd; - struct task_struct *p = NULL; + struct rq *busiest_rq = data; + int busiest_cpu = cpu_of(busiest_rq); + int target_cpu = busiest_rq->push_cpu; + struct rq *target_rq = cpu_rq(target_cpu); + struct sched_domain *sd; + struct task_struct *p = NULL; - raw_spin_lock_irq(&busiest_rq->lock); + raw_spin_lock_irq(&busiest_rq->lock); - /* make sure the requested cpu hasn't gone down in the meantime */ - if (unlikely(busiest_cpu != smp_processor_id() || - !busiest_rq->active_balance)) - goto out_unlock; + /* make sure the requested cpu hasn't gone down in the meantime */ + if (unlikely(busiest_cpu != smp_processor_id() || + !busiest_rq->active_balance)) + goto out_unlock; - /* Is there any task to move? */ - if (busiest_rq->nr_running <= 1) - goto out_unlock; + /* Is there any task to move? */ + if (busiest_rq->nr_running <= 1) + goto out_unlock; - /* - * This condition is "impossible", if it occurs - * we need to fix it. Originally reported by - * Bjorn Helgaas on a 128-cpu setup. - */ - BUG_ON(busiest_rq == target_rq); + /* + * This condition is "impossible", if it occurs + * we need to fix it. Originally reported by + * Bjorn Helgaas on a 128-cpu setup. + */ + BUG_ON(busiest_rq == target_rq); - /* Search for an sd spanning us and the target CPU. */ - rcu_read_lock(); - for_each_domain(target_cpu, sd) { - if ((sd->flags & SD_LOAD_BALANCE) && - cpumask_test_cpu(busiest_cpu, sched_domain_span(sd))) - break; - } + /* Search for an sd spanning us and the target CPU. */ + rcu_read_lock(); + for_each_domain(target_cpu, sd) { + if ((sd->flags & SD_LOAD_BALANCE) && + cpumask_test_cpu(busiest_cpu, sched_domain_span(sd))) + break; + } - if (likely(sd)) { - struct lb_env env = { - .sd = sd, - .dst_cpu = target_cpu, - .dst_rq = target_rq, - .src_cpu = busiest_rq->cpu, - .src_rq = busiest_rq, - .idle = CPU_IDLE, - }; + if (likely(sd)) { + struct lb_env env = { + .sd = sd, + .dst_cpu = target_cpu, + .dst_rq = target_rq, + .src_cpu = busiest_rq->cpu, + .src_rq = busiest_rq, + .idle = CPU_IDLE, + }; - schedstat_inc(sd, alb_count); - - p = detach_one_task(&env); - if (p) - schedstat_inc(sd, alb_pushed); - else - schedstat_inc(sd, alb_failed); - } - rcu_read_unlock(); -out_unlock: - busiest_rq->active_balance = 0; - raw_spin_unlock(&busiest_rq->lock); + schedstat_inc(sd, alb_count); + p = detach_one_task(&env); if (p) - attach_one_task(target_rq, p); + schedstat_inc(sd, alb_pushed); + else + schedstat_inc(sd, alb_failed); + } + rcu_read_unlock(); +out_unlock: + busiest_rq->active_balance = 0; + raw_spin_unlock(&busiest_rq->lock); - local_irq_enable(); + if (p) + attach_one_task(target_rq, p); - return 0; + local_irq_enable(); + + return 0; } static inline int on_null_domain(struct rq *rq) { - return unlikely(!rcu_dereference_sched(rq->sd)); + return unlikely(!rcu_dereference_sched(rq->sd)); } #ifdef CONFIG_NO_HZ_COMMON @@ -7669,19 +7674,19 @@ static inline int on_null_domain(struct rq *rq) * load balancing for all the idle CPUs. */ static struct { - cpumask_var_t idle_cpus_mask; - atomic_t nr_cpus; - unsigned long next_balance; /* in jiffy units */ + cpumask_var_t idle_cpus_mask; + atomic_t nr_cpus; + unsigned long next_balance; /* in jiffy units */ } nohz ____cacheline_aligned; static inline int find_new_ilb(void) { - int ilb = cpumask_first(nohz.idle_cpus_mask); + int ilb = cpumask_first(nohz.idle_cpus_mask); - if (ilb < nr_cpu_ids && idle_cpu(ilb)) - return ilb; + if (ilb < nr_cpu_ids && idle_cpu(ilb)) + return ilb; - return nr_cpu_ids; + return nr_cpu_ids; } /* @@ -7691,73 +7696,73 @@ static inline int find_new_ilb(void) */ static void nohz_balancer_kick(void) { - int ilb_cpu; + int ilb_cpu; - nohz.next_balance++; + nohz.next_balance++; - ilb_cpu = find_new_ilb(); + ilb_cpu = find_new_ilb(); - if (ilb_cpu >= nr_cpu_ids) - return; - - if (test_and_set_bit(NOHZ_BALANCE_KICK, nohz_flags(ilb_cpu))) - return; - /* - * Use smp_send_reschedule() instead of resched_cpu(). - * This way we generate a sched IPI on the target cpu which - * is idle. And the softirq performing nohz idle load balance - * will be run before returning from the IPI. - */ - smp_send_reschedule(ilb_cpu); + if (ilb_cpu >= nr_cpu_ids) return; + + if (test_and_set_bit(NOHZ_BALANCE_KICK, nohz_flags(ilb_cpu))) + return; + /* + * Use smp_send_reschedule() instead of resched_cpu(). + * This way we generate a sched IPI on the target cpu which + * is idle. And the softirq performing nohz idle load balance + * will be run before returning from the IPI. + */ + smp_send_reschedule(ilb_cpu); + return; } static inline void nohz_balance_exit_idle(int cpu) { - if (unlikely(test_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu)))) { - /* - * Completely isolated CPUs don't ever set, so we must test. - */ - if (likely(cpumask_test_cpu(cpu, nohz.idle_cpus_mask))) { - cpumask_clear_cpu(cpu, nohz.idle_cpus_mask); - atomic_dec(&nohz.nr_cpus); - } - clear_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu)); + if (unlikely(test_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu)))) { + /* + * Completely isolated CPUs don't ever set, so we must test. + */ + if (likely(cpumask_test_cpu(cpu, nohz.idle_cpus_mask))) { + cpumask_clear_cpu(cpu, nohz.idle_cpus_mask); + atomic_dec(&nohz.nr_cpus); } + clear_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu)); + } } static inline void set_cpu_sd_state_busy(void) { - struct sched_domain *sd; - int cpu = smp_processor_id(); + struct sched_domain *sd; + int cpu = smp_processor_id(); - rcu_read_lock(); - sd = rcu_dereference(per_cpu(sd_busy, cpu)); + rcu_read_lock(); + sd = rcu_dereference(per_cpu(sd_busy, cpu)); - if (!sd || !sd->nohz_idle) - goto unlock; - sd->nohz_idle = 0; + if (!sd || !sd->nohz_idle) + goto unlock; + sd->nohz_idle = 0; - atomic_inc(&sd->groups->sgc->nr_busy_cpus); + atomic_inc(&sd->groups->sgc->nr_busy_cpus); unlock: - rcu_read_unlock(); + rcu_read_unlock(); } void set_cpu_sd_state_idle(void) { - struct sched_domain *sd; - int cpu = smp_processor_id(); + struct sched_domain *sd; + int cpu = smp_processor_id(); - rcu_read_lock(); - sd = rcu_dereference(per_cpu(sd_busy, cpu)); + rcu_read_lock(); + sd = rcu_dereference(per_cpu(sd_busy, cpu)); - if (!sd || sd->nohz_idle) - goto unlock; - sd->nohz_idle = 1; + if (!sd || sd->nohz_idle) + goto unlock; + sd->nohz_idle = 1; - atomic_dec(&sd->groups->sgc->nr_busy_cpus); + atomic_dec(&sd->groups->sgc->nr_busy_cpus); unlock: - rcu_read_unlock(); + rcu_read_unlock(); } /* @@ -7766,36 +7771,36 @@ unlock: */ void nohz_balance_enter_idle(int cpu) { - /* - * If this cpu is going down, then nothing needs to be done. - */ - if (!cpu_active(cpu)) - return; + /* + * If this cpu is going down, then nothing needs to be done. + */ + if (!cpu_active(cpu)) + return; - if (test_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu))) - return; + if (test_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu))) + return; - /* - * If we're a completely isolated CPU, we don't play. - */ - if (on_null_domain(cpu_rq(cpu))) - return; + /* + * If we're a completely isolated CPU, we don't play. + */ + if (on_null_domain(cpu_rq(cpu))) + return; - cpumask_set_cpu(cpu, nohz.idle_cpus_mask); - atomic_inc(&nohz.nr_cpus); - set_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu)); + cpumask_set_cpu(cpu, nohz.idle_cpus_mask); + atomic_inc(&nohz.nr_cpus); + set_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu)); } static int sched_ilb_notifier(struct notifier_block *nfb, - unsigned long action, void *hcpu) + unsigned long action, void *hcpu) { - switch (action & ~CPU_TASKS_FROZEN) { - case CPU_DYING: - nohz_balance_exit_idle(smp_processor_id()); - return NOTIFY_OK; - default: - return NOTIFY_DONE; - } + switch (action & ~CPU_TASKS_FROZEN) { + case CPU_DYING: + nohz_balance_exit_idle(smp_processor_id()); + return NOTIFY_OK; + default: + return NOTIFY_DONE; + } } #endif @@ -7807,7 +7812,7 @@ static DEFINE_SPINLOCK(balancing); */ void update_max_interval(void) { - max_load_balance_interval = HZ*num_online_cpus()/10; + max_load_balance_interval = HZ*num_online_cpus()/10; } /* @@ -7818,105 +7823,105 @@ void update_max_interval(void) */ static void rebalance_domains(struct rq *rq, enum cpu_idle_type idle) { - int continue_balancing = 1; - int cpu = rq->cpu; - unsigned long interval; - struct sched_domain *sd; - /* Earliest time when we have to do rebalance again */ - unsigned long next_balance = jiffies + 60*HZ; - int update_next_balance = 0; - int need_serialize, need_decay = 0; - u64 max_cost = 0; + int continue_balancing = 1; + int cpu = rq->cpu; + unsigned long interval; + struct sched_domain *sd; + /* Earliest time when we have to do rebalance again */ + unsigned long next_balance = jiffies + 60*HZ; + int update_next_balance = 0; + int need_serialize, need_decay = 0; + u64 max_cost = 0; - update_blocked_averages(cpu); + update_blocked_averages(cpu); - rcu_read_lock(); - for_each_domain(cpu, sd) { - /* - * Decay the newidle max times here because this is a regular - * visit to all the domains. Decay ~1% per second. - */ - if (time_after(jiffies, sd->next_decay_max_lb_cost)) { - sd->max_newidle_lb_cost = - (sd->max_newidle_lb_cost * 253) / 256; - sd->next_decay_max_lb_cost = jiffies + HZ; - need_decay = 1; - } - max_cost += sd->max_newidle_lb_cost; - - if (!(sd->flags & SD_LOAD_BALANCE)) - continue; - - /* - * Stop the load balance at this level. There is another - * CPU in our sched group which is doing load balancing more - * actively. - */ - if (!continue_balancing) { - if (need_decay) - continue; - break; - } - - interval = get_sd_balance_interval(sd, idle != CPU_IDLE); - - need_serialize = sd->flags & SD_SERIALIZE; - if (need_serialize) { - if (!spin_trylock(&balancing)) - goto out; - } - - if (time_after_eq(jiffies, sd->last_balance + interval)) { - if (load_balance(cpu, rq, sd, idle, &continue_balancing)) { - /* - * The LBF_DST_PINNED logic could have changed - * env->dst_cpu, so we can't know our idle - * state even if we migrated tasks. Update it. - */ - idle = idle_cpu(cpu) ? CPU_IDLE : CPU_NOT_IDLE; - } - sd->last_balance = jiffies; - interval = get_sd_balance_interval(sd, idle != CPU_IDLE); - } - if (need_serialize) - spin_unlock(&balancing); -out: - if (time_after(next_balance, sd->last_balance + interval)) { - next_balance = sd->last_balance + interval; - update_next_balance = 1; - } + rcu_read_lock(); + for_each_domain(cpu, sd) { + /* + * Decay the newidle max times here because this is a regular + * visit to all the domains. Decay ~1% per second. + */ + if (time_after(jiffies, sd->next_decay_max_lb_cost)) { + sd->max_newidle_lb_cost = + (sd->max_newidle_lb_cost * 253) / 256; + sd->next_decay_max_lb_cost = jiffies + HZ; + need_decay = 1; } - if (need_decay) { - /* - * Ensure the rq-wide value also decays but keep it at a - * reasonable floor to avoid funnies with rq->avg_idle. - */ - rq->max_idle_balance_cost = - max((u64)sysctl_sched_migration_cost, max_cost); - } - rcu_read_unlock(); + max_cost += sd->max_newidle_lb_cost; + + if (!(sd->flags & SD_LOAD_BALANCE)) + continue; /* - * next_balance will be updated only when there is a need. - * When the cpu is attached to null domain for ex, it will not be - * updated. + * Stop the load balance at this level. There is another + * CPU in our sched group which is doing load balancing more + * actively. */ - if (likely(update_next_balance)) { - rq->next_balance = next_balance; + if (!continue_balancing) { + if (need_decay) + continue; + break; + } + + interval = get_sd_balance_interval(sd, idle != CPU_IDLE); + + need_serialize = sd->flags & SD_SERIALIZE; + if (need_serialize) { + if (!spin_trylock(&balancing)) + goto out; + } + + if (time_after_eq(jiffies, sd->last_balance + interval)) { + if (load_balance(cpu, rq, sd, idle, &continue_balancing)) { + /* + * The LBF_DST_PINNED logic could have changed + * env->dst_cpu, so we can't know our idle + * state even if we migrated tasks. Update it. + */ + idle = idle_cpu(cpu) ? CPU_IDLE : CPU_NOT_IDLE; + } + sd->last_balance = jiffies; + interval = get_sd_balance_interval(sd, idle != CPU_IDLE); + } + if (need_serialize) + spin_unlock(&balancing); +out: + if (time_after(next_balance, sd->last_balance + interval)) { + next_balance = sd->last_balance + interval; + update_next_balance = 1; + } + } + if (need_decay) { + /* + * Ensure the rq-wide value also decays but keep it at a + * reasonable floor to avoid funnies with rq->avg_idle. + */ + rq->max_idle_balance_cost = + max((u64)sysctl_sched_migration_cost, max_cost); + } + rcu_read_unlock(); + + /* + * next_balance will be updated only when there is a need. + * When the cpu is attached to null domain for ex, it will not be + * updated. + */ + if (likely(update_next_balance)) { + rq->next_balance = next_balance; #ifdef CONFIG_NO_HZ_COMMON - /* - * If this CPU has been elected to perform the nohz idle - * balance. Other idle CPUs have already rebalanced with - * nohz_idle_balance() and nohz.next_balance has been - * updated accordingly. This CPU is now running the idle load - * balance for itself and we need to update the - * nohz.next_balance accordingly. - */ - if ((idle == CPU_IDLE) && time_after(nohz.next_balance, rq->next_balance)) - nohz.next_balance = rq->next_balance; + /* + * If this CPU has been elected to perform the nohz idle + * balance. Other idle CPUs have already rebalanced with + * nohz_idle_balance() and nohz.next_balance has been + * updated accordingly. This CPU is now running the idle load + * balance for itself and we need to update the + * nohz.next_balance accordingly. + */ + if ((idle == CPU_IDLE) && time_after(nohz.next_balance, rq->next_balance)) + nohz.next_balance = rq->next_balance; #endif - } + } } #ifdef CONFIG_NO_HZ_COMMON @@ -7926,58 +7931,58 @@ out: */ static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) { - int this_cpu = this_rq->cpu; - struct rq *rq; - int balance_cpu; - /* Earliest time when we have to do rebalance again */ - unsigned long next_balance = jiffies + 60*HZ; - int update_next_balance = 0; + int this_cpu = this_rq->cpu; + struct rq *rq; + int balance_cpu; + /* Earliest time when we have to do rebalance again */ + unsigned long next_balance = jiffies + 60*HZ; + int update_next_balance = 0; - if (idle != CPU_IDLE || - !test_bit(NOHZ_BALANCE_KICK, nohz_flags(this_cpu))) - goto end; + if (idle != CPU_IDLE || + !test_bit(NOHZ_BALANCE_KICK, nohz_flags(this_cpu))) + goto end; - for_each_cpu(balance_cpu, nohz.idle_cpus_mask) { - if (balance_cpu == this_cpu || !idle_cpu(balance_cpu)) - continue; - - /* - * If this cpu gets work to do, stop the load balancing - * work being done for other cpus. Next load - * balancing owner will pick it up. - */ - if (need_resched()) - break; - - rq = cpu_rq(balance_cpu); - - /* - * If time for next balance is due, - * do the balance. - */ - if (time_after_eq(jiffies, rq->next_balance)) { - raw_spin_lock_irq(&rq->lock); - update_rq_clock(rq); - update_cpu_load_idle(rq); - raw_spin_unlock_irq(&rq->lock); - rebalance_domains(rq, CPU_IDLE); - } - - if (time_after(next_balance, rq->next_balance)) { - next_balance = rq->next_balance; - update_next_balance = 1; - } - } + for_each_cpu(balance_cpu, nohz.idle_cpus_mask) { + if (balance_cpu == this_cpu || !idle_cpu(balance_cpu)) + continue; /* - * next_balance will be updated only when there is a need. - * When the CPU is attached to null domain for ex, it will not be - * updated. + * If this cpu gets work to do, stop the load balancing + * work being done for other cpus. Next load + * balancing owner will pick it up. */ - if (likely(update_next_balance)) - nohz.next_balance = next_balance; + if (need_resched()) + break; + + rq = cpu_rq(balance_cpu); + + /* + * If time for next balance is due, + * do the balance. + */ + if (time_after_eq(jiffies, rq->next_balance)) { + raw_spin_lock_irq(&rq->lock); + update_rq_clock(rq); + update_cpu_load_idle(rq); + raw_spin_unlock_irq(&rq->lock); + rebalance_domains(rq, CPU_IDLE); + } + + if (time_after(next_balance, rq->next_balance)) { + next_balance = rq->next_balance; + update_next_balance = 1; + } + } + + /* + * next_balance will be updated only when there is a need. + * When the CPU is attached to null domain for ex, it will not be + * updated. + */ + if (likely(update_next_balance)) + nohz.next_balance = next_balance; end: - clear_bit(NOHZ_BALANCE_KICK, nohz_flags(this_cpu)); + clear_bit(NOHZ_BALANCE_KICK, nohz_flags(this_cpu)); } /* @@ -7993,67 +7998,67 @@ end: */ static inline bool nohz_kick_needed(struct rq *rq) { - unsigned long now = jiffies; - struct sched_domain *sd; - struct sched_group_capacity *sgc; - int nr_busy, cpu = rq->cpu; - bool kick = false; + unsigned long now = jiffies; + struct sched_domain *sd; + struct sched_group_capacity *sgc; + int nr_busy, cpu = rq->cpu; + bool kick = false; - if (unlikely(rq->idle_balance)) - return false; + if (unlikely(rq->idle_balance)) + return false; /* - * We may be recently in ticked or tickless idle mode. At the first - * busy tick after returning from idle, we will update the busy stats. - */ - set_cpu_sd_state_busy(); - nohz_balance_exit_idle(cpu); + * We may be recently in ticked or tickless idle mode. At the first + * busy tick after returning from idle, we will update the busy stats. + */ + set_cpu_sd_state_busy(); + nohz_balance_exit_idle(cpu); - /* - * None are in tickless mode and hence no need for NOHZ idle load - * balancing. - */ - if (likely(!atomic_read(&nohz.nr_cpus))) - return false; + /* + * None are in tickless mode and hence no need for NOHZ idle load + * balancing. + */ + if (likely(!atomic_read(&nohz.nr_cpus))) + return false; - if (time_before(now, nohz.next_balance)) - return false; + if (time_before(now, nohz.next_balance)) + return false; - if (rq->nr_running >= 2) - return true; + if (rq->nr_running >= 2) + return true; - rcu_read_lock(); - sd = rcu_dereference(per_cpu(sd_busy, cpu)); - if (sd) { - sgc = sd->groups->sgc; - nr_busy = atomic_read(&sgc->nr_busy_cpus); - - if (nr_busy > 1) { - kick = true; - goto unlock; - } + rcu_read_lock(); + sd = rcu_dereference(per_cpu(sd_busy, cpu)); + if (sd) { + sgc = sd->groups->sgc; + nr_busy = atomic_read(&sgc->nr_busy_cpus); + if (nr_busy > 1) { + kick = true; + goto unlock; } - sd = rcu_dereference(rq->sd); - if (sd) { - if ((rq->cfs.h_nr_running >= 1) && - check_cpu_capacity(rq, sd)) { - kick = true; - goto unlock; - } - } + } - sd = rcu_dereference(per_cpu(sd_asym, cpu)); - if (sd && (cpumask_first_and(nohz.idle_cpus_mask, - sched_domain_span(sd)) < cpu)) { - kick = true; - goto unlock; + sd = rcu_dereference(rq->sd); + if (sd) { + if ((rq->cfs.h_nr_running >= 1) && + check_cpu_capacity(rq, sd)) { + kick = true; + goto unlock; } + } + + sd = rcu_dereference(per_cpu(sd_asym, cpu)); + if (sd && (cpumask_first_and(nohz.idle_cpus_mask, + sched_domain_span(sd)) < cpu)) { + kick = true; + goto unlock; + } unlock: - rcu_read_unlock(); - return kick; + rcu_read_unlock(); + return kick; } #else static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) { } @@ -8065,20 +8070,20 @@ static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) { } */ static void run_rebalance_domains(struct softirq_action *h) { - struct rq *this_rq = this_rq(); - enum cpu_idle_type idle = this_rq->idle_balance ? - CPU_IDLE : CPU_NOT_IDLE; + struct rq *this_rq = this_rq(); + enum cpu_idle_type idle = this_rq->idle_balance ? + CPU_IDLE : CPU_NOT_IDLE; - /* - * If this cpu has a pending nohz_balance_kick, then do the - * balancing on behalf of the other idle cpus whose ticks are - * stopped. Do nohz_idle_balance *before* rebalance_domains to - * give the idle cpus a chance to load balance. Else we may - * load balance only within the local sched_domain hierarchy - * and abort nohz_idle_balance altogether if we pull some load. - */ - nohz_idle_balance(this_rq, idle); - rebalance_domains(this_rq, idle); + /* + * If this cpu has a pending nohz_balance_kick, then do the + * balancing on behalf of the other idle cpus whose ticks are + * stopped. Do nohz_idle_balance *before* rebalance_domains to + * give the idle cpus a chance to load balance. Else we may + * load balance only within the local sched_domain hierarchy + * and abort nohz_idle_balance altogether if we pull some load. + */ + nohz_idle_balance(this_rq, idle); + rebalance_domains(this_rq, idle); } /* @@ -8086,31 +8091,31 @@ static void run_rebalance_domains(struct softirq_action *h) */ void trigger_load_balance(struct rq *rq) { - /* Don't need to rebalance while attached to NULL domain */ - if (unlikely(on_null_domain(rq))) - return; + /* Don't need to rebalance while attached to NULL domain */ + if (unlikely(on_null_domain(rq))) + return; - if (time_after_eq(jiffies, rq->next_balance)) - raise_softirq(SCHED_SOFTIRQ); + if (time_after_eq(jiffies, rq->next_balance)) + raise_softirq(SCHED_SOFTIRQ); #ifdef CONFIG_NO_HZ_COMMON - if (nohz_kick_needed(rq)) - nohz_balancer_kick(); + if (nohz_kick_needed(rq)) + nohz_balancer_kick(); #endif } static void rq_online_fair(struct rq *rq) { - update_sysctl(); + update_sysctl(); - update_runtime_enabled(rq); + update_runtime_enabled(rq); } static void rq_offline_fair(struct rq *rq) { - update_sysctl(); + update_sysctl(); - /* Ensure any throttled groups are reachable by pick_next_task */ - unthrottle_offline_cfs_rqs(rq); + /* Ensure any throttled groups are reachable by pick_next_task */ + unthrottle_offline_cfs_rqs(rq); } #endif /* CONFIG_SMP */ @@ -8120,16 +8125,16 @@ static void rq_offline_fair(struct rq *rq) */ static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued) { - struct cfs_rq *cfs_rq; - struct sched_entity *se = &curr->se; + struct cfs_rq *cfs_rq; + struct sched_entity *se = &curr->se; - for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); - entity_tick(cfs_rq, se, queued); - } + for_each_sched_entity(se) { + cfs_rq = cfs_rq_of(se); + entity_tick(cfs_rq, se, queued); + } - if (static_branch_unlikely(&sched_numa_balancing)) - task_tick_numa(rq, curr); + if (static_branch_unlikely(&sched_numa_balancing)) + task_tick_numa(rq, curr); } /* @@ -8139,47 +8144,47 @@ static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued) */ static void task_fork_fair(struct task_struct *p) { - struct cfs_rq *cfs_rq; - struct sched_entity *se = &p->se, *curr; - int this_cpu = smp_processor_id(); - struct rq *rq = this_rq(); - unsigned long flags; + struct cfs_rq *cfs_rq; + struct sched_entity *se = &p->se, *curr; + int this_cpu = smp_processor_id(); + struct rq *rq = this_rq(); + unsigned long flags; - raw_spin_lock_irqsave(&rq->lock, flags); + raw_spin_lock_irqsave(&rq->lock, flags); - update_rq_clock(rq); + update_rq_clock(rq); - cfs_rq = task_cfs_rq(current); - curr = cfs_rq->curr; + cfs_rq = task_cfs_rq(current); + curr = cfs_rq->curr; + /* + * Not only the cpu but also the task_group of the parent might have + * been changed after parent->se.parent,cfs_rq were copied to + * child->se.parent,cfs_rq. So call __set_task_cpu() to make those + * of child point to valid ones. + */ + rcu_read_lock(); + __set_task_cpu(p, this_cpu); + rcu_read_unlock(); + + update_curr(cfs_rq); + + if (curr) + se->vruntime = curr->vruntime; + place_entity(cfs_rq, se, 1); + + if (sysctl_sched_child_runs_first && curr && entity_before(curr, se)) { /* - * Not only the cpu but also the task_group of the parent might have - * been changed after parent->se.parent,cfs_rq were copied to - * child->se.parent,cfs_rq. So call __set_task_cpu() to make those - * of child point to valid ones. + * Upon rescheduling, sched_class::put_prev_task() will place + * 'current' within the tree based on its new key value. */ - rcu_read_lock(); - __set_task_cpu(p, this_cpu); - rcu_read_unlock(); + swap(curr->vruntime, se->vruntime); + resched_curr(rq); + } - update_curr(cfs_rq); + se->vruntime -= cfs_rq->min_vruntime; - if (curr) - se->vruntime = curr->vruntime; - place_entity(cfs_rq, se, 1); - - if (sysctl_sched_child_runs_first && curr && entity_before(curr, se)) { - /* - * Upon rescheduling, sched_class::put_prev_task() will place - * 'current' within the tree based on its new key value. - */ - swap(curr->vruntime, se->vruntime); - resched_curr(rq); - } - - se->vruntime -= cfs_rq->min_vruntime; - - raw_spin_unlock_irqrestore(&rq->lock, flags); + raw_spin_unlock_irqrestore(&rq->lock, flags); } /* @@ -8189,106 +8194,106 @@ static void task_fork_fair(struct task_struct *p) static void prio_changed_fair(struct rq *rq, struct task_struct *p, int oldprio) { - if (!task_on_rq_queued(p)) - return; + if (!task_on_rq_queued(p)) + return; - /* - * Reschedule if we are currently running on this runqueue and - * our priority decreased, or if we are not currently running on - * this runqueue and our priority is higher than the current's - */ - if (rq->curr == p) { - if (p->prio > oldprio) - resched_curr(rq); - } else - check_preempt_curr(rq, p, 0); + /* + * Reschedule if we are currently running on this runqueue and + * our priority decreased, or if we are not currently running on + * this runqueue and our priority is higher than the current's + */ + if (rq->curr == p) { + if (p->prio > oldprio) + resched_curr(rq); + } else + check_preempt_curr(rq, p, 0); } static inline bool vruntime_normalized(struct task_struct *p) { - struct sched_entity *se = &p->se; + struct sched_entity *se = &p->se; - /* - * In both the TASK_ON_RQ_QUEUED and TASK_ON_RQ_MIGRATING cases, - * the dequeue_entity(.flags=0) will already have normalized the - * vruntime. - */ - if (p->on_rq) - return true; + /* + * In both the TASK_ON_RQ_QUEUED and TASK_ON_RQ_MIGRATING cases, + * the dequeue_entity(.flags=0) will already have normalized the + * vruntime. + */ + if (p->on_rq) + return true; - /* - * When !on_rq, vruntime of the task has usually NOT been normalized. - * But there are some cases where it has already been normalized: - * - * - A forked child which is waiting for being woken up by - * wake_up_new_task(). - * - A task which has been woken up by try_to_wake_up() and - * waiting for actually being woken up by sched_ttwu_pending(). - */ - if (!se->sum_exec_runtime || p->state == TASK_WAKING) - return true; + /* + * When !on_rq, vruntime of the task has usually NOT been normalized. + * But there are some cases where it has already been normalized: + * + * - A forked child which is waiting for being woken up by + * wake_up_new_task(). + * - A task which has been woken up by try_to_wake_up() and + * waiting for actually being woken up by sched_ttwu_pending(). + */ + if (!se->sum_exec_runtime || p->state == TASK_WAKING) + return true; - return false; + return false; } static void detach_task_cfs_rq(struct task_struct *p) { - struct sched_entity *se = &p->se; - struct cfs_rq *cfs_rq = cfs_rq_of(se); + struct sched_entity *se = &p->se; + struct cfs_rq *cfs_rq = cfs_rq_of(se); - if (!vruntime_normalized(p)) { - /* - * Fix up our vruntime so that the current sleep doesn't - * cause 'unlimited' sleep bonus. - */ - place_entity(cfs_rq, se, 0); - se->vruntime -= cfs_rq->min_vruntime; - } + if (!vruntime_normalized(p)) { + /* + * Fix up our vruntime so that the current sleep doesn't + * cause 'unlimited' sleep bonus. + */ + place_entity(cfs_rq, se, 0); + se->vruntime -= cfs_rq->min_vruntime; + } - /* Catch up with the cfs_rq and remove our load when we leave */ - detach_entity_load_avg(cfs_rq, se); + /* Catch up with the cfs_rq and remove our load when we leave */ + detach_entity_load_avg(cfs_rq, se); } static void attach_task_cfs_rq(struct task_struct *p) { - struct sched_entity *se = &p->se; - struct cfs_rq *cfs_rq = cfs_rq_of(se); + struct sched_entity *se = &p->se; + struct cfs_rq *cfs_rq = cfs_rq_of(se); #ifdef CONFIG_FAIR_GROUP_SCHED - /* - * Since the real-depth could have been changed (only FAIR - * class maintain depth value), reset depth properly. - */ - se->depth = se->parent ? se->parent->depth + 1 : 0; + /* + * Since the real-depth could have been changed (only FAIR + * class maintain depth value), reset depth properly. + */ + se->depth = se->parent ? se->parent->depth + 1 : 0; #endif - /* Synchronize task with its cfs_rq */ - attach_entity_load_avg(cfs_rq, se); + /* Synchronize task with its cfs_rq */ + attach_entity_load_avg(cfs_rq, se); - if (!vruntime_normalized(p)) - se->vruntime += cfs_rq->min_vruntime; + if (!vruntime_normalized(p)) + se->vruntime += cfs_rq->min_vruntime; } static void switched_from_fair(struct rq *rq, struct task_struct *p) { - detach_task_cfs_rq(p); + detach_task_cfs_rq(p); } static void switched_to_fair(struct rq *rq, struct task_struct *p) { - attach_task_cfs_rq(p); + attach_task_cfs_rq(p); - if (task_on_rq_queued(p)) { - /* - * We were most likely switched from sched_rt, so - * kick off the schedule if running, otherwise just see - * if we can still preempt the current task. - */ - if (rq->curr == p) - resched_curr(rq); - else - check_preempt_curr(rq, p, 0); - } + if (task_on_rq_queued(p)) { + /* + * We were most likely switched from sched_rt, so + * kick off the schedule if running, otherwise just see + * if we can still preempt the current task. + */ + if (rq->curr == p) + resched_curr(rq); + else + check_preempt_curr(rq, p, 0); + } } /* Account for a task changing its policy or group. @@ -8298,195 +8303,195 @@ static void switched_to_fair(struct rq *rq, struct task_struct *p) */ static void set_curr_task_fair(struct rq *rq) { - struct sched_entity *se = &rq->curr->se; + struct sched_entity *se = &rq->curr->se; - for_each_sched_entity(se) { - struct cfs_rq *cfs_rq = cfs_rq_of(se); + for_each_sched_entity(se) { + struct cfs_rq *cfs_rq = cfs_rq_of(se); - set_next_entity(cfs_rq, se); - /* ensure bandwidth has been allocated on our new cfs_rq */ - account_cfs_rq_runtime(cfs_rq, 0); - } + set_next_entity(cfs_rq, se); + /* ensure bandwidth has been allocated on our new cfs_rq */ + account_cfs_rq_runtime(cfs_rq, 0); + } } void init_cfs_rq(struct cfs_rq *cfs_rq) { - cfs_rq->tasks_timeline = RB_ROOT; - cfs_rq->min_vruntime = (u64)(-(1LL << 20)); + cfs_rq->tasks_timeline = RB_ROOT; + cfs_rq->min_vruntime = (u64)(-(1LL << 20)); #ifndef CONFIG_64BIT - cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime; + cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime; #endif #ifdef CONFIG_SMP - atomic_long_set(&cfs_rq->removed_load_avg, 0); - atomic_long_set(&cfs_rq->removed_util_avg, 0); + atomic_long_set(&cfs_rq->removed_load_avg, 0); + atomic_long_set(&cfs_rq->removed_util_avg, 0); #endif } #ifdef CONFIG_FAIR_GROUP_SCHED static void task_move_group_fair(struct task_struct *p) { - detach_task_cfs_rq(p); - set_task_rq(p, task_cpu(p)); + detach_task_cfs_rq(p); + set_task_rq(p, task_cpu(p)); #ifdef CONFIG_SMP - /* Tell se's cfs_rq has been changed -- migrated */ - p->se.avg.last_update_time = 0; + /* Tell se's cfs_rq has been changed -- migrated */ + p->se.avg.last_update_time = 0; #endif - attach_task_cfs_rq(p); + attach_task_cfs_rq(p); } void free_fair_sched_group(struct task_group *tg) { - int i; + int i; - destroy_cfs_bandwidth(tg_cfs_bandwidth(tg)); + destroy_cfs_bandwidth(tg_cfs_bandwidth(tg)); - for_each_possible_cpu(i) { - if (tg->cfs_rq) - kfree(tg->cfs_rq[i]); - if (tg->se) - kfree(tg->se[i]); - } + for_each_possible_cpu(i) { + if (tg->cfs_rq) + kfree(tg->cfs_rq[i]); + if (tg->se) + kfree(tg->se[i]); + } - kfree(tg->cfs_rq); - kfree(tg->se); + kfree(tg->cfs_rq); + kfree(tg->se); } int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) { - struct cfs_rq *cfs_rq; - struct sched_entity *se; - int i; + struct cfs_rq *cfs_rq; + struct sched_entity *se; + int i; - tg->cfs_rq = kzalloc(sizeof(cfs_rq) * nr_cpu_ids, GFP_KERNEL); - if (!tg->cfs_rq) - goto err; - tg->se = kzalloc(sizeof(se) * nr_cpu_ids, GFP_KERNEL); - if (!tg->se) - goto err; + tg->cfs_rq = kzalloc(sizeof(cfs_rq) * nr_cpu_ids, GFP_KERNEL); + if (!tg->cfs_rq) + goto err; + tg->se = kzalloc(sizeof(se) * nr_cpu_ids, GFP_KERNEL); + if (!tg->se) + goto err; - tg->shares = NICE_0_LOAD; + tg->shares = NICE_0_LOAD; - init_cfs_bandwidth(tg_cfs_bandwidth(tg)); + init_cfs_bandwidth(tg_cfs_bandwidth(tg)); - for_each_possible_cpu(i) { - cfs_rq = kzalloc_node(sizeof(struct cfs_rq), - GFP_KERNEL, cpu_to_node(i)); - if (!cfs_rq) - goto err; + for_each_possible_cpu(i) { + cfs_rq = kzalloc_node(sizeof(struct cfs_rq), + GFP_KERNEL, cpu_to_node(i)); + if (!cfs_rq) + goto err; - se = kzalloc_node(sizeof(struct sched_entity), - GFP_KERNEL, cpu_to_node(i)); - if (!se) - goto err_free_rq; + se = kzalloc_node(sizeof(struct sched_entity), + GFP_KERNEL, cpu_to_node(i)); + if (!se) + goto err_free_rq; - init_cfs_rq(cfs_rq); - init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]); - init_entity_runnable_average(se); - } + init_cfs_rq(cfs_rq); + init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]); + init_entity_runnable_average(se); + } - return 1; + return 1; err_free_rq: - kfree(cfs_rq); + kfree(cfs_rq); err: - return 0; + return 0; } void unregister_fair_sched_group(struct task_group *tg) { - unsigned long flags; - struct rq *rq; - int cpu; + unsigned long flags; + struct rq *rq; + int cpu; - for_each_possible_cpu(cpu) { - if (tg->se[cpu]) - remove_entity_load_avg(tg->se[cpu]); + for_each_possible_cpu(cpu) { + if (tg->se[cpu]) + remove_entity_load_avg(tg->se[cpu]); - /* - * Only empty task groups can be destroyed; so we can speculatively - * check on_list without danger of it being re-added. - */ - if (!tg->cfs_rq[cpu]->on_list) - continue; + /* + * Only empty task groups can be destroyed; so we can speculatively + * check on_list without danger of it being re-added. + */ + if (!tg->cfs_rq[cpu]->on_list) + continue; - rq = cpu_rq(cpu); + rq = cpu_rq(cpu); - raw_spin_lock_irqsave(&rq->lock, flags); - list_del_leaf_cfs_rq(tg->cfs_rq[cpu]); - raw_spin_unlock_irqrestore(&rq->lock, flags); - } + raw_spin_lock_irqsave(&rq->lock, flags); + list_del_leaf_cfs_rq(tg->cfs_rq[cpu]); + raw_spin_unlock_irqrestore(&rq->lock, flags); + } } void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq, - struct sched_entity *se, int cpu, - struct sched_entity *parent) + struct sched_entity *se, int cpu, + struct sched_entity *parent) { - struct rq *rq = cpu_rq(cpu); + struct rq *rq = cpu_rq(cpu); - cfs_rq->tg = tg; - cfs_rq->rq = rq; - init_cfs_rq_runtime(cfs_rq); + cfs_rq->tg = tg; + cfs_rq->rq = rq; + init_cfs_rq_runtime(cfs_rq); - tg->cfs_rq[cpu] = cfs_rq; - tg->se[cpu] = se; + tg->cfs_rq[cpu] = cfs_rq; + tg->se[cpu] = se; - /* se could be NULL for root_task_group */ - if (!se) - return; + /* se could be NULL for root_task_group */ + if (!se) + return; - if (!parent) { - se->cfs_rq = &rq->cfs; - se->depth = 0; - } else { - se->cfs_rq = parent->my_q; - se->depth = parent->depth + 1; - } + if (!parent) { + se->cfs_rq = &rq->cfs; + se->depth = 0; + } else { + se->cfs_rq = parent->my_q; + se->depth = parent->depth + 1; + } - se->my_q = cfs_rq; - /* guarantee group entities always have weight */ - update_load_set(&se->load, NICE_0_LOAD); - se->parent = parent; + se->my_q = cfs_rq; + /* guarantee group entities always have weight */ + update_load_set(&se->load, NICE_0_LOAD); + se->parent = parent; } static DEFINE_MUTEX(shares_mutex); int sched_group_set_shares(struct task_group *tg, unsigned long shares) { - int i; - unsigned long flags; + int i; + unsigned long flags; - /* - * We can't change the weight of the root cgroup. - */ - if (!tg->se[0]) - return -EINVAL; + /* + * We can't change the weight of the root cgroup. + */ + if (!tg->se[0]) + return -EINVAL; - shares = clamp(shares, scale_load(MIN_SHARES), scale_load(MAX_SHARES)); + shares = clamp(shares, scale_load(MIN_SHARES), scale_load(MAX_SHARES)); - mutex_lock(&shares_mutex); - if (tg->shares == shares) - goto done; + mutex_lock(&shares_mutex); + if (tg->shares == shares) + goto done; - tg->shares = shares; - for_each_possible_cpu(i) { - struct rq *rq = cpu_rq(i); - struct sched_entity *se; + tg->shares = shares; + for_each_possible_cpu(i) { + struct rq *rq = cpu_rq(i); + struct sched_entity *se; - se = tg->se[i]; - /* Propagate contribution to hierarchy */ - raw_spin_lock_irqsave(&rq->lock, flags); + se = tg->se[i]; + /* Propagate contribution to hierarchy */ + raw_spin_lock_irqsave(&rq->lock, flags); - /* Possible calls to update_curr() need rq clock */ - update_rq_clock(rq); - for_each_sched_entity(se) - update_cfs_shares(group_cfs_rq(se)); - raw_spin_unlock_irqrestore(&rq->lock, flags); - } + /* Possible calls to update_curr() need rq clock */ + update_rq_clock(rq); + for_each_sched_entity(se) + update_cfs_shares(group_cfs_rq(se)); + raw_spin_unlock_irqrestore(&rq->lock, flags); + } done: - mutex_unlock(&shares_mutex); - return 0; + mutex_unlock(&shares_mutex); + return 0; } #else /* CONFIG_FAIR_GROUP_SCHED */ @@ -8494,7 +8499,7 @@ void free_fair_sched_group(struct task_group *tg) { } int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) { - return 1; + return 1; } void unregister_fair_sched_group(struct task_group *tg) { } @@ -8504,91 +8509,91 @@ void unregister_fair_sched_group(struct task_group *tg) { } static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task) { - struct sched_entity *se = &task->se; - unsigned int rr_interval = 0; + struct sched_entity *se = &task->se; + unsigned int rr_interval = 0; - /* - * Time slice is 0 for SCHED_OTHER tasks that are on an otherwise - * idle runqueue: - */ - if (rq->cfs.load.weight) - rr_interval = NS_TO_JIFFIES(sched_slice(cfs_rq_of(se), se)); + /* + * Time slice is 0 for SCHED_OTHER tasks that are on an otherwise + * idle runqueue: + */ + if (rq->cfs.load.weight) + rr_interval = NS_TO_JIFFIES(sched_slice(cfs_rq_of(se), se)); - return rr_interval; + return rr_interval; } /* * All the scheduling class methods: */ const struct sched_class fair_sched_class = { - .next = &idle_sched_class, /* 下个优先级的调度类, 所有的调度类通过next链接在一个链表中*/ - .enqueue_task = enqueue_task_fair, - .dequeue_task = dequeue_task_fair, - .yield_task = yield_task_fair, - .yield_to_task = yield_to_task_fair, + .next = &idle_sched_class, /* 下个优先级的调度类, 所有的调度类通过next链接在一个链表中*/ + .enqueue_task = enqueue_task_fair, + .dequeue_task = dequeue_task_fair, + .yield_task = yield_task_fair, + .yield_to_task = yield_to_task_fair, - .check_preempt_curr = check_preempt_wakeup, + .check_preempt_curr = check_preempt_wakeup, - .pick_next_task = pick_next_task_fair, - .put_prev_task = put_prev_task_fair, + .pick_next_task = pick_next_task_fair, + .put_prev_task = put_prev_task_fair, #ifdef CONFIG_SMP - .select_task_rq = select_task_rq_fair, - .migrate_task_rq = migrate_task_rq_fair, + .select_task_rq = select_task_rq_fair, + .migrate_task_rq = migrate_task_rq_fair, - .rq_online = rq_online_fair, - .rq_offline = rq_offline_fair, + .rq_online = rq_online_fair, + .rq_offline = rq_offline_fair, - .task_waking = task_waking_fair, - .task_dead = task_dead_fair, - .set_cpus_allowed = set_cpus_allowed_common, + .task_waking = task_waking_fair, + .task_dead = task_dead_fair, + .set_cpus_allowed = set_cpus_allowed_common, #endif - .set_curr_task = set_curr_task_fair, - .task_tick = task_tick_fair, - .task_fork = task_fork_fair, + .set_curr_task = set_curr_task_fair, + .task_tick = task_tick_fair, + .task_fork = task_fork_fair, - .prio_changed = prio_changed_fair, - .switched_from = switched_from_fair, - .switched_to = switched_to_fair, + .prio_changed = prio_changed_fair, + .switched_from = switched_from_fair, + .switched_to = switched_to_fair, - .get_rr_interval = get_rr_interval_fair, + .get_rr_interval = get_rr_interval_fair, - .update_curr = update_curr_fair, + .update_curr = update_curr_fair, #ifdef CONFIG_FAIR_GROUP_SCHED - .task_move_group = task_move_group_fair, + .task_move_group = task_move_group_fair, #endif }; #ifdef CONFIG_SCHED_DEBUG void print_cfs_stats(struct seq_file *m, int cpu) { - struct cfs_rq *cfs_rq; + struct cfs_rq *cfs_rq; - rcu_read_lock(); - for_each_leaf_cfs_rq(cpu_rq(cpu), cfs_rq) - print_cfs_rq(m, cpu, cfs_rq); - rcu_read_unlock(); + rcu_read_lock(); + for_each_leaf_cfs_rq(cpu_rq(cpu), cfs_rq) + print_cfs_rq(m, cpu, cfs_rq); + rcu_read_unlock(); } #ifdef CONFIG_NUMA_BALANCING void show_numa_stats(struct task_struct *p, struct seq_file *m) { - int node; - unsigned long tsf = 0, tpf = 0, gsf = 0, gpf = 0; + int node; + unsigned long tsf = 0, tpf = 0, gsf = 0, gpf = 0; - for_each_online_node(node) { - if (p->numa_faults) { - tsf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 0)]; - tpf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 1)]; - } - if (p->numa_group) { - gsf = p->numa_group->faults[task_faults_idx(NUMA_MEM, node, 0)], - gpf = p->numa_group->faults[task_faults_idx(NUMA_MEM, node, 1)]; - } - print_numa_stats(m, node, tsf, tpf, gsf, gpf); + for_each_online_node(node) { + if (p->numa_faults) { + tsf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 0)]; + tpf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 1)]; } + if (p->numa_group) { + gsf = p->numa_group->faults[task_faults_idx(NUMA_MEM, node, 0)], + gpf = p->numa_group->faults[task_faults_idx(NUMA_MEM, node, 1)]; + } + print_numa_stats(m, node, tsf, tpf, gsf, gpf); + } } #endif /* CONFIG_NUMA_BALANCING */ #endif /* CONFIG_SCHED_DEBUG */ @@ -8596,12 +8601,12 @@ void show_numa_stats(struct task_struct *p, struct seq_file *m) __init void init_sched_fair_class(void) { #ifdef CONFIG_SMP - open_softirq(SCHED_SOFTIRQ, run_rebalance_domains); + open_softirq(SCHED_SOFTIRQ, run_rebalance_domains); #ifdef CONFIG_NO_HZ_COMMON - nohz.next_balance = jiffies; - zalloc_cpumask_var(&nohz.idle_cpus_mask, GFP_NOWAIT); - cpu_notifier(sched_ilb_notifier, 0); + nohz.next_balance = jiffies; + zalloc_cpumask_var(&nohz.idle_cpus_mask, GFP_NOWAIT); + cpu_notifier(sched_ilb_notifier, 0); #endif #endif /* SMP */ diff --git a/study/kernel/01-process/05-schedule/04-cfs/01-load_weight/README.md b/study/kernel/01-process/05-schedule/04-cfs/02-load_weight/README.md similarity index 97% rename from study/kernel/01-process/05-schedule/04-cfs/01-load_weight/README.md rename to study/kernel/01-process/05-schedule/04-cfs/02-load_weight/README.md index ec7f057..17d7997 100644 --- a/study/kernel/01-process/05-schedule/04-cfs/01-load_weight/README.md +++ b/study/kernel/01-process/05-schedule/04-cfs/02-load_weight/README.md @@ -1,615 +1,615 @@ -Linux进程负荷权重 -======= - - -| 日期 | 内核版本 | 架构| 作者 | GitHub| CSDN | -| ------- |:-------:|:-------:|:-------:|:-------:|:-------:| -| 2016-06-14 | [Linux-4.6](http://lxr.free-electrons.com/source/?v=4.6) | X86 & arm | [gatieme](http://blog.csdn.net/gatieme) | [LinuxDeviceDrivers](https://github.com/gatieme/LDD-LinuxDeviceDrivers) | [Linux进程管理与调度](http://blog.csdn.net/gatieme/article/category/6225543) | - - - -前面我们详细的了解了linux下进程优先级的表示以及其计算的方法, 我们了解到linux针对普通进程和实时进程分别使用静态优先级static_prio和实时优先级rt_priority来指定其默认的优先级别, 然后通过normal_prio函数将他们分别转换为普通优先级normal_prio, 最终换算出动态优先级prio, 动态优先级prio才是内核调度时候有限考虑的优先级字段 - -但是CFS完全公平调度器在调度进程的时候, 进程的重要性不仅是由优先级指定的, 而且还需要考虑保存在task_struct->se.load的负荷权重. - - - -#前景回顾 -------- - - -##进程调度 -------- - -内存中保存了对每个进程的唯一描述, 并通过若干结构与其他进程连接起来. - -**调度器**面对的情形就是这样, 其任务是在程序之间共享CPU时间, 创造并行执行的错觉, 该任务分为两个不同的部分, 其中一个涉及**调度策略**, 另外一个涉及**上下文切换**. - - -内核必须提供一种方法, 在各个进程之间尽可能公平地共享CPU时间, 而同时又要考虑不同的任务优先级. - -调度器的一般原理是, 按所需分配的计算能力, 向系统中每个进程提供最大的公正性, 或者从另外一个角度上说, 他试图确保没有进程被亏待. - - -##进程的分类 -------- - -linux把进程区分为实时进程和非实时进程, 其中非实时进程进一步划分为交互式进程和批处理进程 - -| 类型 | 描述 | 示例 | -| ------- |:-------:|:-------:|:-------:| -| 交互式进程(interactive process) | 此类进程经常与用户进行交互, 因此需要花费很多时间等待键盘和鼠标操作. 当接受了用户的输入后, 进程必须很快被唤醒, 否则用户会感觉系统反应迟钝 | shell, 文本编辑程序和图形应用程序 | -| 批处理进程(batch process) | 此类进程不必与用户交互, 因此经常在后台运行. 因为这样的进程不必很快相应, 因此常受到调度程序的怠慢 | 程序语言的编译程序, 数据库搜索引擎以及科学计算 | -| 实时进程(real-time process) | 这些进程由很强的调度需要, 这样的进程绝不会被低优先级的进程阻塞. 并且他们的响应时间要尽可能的短 | 视频音频应用程序, 机器人控制程序以及从物理传感器上收集数据的程序| - - -##不同进程采用不同的调度策略 -------- - -根据进程的不同分类Linux采用不同的调度策略. - -对于实时进程，采用FIFO, Round Robin或者Earliest Deadline First (EDF)最早截止期限优先调度算法|的调度策略. - -但是普通进程的调度策略就比较麻烦了, 因为普通进程不能简单的只看优先级, 必须公平的占有CPU, 否则很容易出现进程饥饿, 这种情况下用户会感觉操作系统很卡, 响应总是很慢，因此在linux调度器的发展历程中经过了多次重大变动, linux总是希望寻找一个最接近于完美的调度策略来公平快速的调度进程. - - -##linux调度器的演变 -------- - - -一开始的调度器是复杂度为**$O(n)$的始调度算法**(实际上每次会遍历所有任务，所以复杂度为O(n)), 这个算法的缺点是当内核中有很多任务时，调度器本身就会耗费不少时间，所以，从linux2.5开始引入赫赫有名的**$O(1)$调度器** - -然而，linux是集全球很多程序员的聪明才智而发展起来的超级内核，没有最好，只有更好，在$O(1)$调度器风光了没几天就又被另一个更优秀的调度器取代了，它就是**CFS调度器Completely Fair Scheduler**. 这个也是在2.6内核中引入的，具体为2.6.23，即从此版本开始，内核使用CFS作为它的默认调度器，$O(1)$调度器被抛弃了, 其实CFS的发展也是经历了很多阶段，最早期的楼梯算法(SD), 后来逐步对SD算法进行改进出RSDL(Rotating Staircase Deadline Scheduler), 这个算法已经是"完全公平"的雏形了， 直至CFS是最终被内核采纳的调度器, 它从RSDL/SD中吸取了完全公平的思想，不再跟踪进程的睡眠时间，也不再企图区分交互式进程。它将所有的进程都统一对待，这就是公平的含义。CFS的算法和实现都相当简单，众多的测试表明其性能也非常优越 - - - -| 字段 | 版本 | -| ------------- |:-------------:|:-------------:| -| O(n)的始调度算法 | linux-0.11~2.4 | -| O(1)调度器 | linux-2.5 | -| CFS调度器 | linux-2.6~至今 | - - -##Linux的调度器组成 -------- - - -**2个调度器** - -可以用两种方法来激活调度 - -* 一种是直接的, 比如进程打算睡眠或出于其他原因放弃CPU - -* 另一种是通过周期性的机制, 以固定的频率运行, 不时的检测是否有必要 - -因此当前linux的调度程序由两个调度器组成：**主调度器**，**周期性调度器**(两者又统称为**通用调度器(generic scheduler)**或**核心调度器(core scheduler)**) - -并且每个调度器包括两个内容：**调度框架**(其实质就是两个函数框架)及**调度器类** - - - -**6种调度策略** - -linux内核目前实现了6中调度策略(即调度算法), 用于对不同类型的进程进行调度, 或者支持某些特殊的功能 - -* SCHED_NORMAL和SCHED_BATCH调度普通的非实时进程 - -* SCHED_FIFO和SCHED_RR和SCHED_DEADLINE则采用不同的调度策略调度实时进程 - -* SCHED_IDLE则在系统空闲时调用idle进程. - - - -**5个调度器类** - -而依据其调度策略的不同实现了5个调度器类, 一个调度器类可以用一种种或者多种调度策略调度某一类进程, 也可以用于特殊情况或者调度特殊功能的进程. - - -其所属进程的优先级顺序为 -````c -stop_sched_class -> dl_sched_class -> rt_sched_class -> fair_sched_class -> idle_sched_class -``` - -**3个调度实体** - -调度器不限于调度进程, 还可以调度更大的实体, 比如实现组调度. - -这种一般性要求调度器不直接操作进程, 而是处理可调度实体, 因此需要一个通用的数据结构描述这个调度实体,即seched_entity结构, 其实际上就代表了一个调度对象，可以为一个进程，也可以为一个进程组. - -linux中针对当前可调度的实时和非实时进程, 定义了类型为seched_entity的3个调度实体 - -* sched_dl_entity 采用EDF算法调度的实时调度实体 - -* sched_rt_entity 采用Roound-Robin或者FIFO算法调度的实时调度实体 - -* sched_entity 采用CFS算法调度的普通非实时进程的调度实体 - - -**调度器整体框架** - -每个进程都属于某个调度器类(由字段task_struct->sched_class标识), 由调度器类采用进程对应的调度策略调度(由task_struct->policy )进行调度, task_struct也存储了其对应的调度实体标识 - -linux实现了6种调度策略, 依据其调度策略的不同实现了5个调度器类, 一个调度器类可以用一种或者多种调度策略调度某一类进程, 也可以用于特殊情况或者调度特殊功能的进程. - - -| 调度器类 | 调度策略 | 调度策略对应的调度算法 | 调度实体 | 调度实体对应的调度对象 | -| ------- |:-------:|:-------:|:-------:||:-------:| -| stop_sched_class | 无 | 无 | 无 | 特殊情况, 发生在cpu_stop_cpu_callback 进行cpu之间任务迁移migration或者HOTPLUG_CPU的情况下关闭任务 | -| dl_sched_class | SCHED_DEADLINE | Earliest-Deadline-First最早截至时间有限算法 | sched_dl_entity | 采用DEF最早截至时间有限算法调度实时进程 | -| rt_sched_class | SCHED_RR

SCHED_FIFO | Roound-Robin时间片轮转算法

FIFO先进先出算法 | sched_rt_entity | 采用Roound-Robin或者FIFO算法调度的实时调度实体 | -| fair_sched_class | SCHED_NORMAL

SCHED_BATCH | CFS完全公平懂调度算法 |sched_entity | 采用CFS算法普通非实时进程 | -| idle_sched_class | SCHED_IDLE | 无 | 无 |特殊进程, 用于cpu空闲时调度空闲进程idle | - -它们的关系如下图 - -![调度器的组成](../images/level.jpg) - - -##优先级的内核表示 -------- - - -内核使用一些简单的数值范围0~139表示内部优先级, 数值越低, 优先级越高。 - -从0~99的范围专供实时进程使用, nice的值[-20,19]则映射到范围100~139 - - ->实时优先级范围是0到MAX_RT_PRIO-1（即99），而普通进程的静态优先级范围是从MAX_RT_PRIO到MAX_PRIO-1（即100到139）。 - -| 优先级范围 | 描述 | -| ------------- |:-------------:| -| 0——99 | 实时进程 | -| 100——139 | 非实时进程 | - -![内核优先级标度](../images/priority.jpg) - -##进程的优先级表示 -------- - - -```c -struct task_struct -{ - /* 进程优先级 - * prio: 动态优先级，范围为100~139，与静态优先级和补偿(bonus)有关 - * static_prio: 静态优先级，static_prio = 100 + nice + 20 (nice值为-20~19,所以static_prio值为100~139) - * normal_prio: 没有受优先级继承影响的常规优先级，具体见normal_prio函数，跟属于什么类型的进程有关 - */ - int prio, static_prio, normal_prio; - /* 实时进程优先级 */ - unsigned int rt_priority; -} -``` - -**动态优先级 静态优先级 实时优先级** - - -其中task_struct采用了三个成员表示进程的优先级:prio和normal_prio表示动态优先级, static_prio表示进程的静态优先级. - -此外还用了一个字段rt_priority保存了实时进程的优先级 - - -| 字段 | 描述 | -| ------------- |:-------------:| -| static_prio | 用于保存静态优先级, 是进程启动时分配的优先级, ，可以通过nice和sched_setscheduler系统调用来进行修改, 否则在进程运行期间会一直保持恒定 | -| prio | 保存进程的动态优先级 | -| normal_prio | 表示基于进程的静态优先级static_prio和调度策略计算出的优先级. 因此即使普通进程和实时进程具有相同的静态优先级, 其普通优先级也是不同的, 进程分叉(fork)时, 子进程会继承父进程的普通优先级 | -| rt_priority | 用于保存实时优先级 | - - -实时进程的优先级用实时优先级rt_priority来表示 - - - -#负荷权重 -------- - - -##负荷权重结构struct load_weight -------- - -负荷权重用struct load_weight数据结构来表示, 保存着进程权重值weight。其定义在[/include/linux/sched.h, v=4.6, L1195](http://lxr.free-electrons.com/source/include/linux/sched.h?v=4.6#L1195), 如下所示 - -```c -struct load_weight { - unsigned long weight; /* 存储了权重的信息 */ - u32 inv_weight; /* 存储了权重值用于重除的结果 weight * inv_weight = 2^32 */ -}; -``` - - - -##调度实体的负荷权重load -------- - -既然struct load_weight保存着进程的权重信息, 那么作为进程调度的实体, 必须将这个权重值与特定的进程task_struct, 更一般的与通用的调度实体sched_entity相关联 - -struct sched_entity作为进程调度的实体信息, 其内置了load_weight结构用于保存当前调度实体的权重, 参照http://lxr.free-electrons.com/source/include/linux/sched.h?v=4.6#L1195 - -```c -struct sched_entity { - struct load_weight load; /* for load-balancing */ - /* ...... */ -}; -``` - -##进程的负荷权重 - -而进程可以被作为一个调度的实时, 其内部通过存储struct sched_entity se而间接存储了其load_weight信息, 参照http://lxr.free-electrons.com/source/include/linux/sched.h?v=4.6#L1415 - -```c -struct task_struct -{ - /* ...... */ - struct sched_entity se; - /* ...... */ -} -``` - -因此我们就可以通过task_statuct->se.load获取负荷权重的信息, 而set_load_weight负责根据进程类型及其静态优先级计算符合权重. - - -#优先级和权重的转换 -------- - - -##优先级->权重转换表 -------- - - -一般这个概念是这样的, 进程每降低一个nice值(优先级提升), 则多获得10%的CPU时间, 没升高一个nice值(优先级降低), 则放弃10%的CPU时间. - -为执行该策略, 内核需要将优先级转换为权重值, 并提供了一张优先级->权重转换表sched_prio_to_weight, 内核不仅维护了负荷权重自身, 还保存另外一个数值, 用于负荷重除的结果, 即sched_prio_to_wmult数组, 这两个数组中的数据是一一对应的. - - -其中相关的数据结构定义在[kernel/sched/sched.h?v=4.6, L1132](http://lxr.free-electrons.com/source/kernel/sched/sched.h?v=4.6#L1132) - -```c -// http://lxr.free-electrons.com/source/kernel/sched/sched.h?v=4.6#L1132 -/* - * To aid in avoiding the subversion of "niceness" due to uneven distribution - * of tasks with abnormal "nice" values across CPUs the contribution that - * each task makes to its run queue's load is weighted according to its - * scheduling class and "nice" value. For SCHED_NORMAL tasks this is just a - * scaled version of the new time slice allocation that they receive on time - * slice expiry etc. - */ - - -#define WEIGHT_IDLEPRIO 3 /* SCHED_IDLE进程的负荷权重 */ -#define WMULT_IDLEPRIO 1431655765 /* SCHED_IDLE进程负荷权重的重除值 */ - - -extern const int sched_prio_to_weight[40]; -extern const u32 sched_prio_to_wmult[40]; - - -// http://lxr.free-electrons.com/source/kernel/sched/core.c?v=4.6#L8484 -/* -* Nice levels are multiplicative, with a gentle 10% change for every -* nice level changed. I.e. when a CPU-bound task goes from nice 0 to -* nice 1, it will get ~10% less CPU time than another CPU-bound task -* that remained on nice 0. -* -* The "10% effect" is relative and cumulative: from _any_ nice level, -* if you go up 1 level, it's -10% CPU usage, if you go down 1 level -* it's +10% CPU usage. (to achieve that we use a multiplier of 1.25. -* If a task goes up by ~10% and another task goes down by ~10% then -* the relative distance between them is ~25%.) -*/ -const int sched_prio_to_weight[40] = { -/* -20 */ 88761, 71755, 56483, 46273, 36291, -/* -15 */ 29154, 23254, 18705, 14949, 11916, -/* -10 */ 9548, 7620, 6100, 4904, 3906, -/* -5 */ 3121, 2501, 1991, 1586, 1277, -/* 0 */ 1024, 820, 655, 526, 423, -/* 5 */ 335, 272, 215, 172, 137, -/* 10 */ 110, 87, 70, 56, 45, -/* 15 */ 36, 29, 23, 18, 15, -}; - - -/* -* Inverse (2^32/x) values of the sched_prio_to_weight[] array, precalculated. -* -* In cases where the weight does not change often, we can use the -* precalculated inverse to speed up arithmetics by turning divisions -* into multiplications: -*/ -const u32 sched_prio_to_wmult[40] = { -/* -20 */ 48388, 59856, 76040, 92818, 118348, -/* -15 */ 147320, 184698, 229616, 287308, 360437, -/* -10 */ 449829, 563644, 704093, 875809, 1099582, -/* -5 */ 1376151, 1717300, 2157191, 2708050, 3363326, -/* 0 */ 4194304, 5237765, 6557202, 8165337, 10153587, -/* 5 */ 12820798, 15790321, 19976592, 24970740, 31350126, -/* 10 */ 39045157, 49367440, 61356676, 76695844, 95443717, -/* 15 */ 119304647, 148102320, 186737708, 238609294, 286331153, -}; -``` - -对内核使用的范围[-20, 19]中的每个nice级别, sched_prio_to_weight数组都有一个对应项 - -nice [-20, 19] -=> 下标 [0, 39] - -而由于权重`weight` 用`unsigned long` 表示, 因此内核无法直接存储1/weight, 而必须借助于乘法和位移来执行除法的技术. sched_prio_to_wmult数组就存储了这些值, 即sched_prio_to_wmult每个元素的值是2^32/prio_to_weight$每个元素的值. - -可以验证 - -$$sched\_prio\_to\_wmult[i] = \frac{2^{32}}{sched\_prio\_to\_weight[i]}$$ - -同时我们可以看到其定义了两个宏WEIGHT_IDLEPRIO和WMULT_IDLEPRIO这两个宏对应的就是SCHED_IDLE调度的进程的负荷权重信息, 因为要保证SCHED_IDLE进程的最低优先级和最低的负荷权重. 这点信息我们可以在后面分析set_load_weight函数的时候可以看到 - -可以验证 - -$$\frac{2^{32}}{WEIGHT_IDLEPRIO} = WMULT_IDLEPRIO$$ - - -##linux-4.4之前的shced_prio_to_weight和sched_prio_to_wmult -------- - -关于优先级->权重转换表sched_prio_to_weight - -在linux-4.4之前的内核中, 优先级权重转换表用prio_to_weight表示, 定义在[kernel/sched/sched.h, line 1116](http://lxr.free-electrons.com/source/kernel/sched/sched.h?v=4.4#L1116), 与它一同定义的还有prio_to_wmult, 在[kernel/sched/sched.h, line 1139](http://lxr.free-electrons.com/source/kernel/sched/sched.h?v=4.4#L1139) -均被定义为static const - -但是其实这种能够方式不太符合规范的编码风格, 因此常规来说, 我们的头文件中不应该存储结构的定义, 即为了是程序的模块结构更加清晰, 头文件中尽量只包含宏或者声明, 而将具体的定义， 需要分配存储空间的代码放在源文件中. - -否则如果在头文件中定义全局变量，并且将此全局变量赋初值，那么在多个引用此头文件的C文件中同样存在相同变量名的拷贝，关键是此变量被赋了初值，所以编译器就会将此变量放入DATA段，最终在连接阶段，会在DATA段中存在多个相同的变量，它无法将这些变量统一成一个变量，也就是仅为此变量分配一个空间，而不是多份空间，假定这个变量在头文件没有赋初值，编译器就会将之放入BSS段，连接器会对BSS段的多个同名变量仅分配一个存储空间 - -因此在新的内核中, 内核黑客们将这两个变量存放在了[kernel/sched/core.c](http://lxr.free-electrons.com/source/kernel/sched/core.c?v=4.6#L8472), 并加上了sched_前缀, 以表明这些变量是在进程调度的过程中使用的, 而在[kernel/sched/sched.h, line 1144](http://lxr.free-electrons.com/source/kernel/sched/sched.h?v=4.6#L1144)中则只包含了他们的声明. - - -下面我们列出优先级权重转换表定义更新后对比项 - - -| 内核版本 | 实现 | 地址 | -| ------------- |:-------------:|:-------------:| -| <= linux-4.4 | static const int prio_to_weight[40] | [kernel/sched/sched.h, line 1116](http://lxr.free-electrons.com/source/kernel/sched/sched.h?v=4.4#L1116) | -| >=linux-4.5 | const int sched_prio_to_weight[40] | 声明在[kernel/sched/sched.h, line 1144](http://lxr.free-electrons.com/source/kernel/sched/sched.h?v=4.6#L1144), 定义在[kernel/sched/core.c](http://lxr.free-electrons.com/source/kernel/sched/core.c?v=4.6#L8472) - -其定义并没有发生变化, 依然是一个一对一NICE to WEIGHT的转换表 - - -##1.25的乘积因子 -------- - - -各数组之间的乘积因子是1.25. 要知道为何使用该因子, 可考虑下面的例子 - -两个进程A和B在nice级别0, 即静态优先级120运行, 因此两个进程的CPU份额相同, 都是50%, nice级别为0的进程, 查其权重表可知是1024. 每个进程的份额是1024/(1024+1024)=0.5, 即50% - -如果进程B的优先级+1(优先级降低), 成为nice=1, 那么其CPU份额应该减少10%, 换句话说进程A得到的总的CPU应该是55%, 而进程B应该是45%. 优先级增加1导致权重减少, 即1024/1.25=820, 而进程A仍旧是1024, 则进程A现在将得到的CPU份额是1024/(1024+820=0.55, 而进程B的CPU份额则是820/(1024+820)=0.45. 这样就正好产生了10%的差值. - - -#进程负荷权重的计算 -------- - - -set_load_weight负责根据非实时进程类型极其静态优先级计算符合权重 -而实时进程不需要CFS调度, 因此无需计算其负荷权重值 - - ->早期的代码中实时进程也是计算其负荷权重的, 但是只是采用一些方法保持其权重值较大 -> ->在早期有些版本中, set_load_weight中实时进程的权重是普通进程的两倍, 后来又设置成0, 直到后来linux-2.6.37开始不再设置实时进程的优先级, 因此这本身就是一个无用的工作 -> ->而另一方面, SCHED_IDLE进程的权值总是非常小, 普通非实时进程则根据其静态优先级设置对应的负荷权重 - - -##set_load_weight依据静态优先级设置进程的负荷权重 -------- - - -```c -static void set_load_weight(struct task_struct *p) -{ - /* 由于数组中的下标是0~39, 普通进程的优先级是[100~139] - 因此通过static_prio - MAX_RT_PRIO将静态优先级转换成为数组下标 - */ - int prio = p->static_prio - MAX_RT_PRIO; - /* 取得指向进程task负荷权重的指针load, - 下面修改load就是修改进程的负荷权重 */ - struct load_weight *load = &p->se.load; - - /* - * SCHED_IDLE tasks get minimal weight: - * 必须保证SCHED_IDLE进程的负荷权重最小 - * 其权重weight就是WEIGHT_IDLEPRIO - * 而权重的重除结果就是WMULT_IDLEPRIO - */ - if (p->policy == SCHED_IDLE) { - load->weight = scale_load(WEIGHT_IDLEPRIO); - load->inv_weight = WMULT_IDLEPRIO; - return; - } - - /* 设置进程的负荷权重weight和权重的重除值inv_weight */ - load->weight = scale_load(prio_to_weight[prio]); - load->inv_weight = prio_to_wmult[prio]; -} -``` - -##scale_load取得负荷权重的值 -------- - -其中scale_load是一个宏, 定义在[include/linux/sched.h, line 785](http://lxr.free-electrons.com/source/include/linux/sched.h?v=3.9?v=4.6#L785) - - -```c -#if 0 /* BITS_PER_LONG > 32 -- currently broken: it increases power usage under light load */ -# define SCHED_LOAD_RESOLUTION 10 -# define scale_load(w) ((w) << SCHED_LOAD_RESOLUTION) -# define scale_load_down(w) ((w) >> SCHED_LOAD_RESOLUTION) -#else -# define SCHED_LOAD_RESOLUTION 0 -# define scale_load(w) (w) -# define scale_load_down(w) (w) -#endif -``` - -我们可以看到目前版本的scale_load其实什么也没做就是简单取了个值, 但是我们注意到负荷权重仍然保留了SCHED_LOAD_RESOLUTION不为0的情形, 只不过目前因为效率原因和功耗问题没有启用而已 - - -##set_load_weight的演变 -------- - -linux内核的调度器经过了不同阶段的发展, 但是即使是同一个调度器其算法也不是一成不变的, 也在不停的改进和优化 - -| 内核版本 | 实现 | 地址 | -| ------------- |:-------------:|:-------------:| -| 2.6.18~2.6.22 | 实时进程的权重用RTPRIO_TO_LOAD_WEIGHT(p->rt_priority);转换 | [kernel/sched.c#L746](http://lxr.linux.no/linux+v2.6.18/kernel/sched.c#L746) | -| 2.6.23~2.6.34 | 实时进程的权重为非实时权重的二倍 | [kernel/sched.c#L1836](http://lxr.linux.no/linux+v2.6.32/kernel/sched.c#1836) | -| 2.6.35~2.6.36| 实时进程的权重设置为0, 重除值设置为WMULT_CONST | [kernel/sched.c, L1859](http://lxr.linux.no/linux+v2.6.36/kernel/sched.c#L1859) | -| 2.6.37~至今4.6 | 实时进程不再设置权重 | 其中<= linux-3.2时, 代码在sched.c中
3.3~4.4之后, 增加了[sched/core.c](http://lxr.linux.no/linux+v4.4/kernel/sched/core.c#L812)文件调度的核心代码在此存放
4.5~至今, [修改prio_to_weight为sched_prio_to_weight](http://lxr.linux.no/linux+v4.5/kernel/sched/core.c#L813), 并将声明存放头文件中 | -#就绪队列的负荷权重 -------- - -不仅进程, 就绪队列也关联到一个负荷权重. 这个我们在前面讲[Linux进程调度器的设计--Linux进程的管理与调度(十七）](http://blog.csdn.net/gatieme/article/details/51702662)的时候提到过了在cpu的就绪队列rq和cfs调度器的就绪队列cfs_rq中都保存了其load_weight. - -这样不仅确保就绪队列能够跟踪记录有多少进程在运行, 而且还能将进程的权重添加到就绪队列中. - - - -##cfs就绪队列的负荷权重 -------- - -```c -// http://lxr.free-electrons.com/source/kernel/sched/sched.h?v=4.6#L596 -struct rq -{ - /* ...... */ - /* capture load from *all* tasks on this cpu: */ - struct load_weight load; - /* ...... */ -}; - -// http://lxr.free-electrons.com/source/kernel/sched/sched.h?v=4.6#L361 -/* CFS-related fields in a runqueue */ -struct cfs_rq -{ - struct load_weight load; - unsigned int nr_running, h_nr_running; - /* ...... */ -}; - -// http://lxr.free-electrons.com/source/kernel/sched/sched.h?v=4.6#L596 -struct rt_rq中不需要负荷权重 - -// http://lxr.free-electrons.com/source/kernel/sched/sched.h?v=4.6#L490 -struct dl_rq中不需要负荷权重 -``` - -由于负荷权重仅用于调度普通进程(非实时进程), 因此只在cpu的就绪队列队列rq和cfs调度器的就绪队列cfs_rq上需要保存其就绪队列的信息, 而实时进程的就绪队列rt_rq和dl_rq -是不需要保存负荷权重的. - - -##就绪队列的负荷权重计算 -------- - - -就绪队列的负荷权重存储的其实就是队列上所有进程的负荷权重的总和, 因此每次进程被加到就绪队列的时候, 就需要在就绪队列的负荷权重中加上进程的负荷权重, 同时由于就绪队列的不是一个单独被调度的实体, 也就不需要优先级到负荷权重的转换, 因而其不需要负荷权重的重除字段, 即inv_weight = 0; - -因此进程从就绪队列上入队或者出队的时候, 就绪队列的负荷权重就加上或者减去进程的负荷权重, 但是 - - -```c -//struct load_weight { - /* 就绪队列的负荷权重 +/- 入队/出队进程的负荷权重 */ - unsigned long weight +/- task_struct->se->load->weight; - /* 就绪队列负荷权重的重除字段无用途，所以始终置0 */ - u32 inv_weight = 0; -//}; -``` - -因此内核为我们提供了增加/减少/重置就绪队列负荷权重的的函数, 分别是update_load_add, update_load_sub, update_load_set - -```c -/* 使得lw指向的负荷权重的值增加inc, 用于进程进入就绪队列时调用 - * 进程入队 account_entity_enqueue kernel/sched/fair.c#L2422 - */ - -static inline void update_load_add(struct load_weight *lw, unsigned long inc) -{ - lw->weight += inc; - lw->inv_weight = 0; -} - -/* 使得lw指向的负荷权重的值减少inc, 用于进程调出就绪队列时调用 - * 进程出队 account_entity_dequeue kernel/sched/fair.c#L2422*/ -static inline void update_load_sub(struct load_weight *lw, unsigned long dec) -{ - lw->weight -= dec; - lw->inv_weight = 0; -} - -static inline void update_load_set(struct load_weight *lw, unsigned long w) -{ - lw->weight = w; - lw->inv_weight = 0; -} -```` - - -| 函数 | 描述 | 调用时机 | 定义位置 | 调用位置 | -| ------------- |:-------------:|:-------------:| -| update_load_add | 使得lw指向的负荷权重的值增加inc | 用于进程进入就绪队列时调用 | [kernel/sched/fair.c, L117](http://lxr.free-electrons.com/source/kernel/sched/fair.c?v=4.6#L117) | [account_entity_enqueue两处](http://lxr.free-electrons.com/source/kernel/sched/fair.c?v=4.6#L2420), [sched_slice](http://lxr.free-electrons.com/source/kernel/sched/fair.c?v=4.6#L640) | -| update_load_sub | 使得lw指向的负荷权重的值减少inc | 用于进程调出就绪队列时调用 | [update_load_sub, L123](http://lxr.free-electrons.com/source/kernel/sched/fair.c?v=4.6#L123) | [account_entity_dequeue两处](http://lxr.free-electrons.com/source/kernel/sched/fair.c?v=4.6#L2437) | -| update_load_set | - - ->其中sched_slice函数计算当前进程在调度延迟内期望的运行时间, 它根据cfs就绪队列中进程数确定一个最长时间间隔，然后看在该时间间隔内当前进程按照权重比例执行 - - -#总结 -------- - - -**负荷权重load_weight** -CFS完全公平调度器在调度非实时进程的时候, 进程的重要性不仅是由优先级指定的, 还需要考虑保存在task_struct->se.load的负荷权重. - - -**转换表prio_to_weight和重除表ched_prio_to_wmult** -这个负荷权重用struct load_weight, 其包含了名为weight的负荷权重信息, 为了方便快速的将静态优先级转换成权重值, 内核提供了一个长为40的prio_to_weight数组方便转换, 静态优先级[100~139], 对应nice值[-20, 19], 对应数组中的下标[0, 39] - -由于权重`weight` 用`unsigned long` 表示, 因此内核无法直接存储1/weight, 而必须借助于乘法和位移来执行除法的技术. sched_prio_to_wmult数组就存储了这些值, 即sched_prio_to_wmult每个元素的值是2^32/prio_to_weight$每个元素的值. - - -对于SCHED_IDLE进程其优先级最低, 因此其负荷权重也要尽可能的小, 因此内核用WEIGHT_IDLEPRIO( = 3)和WMULT_IDLEPRIO分别表示了SCHED_IDLE进程的负荷权重和重除值. - - -**调度实体负荷权重的计算** - ->既然CFS把负荷权重作为进程调度的一个重要依据, 那么我们就需要了解调度器是如何计算进程或者调度实体的负荷权重的. - - -有了prio_to_weight和ched_prio_to_wmult这两个转换表, 我们就可以很方便的将非实时进程的静态优先级转换成负荷权重, 这个其实就是一个很简单的查表得过程, 内核用set_load_weight完成这个工作, 同时也保证了SCHED_LDLE进程的负荷权重最小 - - -* 将进程的静态优先级[100, 139]转换成数组下标[0, 39] - -* 如果进程是SCHED_IDLE调度, 则负荷权重直赋值为WEIGHT_IDLEPRIO( = 3)和WMULT_IDLEPRIO - -* 对于普通进程, 从prio_to_weight和sched_prio_to_wmult中查找出对应优先级的负荷权重值和重除值 - - -现在的内核中是实时进程是不依靠负荷权重的, 因此也就不需要计算实时进程的负荷权重, 但是早期的内核中实时进程的负荷权重设置为普通进程的两倍, 以保证权重比非实时进程大 - -**调度实体的负荷权重** - -既然load_weight保存着进程的权重信息, 那么作为进程调度的实体, 必须将这个权重值与特定的进程task_struct, 更一般的与通用的调度实体sched_entity相关联 - -sched_entity作为进程调度的实体信息, 其内置了load_weight结构用于保存当前调度实体的权重, [参照](http://lxr.free-electrons.com/source/include/linux/sched.h?v=4.6#L1195) - - -同时进程作为调度实体的最一般单位, 其load_weight就存储在其struct sched_entity *se成员中, 可以通过task_struct->se.load访问进程的负荷权重. - - -**就绪队列的负荷权重** - -然后内核也在全局cpu就绪队列rq中cfs的就绪队列cfs_rq中保存了load_weight, 这就可以很方便的统计出整个就绪队列的负荷权重总和, 为进程调度提供参考, 因此在每次进程入队或者出队的时候就需要通过修改就绪队列的负荷权重, 内核为我们提供了增加/减少/重置就绪队列负荷权重的的函数, 分别是update_load_add, update_load_sub, update_load_set, 而由于就绪队列的负荷权重只关心权重值, 因此其重除字段inv_weight恒为0 - -同时需要**注意**的是, 由于实时进程不依赖于负荷权重的, 因此实时进程的就绪队列rt_qt和dl_rq中不需要存储load_weight. - - - +Linux CFS调度器之负荷权重load_weight +======= + + +| 日期 | 内核版本 | 架构| 作者 | GitHub| CSDN | +| ------- |:-------:|:-------:|:-------:|:-------:|:-------:| +| 2016-06-14 | [Linux-4.6](http://lxr.free-electrons.com/source/?v=4.6) | X86 & arm | [gatieme](http://blog.csdn.net/gatieme) | [LinuxDeviceDrivers](https://github.com/gatieme/LDD-LinuxDeviceDrivers) | [Linux进程管理与调度](http://blog.csdn.net/gatieme/article/category/6225543) | + + + +前面我们详细的了解了linux下进程优先级的表示以及其计算的方法, 我们了解到linux针对普通进程和实时进程分别使用静态优先级static_prio和实时优先级rt_priority来指定其默认的优先级别, 然后通过normal_prio函数将他们分别转换为普通优先级normal_prio, 最终换算出动态优先级prio, 动态优先级prio才是内核调度时候有限考虑的优先级字段 + +但是CFS完全公平调度器在调度进程的时候, 进程的重要性不仅是由优先级指定的, 而且还需要考虑保存在task_struct->se.load的负荷权重. + + + +#前景回顾 +------- + + +##进程调度 +------- + +内存中保存了对每个进程的唯一描述, 并通过若干结构与其他进程连接起来. + +**调度器**面对的情形就是这样, 其任务是在程序之间共享CPU时间, 创造并行执行的错觉, 该任务分为两个不同的部分, 其中一个涉及**调度策略**, 另外一个涉及**上下文切换**. + + +内核必须提供一种方法, 在各个进程之间尽可能公平地共享CPU时间, 而同时又要考虑不同的任务优先级. + +调度器的一般原理是, 按所需分配的计算能力, 向系统中每个进程提供最大的公正性, 或者从另外一个角度上说, 他试图确保没有进程被亏待. + + +##进程的分类 +------- + +linux把进程区分为实时进程和非实时进程, 其中非实时进程进一步划分为交互式进程和批处理进程 + +| 类型 | 描述 | 示例 | +| ------- |:-------:|:-------:|:-------:| +| 交互式进程(interactive process) | 此类进程经常与用户进行交互, 因此需要花费很多时间等待键盘和鼠标操作. 当接受了用户的输入后, 进程必须很快被唤醒, 否则用户会感觉系统反应迟钝 | shell, 文本编辑程序和图形应用程序 | +| 批处理进程(batch process) | 此类进程不必与用户交互, 因此经常在后台运行. 因为这样的进程不必很快相应, 因此常受到调度程序的怠慢 | 程序语言的编译程序, 数据库搜索引擎以及科学计算 | +| 实时进程(real-time process) | 这些进程由很强的调度需要, 这样的进程绝不会被低优先级的进程阻塞. 并且他们的响应时间要尽可能的短 | 视频音频应用程序, 机器人控制程序以及从物理传感器上收集数据的程序| + + +##不同进程采用不同的调度策略 +------- + +根据进程的不同分类Linux采用不同的调度策略. + +对于实时进程，采用FIFO, Round Robin或者Earliest Deadline First (EDF)最早截止期限优先调度算法|的调度策略. + +但是普通进程的调度策略就比较麻烦了, 因为普通进程不能简单的只看优先级, 必须公平的占有CPU, 否则很容易出现进程饥饿, 这种情况下用户会感觉操作系统很卡, 响应总是很慢，因此在linux调度器的发展历程中经过了多次重大变动, linux总是希望寻找一个最接近于完美的调度策略来公平快速的调度进程. + + +##linux调度器的演变 +------- + + +一开始的调度器是复杂度为**$O(n)$的始调度算法**(实际上每次会遍历所有任务，所以复杂度为O(n)), 这个算法的缺点是当内核中有很多任务时，调度器本身就会耗费不少时间，所以，从linux2.5开始引入赫赫有名的**$O(1)$调度器** + +然而，linux是集全球很多程序员的聪明才智而发展起来的超级内核，没有最好，只有更好，在$O(1)$调度器风光了没几天就又被另一个更优秀的调度器取代了，它就是**CFS调度器Completely Fair Scheduler**. 这个也是在2.6内核中引入的，具体为2.6.23，即从此版本开始，内核使用CFS作为它的默认调度器，$O(1)$调度器被抛弃了, 其实CFS的发展也是经历了很多阶段，最早期的楼梯算法(SD), 后来逐步对SD算法进行改进出RSDL(Rotating Staircase Deadline Scheduler), 这个算法已经是"完全公平"的雏形了， 直至CFS是最终被内核采纳的调度器, 它从RSDL/SD中吸取了完全公平的思想，不再跟踪进程的睡眠时间，也不再企图区分交互式进程。它将所有的进程都统一对待，这就是公平的含义。CFS的算法和实现都相当简单，众多的测试表明其性能也非常优越 + + + +| 字段 | 版本 | +| ------------- |:-------------:|:-------------:| +| O(n)的始调度算法 | linux-0.11~2.4 | +| O(1)调度器 | linux-2.5 | +| CFS调度器 | linux-2.6~至今 | + + +##Linux的调度器组成 +------- + + +**2个调度器** + +可以用两种方法来激活调度 + +* 一种是直接的, 比如进程打算睡眠或出于其他原因放弃CPU + +* 另一种是通过周期性的机制, 以固定的频率运行, 不时的检测是否有必要 + +因此当前linux的调度程序由两个调度器组成：**主调度器**，**周期性调度器**(两者又统称为**通用调度器(generic scheduler)**或**核心调度器(core scheduler)**) + +并且每个调度器包括两个内容：**调度框架**(其实质就是两个函数框架)及**调度器类** + + + +**6种调度策略** + +linux内核目前实现了6中调度策略(即调度算法), 用于对不同类型的进程进行调度, 或者支持某些特殊的功能 + +* SCHED_NORMAL和SCHED_BATCH调度普通的非实时进程 + +* SCHED_FIFO和SCHED_RR和SCHED_DEADLINE则采用不同的调度策略调度实时进程 + +* SCHED_IDLE则在系统空闲时调用idle进程. + + + +**5个调度器类** + +而依据其调度策略的不同实现了5个调度器类, 一个调度器类可以用一种种或者多种调度策略调度某一类进程, 也可以用于特殊情况或者调度特殊功能的进程. + + +其所属进程的优先级顺序为 +````c +stop_sched_class -> dl_sched_class -> rt_sched_class -> fair_sched_class -> idle_sched_class +``` + +**3个调度实体** + +调度器不限于调度进程, 还可以调度更大的实体, 比如实现组调度. + +这种一般性要求调度器不直接操作进程, 而是处理可调度实体, 因此需要一个通用的数据结构描述这个调度实体,即seched_entity结构, 其实际上就代表了一个调度对象，可以为一个进程，也可以为一个进程组. + +linux中针对当前可调度的实时和非实时进程, 定义了类型为seched_entity的3个调度实体 + +* sched_dl_entity 采用EDF算法调度的实时调度实体 + +* sched_rt_entity 采用Roound-Robin或者FIFO算法调度的实时调度实体 + +* sched_entity 采用CFS算法调度的普通非实时进程的调度实体 + + +**调度器整体框架** + +每个进程都属于某个调度器类(由字段task_struct->sched_class标识), 由调度器类采用进程对应的调度策略调度(由task_struct->policy )进行调度, task_struct也存储了其对应的调度实体标识 + +linux实现了6种调度策略, 依据其调度策略的不同实现了5个调度器类, 一个调度器类可以用一种或者多种调度策略调度某一类进程, 也可以用于特殊情况或者调度特殊功能的进程. + + +| 调度器类 | 调度策略 | 调度策略对应的调度算法 | 调度实体 | 调度实体对应的调度对象 | +| ------- |:-------:|:-------:|:-------:||:-------:| +| stop_sched_class | 无 | 无 | 无 | 特殊情况, 发生在cpu_stop_cpu_callback 进行cpu之间任务迁移migration或者HOTPLUG_CPU的情况下关闭任务 | +| dl_sched_class | SCHED_DEADLINE | Earliest-Deadline-First最早截至时间有限算法 | sched_dl_entity | 采用DEF最早截至时间有限算法调度实时进程 | +| rt_sched_class | SCHED_RR

SCHED_FIFO | Roound-Robin时间片轮转算法

FIFO先进先出算法 | sched_rt_entity | 采用Roound-Robin或者FIFO算法调度的实时调度实体 | +| fair_sched_class | SCHED_NORMAL

SCHED_BATCH | CFS完全公平懂调度算法 |sched_entity | 采用CFS算法普通非实时进程 | +| idle_sched_class | SCHED_IDLE | 无 | 无 |特殊进程, 用于cpu空闲时调度空闲进程idle | + +它们的关系如下图 + +![调度器的组成](../images/level.jpg) + + +##优先级的内核表示 +------- + + +内核使用一些简单的数值范围0~139表示内部优先级, 数值越低, 优先级越高。 + +从0~99的范围专供实时进程使用, nice的值[-20,19]则映射到范围100~139 + + +>实时优先级范围是0到MAX_RT_PRIO-1（即99），而普通进程的静态优先级范围是从MAX_RT_PRIO到MAX_PRIO-1（即100到139）。 + +| 优先级范围 | 描述 | +| ------------- |:-------------:| +| 0——99 | 实时进程 | +| 100——139 | 非实时进程 | + +![内核优先级标度](../images/priority.jpg) + +##进程的优先级表示 +------- + + +```c +struct task_struct +{ + /* 进程优先级 + * prio: 动态优先级，范围为100~139，与静态优先级和补偿(bonus)有关 + * static_prio: 静态优先级，static_prio = 100 + nice + 20 (nice值为-20~19,所以static_prio值为100~139) + * normal_prio: 没有受优先级继承影响的常规优先级，具体见normal_prio函数，跟属于什么类型的进程有关 + */ + int prio, static_prio, normal_prio; + /* 实时进程优先级 */ + unsigned int rt_priority; +} +``` + +**动态优先级 静态优先级 实时优先级** + + +其中task_struct采用了三个成员表示进程的优先级:prio和normal_prio表示动态优先级, static_prio表示进程的静态优先级. + +此外还用了一个字段rt_priority保存了实时进程的优先级 + + +| 字段 | 描述 | +| ------------- |:-------------:| +| static_prio | 用于保存静态优先级, 是进程启动时分配的优先级, ，可以通过nice和sched_setscheduler系统调用来进行修改, 否则在进程运行期间会一直保持恒定 | +| prio | 保存进程的动态优先级 | +| normal_prio | 表示基于进程的静态优先级static_prio和调度策略计算出的优先级. 因此即使普通进程和实时进程具有相同的静态优先级, 其普通优先级也是不同的, 进程分叉(fork)时, 子进程会继承父进程的普通优先级 | +| rt_priority | 用于保存实时优先级 | + + +实时进程的优先级用实时优先级rt_priority来表示 + + + +#负荷权重 +------- + + +##负荷权重结构struct load_weight +------- + +负荷权重用struct load_weight数据结构来表示, 保存着进程权重值weight。其定义在[/include/linux/sched.h, v=4.6, L1195](http://lxr.free-electrons.com/source/include/linux/sched.h?v=4.6#L1195), 如下所示 + +```c +struct load_weight { + unsigned long weight; /* 存储了权重的信息 */ + u32 inv_weight; /* 存储了权重值用于重除的结果 weight * inv_weight = 2^32 */ +}; +``` + + + +##调度实体的负荷权重load +------- + +既然struct load_weight保存着进程的权重信息, 那么作为进程调度的实体, 必须将这个权重值与特定的进程task_struct, 更一般的与通用的调度实体sched_entity相关联 + +struct sched_entity作为进程调度的实体信息, 其内置了load_weight结构用于保存当前调度实体的权重, 参照http://lxr.free-electrons.com/source/include/linux/sched.h?v=4.6#L1195 + +```c +struct sched_entity { + struct load_weight load; /* for load-balancing */ + /* ...... */ +}; +``` + +##进程的负荷权重 + +而进程可以被作为一个调度的实时, 其内部通过存储struct sched_entity se而间接存储了其load_weight信息, 参照http://lxr.free-electrons.com/source/include/linux/sched.h?v=4.6#L1415 + +```c +struct task_struct +{ + /* ...... */ + struct sched_entity se; + /* ...... */ +} +``` + +因此我们就可以通过task_statuct->se.load获取负荷权重的信息, 而set_load_weight负责根据进程类型及其静态优先级计算符合权重. + + +#优先级和权重的转换 +------- + + +##优先级->权重转换表 +------- + + +一般这个概念是这样的, 进程每降低一个nice值(优先级提升), 则多获得10%的CPU时间, 没升高一个nice值(优先级降低), 则放弃10%的CPU时间. + +为执行该策略, 内核需要将优先级转换为权重值, 并提供了一张优先级->权重转换表sched_prio_to_weight, 内核不仅维护了负荷权重自身, 还保存另外一个数值, 用于负荷重除的结果, 即sched_prio_to_wmult数组, 这两个数组中的数据是一一对应的. + + +其中相关的数据结构定义在[kernel/sched/sched.h?v=4.6, L1132](http://lxr.free-electrons.com/source/kernel/sched/sched.h?v=4.6#L1132) + +```c +// http://lxr.free-electrons.com/source/kernel/sched/sched.h?v=4.6#L1132 +/* + * To aid in avoiding the subversion of "niceness" due to uneven distribution + * of tasks with abnormal "nice" values across CPUs the contribution that + * each task makes to its run queue's load is weighted according to its + * scheduling class and "nice" value. For SCHED_NORMAL tasks this is just a + * scaled version of the new time slice allocation that they receive on time + * slice expiry etc. + */ + + +#define WEIGHT_IDLEPRIO 3 /* SCHED_IDLE进程的负荷权重 */ +#define WMULT_IDLEPRIO 1431655765 /* SCHED_IDLE进程负荷权重的重除值 */ + + +extern const int sched_prio_to_weight[40]; +extern const u32 sched_prio_to_wmult[40]; + + +// http://lxr.free-electrons.com/source/kernel/sched/core.c?v=4.6#L8484 +/* +* Nice levels are multiplicative, with a gentle 10% change for every +* nice level changed. I.e. when a CPU-bound task goes from nice 0 to +* nice 1, it will get ~10% less CPU time than another CPU-bound task +* that remained on nice 0. +* +* The "10% effect" is relative and cumulative: from _any_ nice level, +* if you go up 1 level, it's -10% CPU usage, if you go down 1 level +* it's +10% CPU usage. (to achieve that we use a multiplier of 1.25. +* If a task goes up by ~10% and another task goes down by ~10% then +* the relative distance between them is ~25%.) +*/ +const int sched_prio_to_weight[40] = { +/* -20 */ 88761, 71755, 56483, 46273, 36291, +/* -15 */ 29154, 23254, 18705, 14949, 11916, +/* -10 */ 9548, 7620, 6100, 4904, 3906, +/* -5 */ 3121, 2501, 1991, 1586, 1277, +/* 0 */ 1024, 820, 655, 526, 423, +/* 5 */ 335, 272, 215, 172, 137, +/* 10 */ 110, 87, 70, 56, 45, +/* 15 */ 36, 29, 23, 18, 15, +}; + + +/* +* Inverse (2^32/x) values of the sched_prio_to_weight[] array, precalculated. +* +* In cases where the weight does not change often, we can use the +* precalculated inverse to speed up arithmetics by turning divisions +* into multiplications: +*/ +const u32 sched_prio_to_wmult[40] = { +/* -20 */ 48388, 59856, 76040, 92818, 118348, +/* -15 */ 147320, 184698, 229616, 287308, 360437, +/* -10 */ 449829, 563644, 704093, 875809, 1099582, +/* -5 */ 1376151, 1717300, 2157191, 2708050, 3363326, +/* 0 */ 4194304, 5237765, 6557202, 8165337, 10153587, +/* 5 */ 12820798, 15790321, 19976592, 24970740, 31350126, +/* 10 */ 39045157, 49367440, 61356676, 76695844, 95443717, +/* 15 */ 119304647, 148102320, 186737708, 238609294, 286331153, +}; +``` + +对内核使用的范围[-20, 19]中的每个nice级别, sched_prio_to_weight数组都有一个对应项 + +nice [-20, 19] -=> 下标 [0, 39] + +而由于权重`weight` 用`unsigned long` 表示, 因此内核无法直接存储1/weight, 而必须借助于乘法和位移来执行除法的技术. sched_prio_to_wmult数组就存储了这些值, 即sched_prio_to_wmult每个元素的值是2^32/prio_to_weight$每个元素的值. + +可以验证 + +$$sched\_prio\_to\_wmult[i] = \frac{2^{32}}{sched\_prio\_to\_weight[i]}$$ + +同时我们可以看到其定义了两个宏WEIGHT_IDLEPRIO和WMULT_IDLEPRIO这两个宏对应的就是SCHED_IDLE调度的进程的负荷权重信息, 因为要保证SCHED_IDLE进程的最低优先级和最低的负荷权重. 这点信息我们可以在后面分析set_load_weight函数的时候可以看到 + +可以验证 + +$$\frac{2^{32}}{WEIGHT_IDLEPRIO} = WMULT_IDLEPRIO$$ + + +##linux-4.4之前的shced_prio_to_weight和sched_prio_to_wmult +------- + +关于优先级->权重转换表sched_prio_to_weight + +在linux-4.4之前的内核中, 优先级权重转换表用prio_to_weight表示, 定义在[kernel/sched/sched.h, line 1116](http://lxr.free-electrons.com/source/kernel/sched/sched.h?v=4.4#L1116), 与它一同定义的还有prio_to_wmult, 在[kernel/sched/sched.h, line 1139](http://lxr.free-electrons.com/source/kernel/sched/sched.h?v=4.4#L1139) +均被定义为static const + +但是其实这种能够方式不太符合规范的编码风格, 因此常规来说, 我们的头文件中不应该存储结构的定义, 即为了是程序的模块结构更加清晰, 头文件中尽量只包含宏或者声明, 而将具体的定义， 需要分配存储空间的代码放在源文件中. + +否则如果在头文件中定义全局变量，并且将此全局变量赋初值，那么在多个引用此头文件的C文件中同样存在相同变量名的拷贝，关键是此变量被赋了初值，所以编译器就会将此变量放入DATA段，最终在连接阶段，会在DATA段中存在多个相同的变量，它无法将这些变量统一成一个变量，也就是仅为此变量分配一个空间，而不是多份空间，假定这个变量在头文件没有赋初值，编译器就会将之放入BSS段，连接器会对BSS段的多个同名变量仅分配一个存储空间 + +因此在新的内核中, 内核黑客们将这两个变量存放在了[kernel/sched/core.c](http://lxr.free-electrons.com/source/kernel/sched/core.c?v=4.6#L8472), 并加上了sched_前缀, 以表明这些变量是在进程调度的过程中使用的, 而在[kernel/sched/sched.h, line 1144](http://lxr.free-electrons.com/source/kernel/sched/sched.h?v=4.6#L1144)中则只包含了他们的声明. + + +下面我们列出优先级权重转换表定义更新后对比项 + + +| 内核版本 | 实现 | 地址 | +| ------------- |:-------------:|:-------------:| +| <= linux-4.4 | static const int prio_to_weight[40] | [kernel/sched/sched.h, line 1116](http://lxr.free-electrons.com/source/kernel/sched/sched.h?v=4.4#L1116) | +| >=linux-4.5 | const int sched_prio_to_weight[40] | 声明在[kernel/sched/sched.h, line 1144](http://lxr.free-electrons.com/source/kernel/sched/sched.h?v=4.6#L1144), 定义在[kernel/sched/core.c](http://lxr.free-electrons.com/source/kernel/sched/core.c?v=4.6#L8472) + +其定义并没有发生变化, 依然是一个一对一NICE to WEIGHT的转换表 + + +##1.25的乘积因子 +------- + + +各数组之间的乘积因子是1.25. 要知道为何使用该因子, 可考虑下面的例子 + +两个进程A和B在nice级别0, 即静态优先级120运行, 因此两个进程的CPU份额相同, 都是50%, nice级别为0的进程, 查其权重表可知是1024. 每个进程的份额是1024/(1024+1024)=0.5, 即50% + +如果进程B的优先级+1(优先级降低), 成为nice=1, 那么其CPU份额应该减少10%, 换句话说进程A得到的总的CPU应该是55%, 而进程B应该是45%. 优先级增加1导致权重减少, 即1024/1.25=820, 而进程A仍旧是1024, 则进程A现在将得到的CPU份额是1024/(1024+820=0.55, 而进程B的CPU份额则是820/(1024+820)=0.45. 这样就正好产生了10%的差值. + + +#进程负荷权重的计算 +------- + + +set_load_weight负责根据非实时进程类型极其静态优先级计算符合权重 +而实时进程不需要CFS调度, 因此无需计算其负荷权重值 + + +>早期的代码中实时进程也是计算其负荷权重的, 但是只是采用一些方法保持其权重值较大 +> +>在早期有些版本中, set_load_weight中实时进程的权重是普通进程的两倍, 后来又设置成0, 直到后来linux-2.6.37开始不再设置实时进程的优先级, 因此这本身就是一个无用的工作 +> +>而另一方面, SCHED_IDLE进程的权值总是非常小, 普通非实时进程则根据其静态优先级设置对应的负荷权重 + + +##set_load_weight依据静态优先级设置进程的负荷权重 +------- + + +```c +static void set_load_weight(struct task_struct *p) +{ + /* 由于数组中的下标是0~39, 普通进程的优先级是[100~139] + 因此通过static_prio - MAX_RT_PRIO将静态优先级转换成为数组下标 + */ + int prio = p->static_prio - MAX_RT_PRIO; + /* 取得指向进程task负荷权重的指针load, + 下面修改load就是修改进程的负荷权重 */ + struct load_weight *load = &p->se.load; + + /* + * SCHED_IDLE tasks get minimal weight: + * 必须保证SCHED_IDLE进程的负荷权重最小 + * 其权重weight就是WEIGHT_IDLEPRIO + * 而权重的重除结果就是WMULT_IDLEPRIO + */ + if (p->policy == SCHED_IDLE) { + load->weight = scale_load(WEIGHT_IDLEPRIO); + load->inv_weight = WMULT_IDLEPRIO; + return; + } + + /* 设置进程的负荷权重weight和权重的重除值inv_weight */ + load->weight = scale_load(prio_to_weight[prio]); + load->inv_weight = prio_to_wmult[prio]; +} +``` + +##scale_load取得负荷权重的值 +------- + +其中scale_load是一个宏, 定义在[include/linux/sched.h, line 785](http://lxr.free-electrons.com/source/include/linux/sched.h?v=3.9?v=4.6#L785) + + +```c +#if 0 /* BITS_PER_LONG > 32 -- currently broken: it increases power usage under light load */ +# define SCHED_LOAD_RESOLUTION 10 +# define scale_load(w) ((w) << SCHED_LOAD_RESOLUTION) +# define scale_load_down(w) ((w) >> SCHED_LOAD_RESOLUTION) +#else +# define SCHED_LOAD_RESOLUTION 0 +# define scale_load(w) (w) +# define scale_load_down(w) (w) +#endif +``` + +我们可以看到目前版本的scale_load其实什么也没做就是简单取了个值, 但是我们注意到负荷权重仍然保留了SCHED_LOAD_RESOLUTION不为0的情形, 只不过目前因为效率原因和功耗问题没有启用而已 + + +##set_load_weight的演变 +------- + +linux内核的调度器经过了不同阶段的发展, 但是即使是同一个调度器其算法也不是一成不变的, 也在不停的改进和优化 + +| 内核版本 | 实现 | 地址 | +| ------------- |:-------------:|:-------------:| +| 2.6.18~2.6.22 | 实时进程的权重用RTPRIO_TO_LOAD_WEIGHT(p->rt_priority);转换 | [kernel/sched.c#L746](http://lxr.linux.no/linux+v2.6.18/kernel/sched.c#L746) | +| 2.6.23~2.6.34 | 实时进程的权重为非实时权重的二倍 | [kernel/sched.c#L1836](http://lxr.linux.no/linux+v2.6.32/kernel/sched.c#1836) | +| 2.6.35~2.6.36| 实时进程的权重设置为0, 重除值设置为WMULT_CONST | [kernel/sched.c, L1859](http://lxr.linux.no/linux+v2.6.36/kernel/sched.c#L1859) | +| 2.6.37~至今4.6 | 实时进程不再设置权重 | 其中<= linux-3.2时, 代码在sched.c中
3.3~4.4之后, 增加了[sched/core.c](http://lxr.linux.no/linux+v4.4/kernel/sched/core.c#L812)文件调度的核心代码在此存放
4.5~至今, [修改prio_to_weight为sched_prio_to_weight](http://lxr.linux.no/linux+v4.5/kernel/sched/core.c#L813), 并将声明存放头文件中 | +#就绪队列的负荷权重 +------- + +不仅进程, 就绪队列也关联到一个负荷权重. 这个我们在前面讲[Linux进程调度器的设计--Linux进程的管理与调度(十七）](http://blog.csdn.net/gatieme/article/details/51702662)的时候提到过了在cpu的就绪队列rq和cfs调度器的就绪队列cfs_rq中都保存了其load_weight. + +这样不仅确保就绪队列能够跟踪记录有多少进程在运行, 而且还能将进程的权重添加到就绪队列中. + + + +##cfs就绪队列的负荷权重 +------- + +```c +// http://lxr.free-electrons.com/source/kernel/sched/sched.h?v=4.6#L596 +struct rq +{ + /* ...... */ + /* capture load from *all* tasks on this cpu: */ + struct load_weight load; + /* ...... */ +}; + +// http://lxr.free-electrons.com/source/kernel/sched/sched.h?v=4.6#L361 +/* CFS-related fields in a runqueue */ +struct cfs_rq +{ + struct load_weight load; + unsigned int nr_running, h_nr_running; + /* ...... */ +}; + +// http://lxr.free-electrons.com/source/kernel/sched/sched.h?v=4.6#L596 +struct rt_rq中不需要负荷权重 + +// http://lxr.free-electrons.com/source/kernel/sched/sched.h?v=4.6#L490 +struct dl_rq中不需要负荷权重 +``` + +由于负荷权重仅用于调度普通进程(非实时进程), 因此只在cpu的就绪队列队列rq和cfs调度器的就绪队列cfs_rq上需要保存其就绪队列的信息, 而实时进程的就绪队列rt_rq和dl_rq +是不需要保存负荷权重的. + + +##就绪队列的负荷权重计算 +------- + + +就绪队列的负荷权重存储的其实就是队列上所有进程的负荷权重的总和, 因此每次进程被加到就绪队列的时候, 就需要在就绪队列的负荷权重中加上进程的负荷权重, 同时由于就绪队列的不是一个单独被调度的实体, 也就不需要优先级到负荷权重的转换, 因而其不需要负荷权重的重除字段, 即inv_weight = 0; + +因此进程从就绪队列上入队或者出队的时候, 就绪队列的负荷权重就加上或者减去进程的负荷权重, 但是 + + +```c +//struct load_weight { + /* 就绪队列的负荷权重 +/- 入队/出队进程的负荷权重 */ + unsigned long weight +/- task_struct->se->load->weight; + /* 就绪队列负荷权重的重除字段无用途，所以始终置0 */ + u32 inv_weight = 0; +//}; +``` + +因此内核为我们提供了增加/减少/重置就绪队列负荷权重的的函数, 分别是update_load_add, update_load_sub, update_load_set + +```c +/* 使得lw指向的负荷权重的值增加inc, 用于进程进入就绪队列时调用 + * 进程入队 account_entity_enqueue kernel/sched/fair.c#L2422 + */ + +static inline void update_load_add(struct load_weight *lw, unsigned long inc) +{ + lw->weight += inc; + lw->inv_weight = 0; +} + +/* 使得lw指向的负荷权重的值减少inc, 用于进程调出就绪队列时调用 + * 进程出队 account_entity_dequeue kernel/sched/fair.c#L2422*/ +static inline void update_load_sub(struct load_weight *lw, unsigned long dec) +{ + lw->weight -= dec; + lw->inv_weight = 0; +} + +static inline void update_load_set(struct load_weight *lw, unsigned long w) +{ + lw->weight = w; + lw->inv_weight = 0; +} +```` + + +| 函数 | 描述 | 调用时机 | 定义位置 | 调用位置 | +| ------------- |:-------------:|:-------------:| +| update_load_add | 使得lw指向的负荷权重的值增加inc | 用于进程进入就绪队列时调用 | [kernel/sched/fair.c, L117](http://lxr.free-electrons.com/source/kernel/sched/fair.c?v=4.6#L117) | [account_entity_enqueue两处](http://lxr.free-electrons.com/source/kernel/sched/fair.c?v=4.6#L2420), [sched_slice](http://lxr.free-electrons.com/source/kernel/sched/fair.c?v=4.6#L640) | +| update_load_sub | 使得lw指向的负荷权重的值减少inc | 用于进程调出就绪队列时调用 | [update_load_sub, L123](http://lxr.free-electrons.com/source/kernel/sched/fair.c?v=4.6#L123) | [account_entity_dequeue两处](http://lxr.free-electrons.com/source/kernel/sched/fair.c?v=4.6#L2437) | +| update_load_set | + + +>其中sched_slice函数计算当前进程在调度延迟内期望的运行时间, 它根据cfs就绪队列中进程数确定一个最长时间间隔，然后看在该时间间隔内当前进程按照权重比例执行 + + +#总结 +------- + + +**负荷权重load_weight** +CFS完全公平调度器在调度非实时进程的时候, 进程的重要性不仅是由优先级指定的, 还需要考虑保存在task_struct->se.load的负荷权重. + + +**转换表prio_to_weight和重除表ched_prio_to_wmult** +这个负荷权重用struct load_weight, 其包含了名为weight的负荷权重信息, 为了方便快速的将静态优先级转换成权重值, 内核提供了一个长为40的prio_to_weight数组方便转换, 静态优先级[100~139], 对应nice值[-20, 19], 对应数组中的下标[0, 39] + +由于权重`weight` 用`unsigned long` 表示, 因此内核无法直接存储1/weight, 而必须借助于乘法和位移来执行除法的技术. sched_prio_to_wmult数组就存储了这些值, 即sched_prio_to_wmult每个元素的值是2^32/prio_to_weight$每个元素的值. + + +对于SCHED_IDLE进程其优先级最低, 因此其负荷权重也要尽可能的小, 因此内核用WEIGHT_IDLEPRIO( = 3)和WMULT_IDLEPRIO分别表示了SCHED_IDLE进程的负荷权重和重除值. + + +**调度实体负荷权重的计算** + +>既然CFS把负荷权重作为进程调度的一个重要依据, 那么我们就需要了解调度器是如何计算进程或者调度实体的负荷权重的. + + +有了prio_to_weight和ched_prio_to_wmult这两个转换表, 我们就可以很方便的将非实时进程的静态优先级转换成负荷权重, 这个其实就是一个很简单的查表得过程, 内核用set_load_weight完成这个工作, 同时也保证了SCHED_LDLE进程的负荷权重最小 + + +* 将进程的静态优先级[100, 139]转换成数组下标[0, 39] + +* 如果进程是SCHED_IDLE调度, 则负荷权重直赋值为WEIGHT_IDLEPRIO( = 3)和WMULT_IDLEPRIO + +* 对于普通进程, 从prio_to_weight和sched_prio_to_wmult中查找出对应优先级的负荷权重值和重除值 + + +现在的内核中是实时进程是不依靠负荷权重的, 因此也就不需要计算实时进程的负荷权重, 但是早期的内核中实时进程的负荷权重设置为普通进程的两倍, 以保证权重比非实时进程大 + +**调度实体的负荷权重** + +既然load_weight保存着进程的权重信息, 那么作为进程调度的实体, 必须将这个权重值与特定的进程task_struct, 更一般的与通用的调度实体sched_entity相关联 + +sched_entity作为进程调度的实体信息, 其内置了load_weight结构用于保存当前调度实体的权重, [参照](http://lxr.free-electrons.com/source/include/linux/sched.h?v=4.6#L1195) + + +同时进程作为调度实体的最一般单位, 其load_weight就存储在其struct sched_entity *se成员中, 可以通过task_struct->se.load访问进程的负荷权重. + + +**就绪队列的负荷权重** + +然后内核也在全局cpu就绪队列rq中cfs的就绪队列cfs_rq中保存了load_weight, 这就可以很方便的统计出整个就绪队列的负荷权重总和, 为进程调度提供参考, 因此在每次进程入队或者出队的时候就需要通过修改就绪队列的负荷权重, 内核为我们提供了增加/减少/重置就绪队列负荷权重的的函数, 分别是update_load_add, update_load_sub, update_load_set, 而由于就绪队列的负荷权重只关心权重值, 因此其重除字段inv_weight恒为0 + +同时需要**注意**的是, 由于实时进程不依赖于负荷权重的, 因此实时进程的就绪队列rt_qt和dl_rq中不需要存储load_weight. + + + diff --git a/study/kernel/01-process/05-schedule/04-cfs/02-vruntime/README.md b/study/kernel/01-process/05-schedule/04-cfs/03-vruntime/README.md similarity index 97% rename from study/kernel/01-process/05-schedule/04-cfs/02-vruntime/README.md rename to study/kernel/01-process/05-schedule/04-cfs/03-vruntime/README.md index 08daf18..38efd73 100644 --- a/study/kernel/01-process/05-schedule/04-cfs/02-vruntime/README.md +++ b/study/kernel/01-process/05-schedule/04-cfs/03-vruntime/README.md @@ -1,4 +1,4 @@ -Linux进程调度器的设计 +Linux CFS调度器之虚拟时钟与调度延迟 ======= diff --git a/study/kernel/01-process/05-schedule/04-cfs/02-vruntime/update_curr.c b/study/kernel/01-process/05-schedule/04-cfs/03-vruntime/update_curr.c similarity index 100% rename from study/kernel/01-process/05-schedule/04-cfs/02-vruntime/update_curr.c rename to study/kernel/01-process/05-schedule/04-cfs/03-vruntime/update_curr.c diff --git a/study/kernel/01-process/05-schedule/04-cfs/02-vruntime/update_min_vruntime.c b/study/kernel/01-process/05-schedule/04-cfs/03-vruntime/update_min_vruntime.c similarity index 100% rename from study/kernel/01-process/05-schedule/04-cfs/02-vruntime/update_min_vruntime.c rename to study/kernel/01-process/05-schedule/04-cfs/03-vruntime/update_min_vruntime.c diff --git a/study/kernel/01-process/05-schedule/04-cfs/04-queue/README.md b/study/kernel/01-process/05-schedule/04-cfs/04-queue/README.md new file mode 100644 index 0000000..1adc81e --- /dev/null +++ b/study/kernel/01-process/05-schedule/04-cfs/04-queue/README.md @@ -0,0 +1,162 @@ +Linux CFS调度器之队列操作 +======= + + +| 日期 | 内核版本 | 架构| 作者 | GitHub| CSDN | +| ------- |:-------:|:-------:|:-------:|:-------:|:-------:| +| 2016-06-29 | [Linux-4.6](http://lxr.free-electrons.com/source/?v=4.6) | X86 & arm | [gatieme](http://blog.csdn.net/gatieme) | [LinuxDeviceDrivers](https://github.com/gatieme/LDD-LinuxDeviceDrivers) | [Linux进程管理与调度](http://blog.csdn.net/gatieme/article/details/51456569) | + + + +CFS负责处理普通非实时进程, 这类进程是我们linux中最普遍的进程 + + +#前景回顾 +------- + +## CFS调度算法 +------- + +**CFS调度算法的思想** + +理想状态下每个进程都能获得相同的时间片，并且同时运行在CPU上，但实际上一个CPU同一时刻运行的进程只能有一个。也就是说，当一个进程占用CPU时，其他进程就必须等待。CFS为了实现公平，必须惩罚当前正在运行的进程，以使那些正在等待的进程下次被调度. + +## 负荷权重和虚拟时钟 + +**虚拟时钟是红黑树排序的依据** + +具体实现时，CFS通过每个进程的**虚拟运行时间(vruntime)**来衡量哪个进程最值得被调度. CFS中的就绪队列是一棵以vruntime为键值的红黑树，虚拟时间越小的进程越靠近整个红黑树的最左端。因此，调度器每次选择位于红黑树最左端的那个进程，该进程的vruntime最小. + +**优先级计算负荷权重, 负荷权重和当前时间计算出虚拟运行时间** + +虚拟运行时间是通过进程的实际运行时间和进程的权重(weight)计算出来的。在CFS调度器中，将进程优先级这个概念弱化，而是强调进程的权重。一个进程的权重越大，则说明这个进程更需要运行，因此它的虚拟运行时间就越小，这样被调度的机会就越大。而，CFS调度器中的权重在内核是对用户态进程的优先级nice值, 通过prio_to_weight数组进行nice值和权重的转换而计算出来的 + + +**虚拟时钟相关公式** + + linux内核采用了计算公式： + +| 属性 | 公式 | 描述 | +|:-------:|:-------:| +| ideal_time | sum_runtime *se.weight/cfs_rq.weight | 每个进程应该运行的时间 | +| sum_exec_runtime | | 运行队列中所有任务运行完一遍的时间 | +| se.weight | | 当前进程的权重 | +| cfs.weight | | 整个cfs_rq的总权重 | + +这里se.weight和cfs.weight根据上面讲解我们可以算出, sum_runtime是怎们计算的呢，linux内核中这是个经验值，其经验公式是 + +| 条件 | 公式 | +|:-------:|:-------:| +| 进程数 > sched_nr_latency | sum_runtime=sysctl_sched_min_granularity *nr_running | +| 进程数 <=sched_nr_latency | sum_runtime=sysctl_sched_latency = 20ms | + +>注：sysctl_sched_min_granularity =4ms +> +>sched_nr_latency是内核在一个延迟周期中处理的最大活动进程数目 + +linux内核代码中是通过一个叫vruntime的变量来实现上面的原理的，即： + +每一个进程拥有一个vruntime,每次需要调度的时候就选运行队列中拥有最小vruntime的那个进程来运行，vruntime在时钟中断里面被维护，每次时钟中断都要更新当前进程的vruntime,即vruntime以如下公式逐渐增长： + + +| 条件 | 公式 | +|:-------:|:-------:| +| curr.nice!=NICE_0_LOAD | vruntime += delta* NICE_0_LOAD/se.weight; | +| curr.nice=NICE_0_LOAD | vruntime += delta; | + + +##今日内容--CFS进程入队和出队 +------- + + +完全公平调度器CFS中有两个函数可用来增删队列的成员:enqueue_task_fair和dequeue_task_fair分别用来向CFS就绪队列中添加或者删除进程 + + +# enqueue_task_fair入队操作 +------- + +## enque_task_fair函数 +------- + +向就绪队列中放置新进程的工作由函数enqueue_task_fair函数完成, 该函数定义在[kernel/sched/fair.c, line 5442](http://lxr.free-electrons.com/source/kernel/sched/fair.c?v=4.6#L5442), 其函数原型如下 + +该函数将task_struct *p所指向的进程插入到rq所在的就绪队列中, 除了指向所述的就绪队列rq和task_struct的指针外, 该函数还有另外一个参数wakeup. 这使得可以指定入队的进程是否最近才被唤醒并转换为运行状态(此时需指定wakeup = 1), 还是此前就是可运行的(那么wakeup = 0). + +```c +static void +enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) +``` +enqueue_task_fair的执行流程如下 + +* 如果通过struct sched_entity的on_rq成员判断进程已经在就绪队列上, 则无事可做. + +* 否则, 具体的工作委托给enqueue_entity完成, 其中内核会借机用update_curr更新统计量. + +* + +## for_each_sched_entity +------- + +首先内核查找到待天机进程p所在的调度实体信息, 然后通过for_each_sched_entity循环所有调度实体, + +```c +// enqueue_task_fair函数 + struct cfs_rq *cfs_rq; + struct sched_entity *se = &p->se; + + for_each_sched_entity(se) + { + /* ...... */ + } +```` + +但是有个疑问是, 进程p所在的调度时提就一个为嘛要循环才能遍历啊, 这是因为为了支持组调度.组调度下调度实体是有层次结构的, 我们将进程加入的时候, 同时要更新其父调度实体的调度信息, 而非组调度情况下, 就不需要调度实体的层次结构 + +linux对组调度的支持可以通过CONFIG_FAIR_GROUP_SCHED来启用, 在启用和不启用的条件下, 内核对很多函数的实现也会因条件而异, 这点对for_each_sched_entity函数尤为明显, 参见[启用CONFIG_FAIR_GROUP_SCHED](http://lxr.free-electrons.com/source/kernel/sched/fair.c?v=4.6#L246)和[不启用CONFIG_FAIR_GROUP_SCHED](http://lxr.free-electrons.com/source/kernel/sched/fair.c?v=4.6#L367) + +```c +#ifdef CONFIG_FAIR_GROUP_SCHED + +/* An entity is a task if it doesn't "own" a runqueue */ +#define entity_is_task(se) (!se->my_q) + +// http://lxr.free-electrons.com/source/kernel/sched/fair.c?v=4.6#L266 +/* Walk up scheduling entities hierarchy */ +#define for_each_sched_entity(se) \ + for (; se; se = se->parent) + + #else /* !CONFIG_FAIR_GROUP_SCHED */ + +#define entity_is_task(se) 1 + +// http://lxr.free-electrons.com/source/kernel/sched/fair.c?v=4.6#L381 +#define for_each_sched_entity(se) \ + for (; se; se = NULL) +``` + +* 如果通过struct sched_entity的on_rq成员判断进程已经在就绪队列上, 则无事可做. + +* 否则, 具体的工作委托给enqueue_entity完成, 其中内核会借机用update_curr更新统计量. + +```c +// enqueue_task_fair函数 + /* 如果当前进程已经在就绪队列上 */ + if (se->on_rq) + break; + + /* 获取到当前进程所在的cfs_rq就绪队列 */ + cfs_rq = cfs_rq_of(se); + /* 内核委托enqueue_entity完成真正的插入工作 */ + enqueue_entity(cfs_rq, se, flags); +```` + + +##enqueue_entity插入进程 +------- + +首先如果进程最近正在运行, 其虚拟时间时间仍然有效, 那么(除非它当前在执行中)他可以 + +```c + +```` + diff --git a/study/kernel/01-process/05-schedule/04-cfs/04-queue/dequeue_task_fair.c b/study/kernel/01-process/05-schedule/04-cfs/04-queue/dequeue_task_fair.c new file mode 100644 index 0000000..0d15953 --- /dev/null +++ b/study/kernel/01-process/05-schedule/04-cfs/04-queue/dequeue_task_fair.c @@ -0,0 +1,57 @@ +/* + * The dequeue_task method is called before nr_running is + * decreased. We remove the task from the rbtree and + * update the fair scheduling stats: + */ +static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) +{ + struct cfs_rq *cfs_rq; + struct sched_entity *se = &p->se; + int task_sleep = flags & DEQUEUE_SLEEP; + + for_each_sched_entity(se) { + cfs_rq = cfs_rq_of(se); + dequeue_entity(cfs_rq, se, flags); + + /* + * end evaluation on encountering a throttled cfs_rq + * + * note: in the case of encountering a throttled cfs_rq we will + * post the final h_nr_running decrement below. + */ + if (cfs_rq_throttled(cfs_rq)) + break; + cfs_rq->h_nr_running--; + + /* Don't dequeue parent if it has other entities besides us */ + if (cfs_rq->load.weight) { + /* + * Bias pick_next to pick a task from this cfs_rq, as + * p is sleeping when it is within its sched_slice. + */ + if (task_sleep && parent_entity(se)) + set_next_buddy(parent_entity(se)); + + /* avoid re-evaluating load for this entity */ + se = parent_entity(se); + break; + } + flags |= DEQUEUE_SLEEP; + } + + for_each_sched_entity(se) { + cfs_rq = cfs_rq_of(se); + cfs_rq->h_nr_running--; + + if (cfs_rq_throttled(cfs_rq)) + break; + + update_load_avg(se, 1); + update_cfs_shares(cfs_rq); + } + + if (!se) + sub_nr_running(rq, 1); + + hrtick_update(rq); +} diff --git a/study/kernel/01-process/05-schedule/04-cfs/04-queue/enqueue_task_fair.c b/study/kernel/01-process/05-schedule/04-cfs/04-queue/enqueue_task_fair.c new file mode 100644 index 0000000..2da63dc --- /dev/null +++ b/study/kernel/01-process/05-schedule/04-cfs/04-queue/enqueue_task_fair.c @@ -0,0 +1,51 @@ +/* + * The enqueue_task method is called before nr_running is + * increased. Here we update the fair scheduling stats and + * then put the task into the rbtree: + */ +static void +enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) +{ + struct cfs_rq *cfs_rq; + struct sched_entity *se = &p->se; + + for_each_sched_entity(se) + { + /* 如果当前进程已经在就绪队列上 */ + if (se->on_rq) + break; + + /* 获取到当前进程所在的cfs_rq就绪队列 */ + cfs_rq = cfs_rq_of(se); + /* 内核委托enqueue_entity完成真正的插入工作 */ + enqueue_entity(cfs_rq, se, flags); + + /* + * end evaluation on encountering a throttled cfs_rq + * + * note: in the case of encountering a throttled cfs_rq we will + * post the final h_nr_running increment below. + */ + if (cfs_rq_throttled(cfs_rq)) + break; + cfs_rq->h_nr_running++; + + flags = ENQUEUE_WAKEUP; + } + + for_each_sched_entity(se) { + cfs_rq = cfs_rq_of(se); + cfs_rq->h_nr_running++; + + if (cfs_rq_throttled(cfs_rq)) + break; + + update_load_avg(se, 1); + update_cfs_shares(cfs_rq); + } + + if (!se) + add_nr_running(rq, 1); + + hrtick_update(rq); +} \ No newline at end of file