From a221a217a891e4580607c6a3a1fe3553c989dbac Mon Sep 17 00:00:00 2001 From: Cheng Jian Date: Thu, 3 Nov 2022 11:25:58 +0800 Subject: [PATCH] scheduler/load_balance: Load Balance GraphViz --- .../07-cfs/10-load_balance/load_balance.gv | 30 ++++++++++ .../10-load_balance/load_balance_all.gv | 56 +++++++++++++++++++ .../07-cfs/10-load_balance/newidle_balance.gv | 27 +++++++++ .../10-load_balance/nohz_idle_balance.gv | 36 ++++++++++++ .../10-load_balance/tick_load_balance.gv | 18 ++++++ 5 files changed, 167 insertions(+) create mode 100644 study/kernel/01-process/05-schedule/07-cfs/10-load_balance/load_balance.gv create mode 100644 study/kernel/01-process/05-schedule/07-cfs/10-load_balance/load_balance_all.gv create mode 100644 study/kernel/01-process/05-schedule/07-cfs/10-load_balance/newidle_balance.gv create mode 100644 study/kernel/01-process/05-schedule/07-cfs/10-load_balance/nohz_idle_balance.gv create mode 100644 study/kernel/01-process/05-schedule/07-cfs/10-load_balance/tick_load_balance.gv diff --git a/study/kernel/01-process/05-schedule/07-cfs/10-load_balance/load_balance.gv b/study/kernel/01-process/05-schedule/07-cfs/10-load_balance/load_balance.gv new file mode 100644 index 0000000..a95eab3 --- /dev/null +++ b/study/kernel/01-process/05-schedule/07-cfs/10-load_balance/load_balance.gv @@ -0,0 +1,30 @@ +digraph load_balance { + + // 周期性负载均衡器 run_rebalance_domains + //rebalance_domains -> for_each_domain_LB -> {update_newidle_cost, get_sd_balance_interval, load_balance, update_next_balance}; + rebalance_domains -> load_balance; + + // IDLE 负载均衡器 + //newidle_balance -> {update_misfit_status, update_next_balance, update_blocked_averages, for_each_domain_ILB} -> nohz_newidle_balance; + //for_each_domain_ILB -> {update_next_balance, load_balance, update_newidle_cost}; + newidle_balance -> load_balance; + + // 负载均衡核心逻辑 + load_balance:s -> {should_we_balance, find_busiest_group, find_busiest_queue, detach_tasks, attach_tasks, need_active_balance, stop_one_cpu_nowait}; + should_we_balance -> {group_balance_mask, group_balance_cpu}; + find_busiest_group -> {init_sd_lb_stats, update_sd_lb_stats, sched_energy_enabled, compare_local_busiest_group_type, calculate_imbalance}; + find_busiest_queue -> for_each_sched_group_span -> {fbq_classify_rq, switch_env_migration_type_FBQ}; + detach_tasks -> {can_migrate_task, switch_env_migration_type_DT, detach_task}; + attach_tasks -> {attach_task}; + need_active_balance -> stop_one_cpu_nowait -> active_load_balance_cpu_stop [style=dashed]; + active_load_balance_cpu_stop -> {detach_one_task, attach_one_task}; + + rebalance_domains [label="rebalance_domains"; style=filled; fillcolor=palegreen; shape=oval]; + newidle_balance [label="newidle_balance"; style=filled; fillcolor=palegreen; shape=oval]; + + load_balance [label="load_balance"; style=filled; fillcolor=palegreen; shape=oval]; + find_busiest_group [label="find_busiest_group"; style=filled; fillcolor=lightyellow; shape=oval]; + find_busiest_queue [label="find_busiest_queue"; style=filled; fillcolor=lightyellow; shape=oval]; + active_load_balance_cpu_stop [label="active_load_balance_cpu_stop"; style=filled; fillcolor=lightyellow; shape=oval]; + can_migrate_task [label="can_migrate_task"; style=filled; fillcolor=lightyellow; shape=oval]; +} \ No newline at end of file diff --git a/study/kernel/01-process/05-schedule/07-cfs/10-load_balance/load_balance_all.gv b/study/kernel/01-process/05-schedule/07-cfs/10-load_balance/load_balance_all.gv new file mode 100644 index 0000000..129bfa0 --- /dev/null +++ b/study/kernel/01-process/05-schedule/07-cfs/10-load_balance/load_balance_all.gv @@ -0,0 +1,56 @@ +digraph T { + + // 周期性负载均衡器 run_rebalance_domains + scheduler_tick -> trigger_load_balance -> {raise_softirq_SCHED_SOFTIRQ, nohz_balancer_kick}; + raise_softirq_SCHED_SOFTIRQ -> run_rebalance_domains [style=dashed]; + run_rebalance_domains -> {nohz_idle_balance, update_blocked_averages, rebalance_domains}; + + rebalance_domains -> for_each_domain_LB -> {update_newidle_cost, get_sd_balance_interval, load_balance, update_next_balance}; + + // NOHZ 负载均衡器 nohz_idle_balance + nohz_idle_balance -> {clear_nohz_idle_balance, _nohz_idle_balance}; + _nohz_idle_balance -> for_each_nohz_idle_cpus_mask -> {has_NOHZ_STATS_KICK, has_NOHZ_BALANCE_KICK}; + has_NOHZ_STATS_KICK -> update_nohz_stats -> update_blocked_averages; + has_NOHZ_BALANCE_KICK -> rebalance_domains; + + // IDLE 负载均衡器 + __schedule -> pick_next_task -> __pick_next_task -> pick_next_task_fair -> newidle_balance; + pick_next_task -> put_prev_task_balance + put_prev_task_balance -> for_class_range_prev_to_idle -> balance -> balance_fair -> newidle_balance; + __pick_next_task -> {pick_next_task_fair, pick_next_task_idle, put_prev_task_balance, class_pick_next_task}; + newidle_balance -> {update_misfit_status, update_next_balance, update_blocked_averages, for_each_domain_ILB} -> nohz_newidle_balance; + + for_each_domain_ILB -> {update_next_balance, load_balance, update_newidle_cost}; + nohz_newidle_balance -> set_NOHZ_NEWILB_KICK + + // 负载均衡核心逻辑 + load_balance:s -> {should_we_balance, find_busiest_group, find_busiest_queue, detach_tasks, attach_tasks, need_active_balance, stop_one_cpu_nowait}; + should_we_balance -> {group_balance_mask, group_balance_cpu}; + find_busiest_group -> {init_sd_lb_stats, update_sd_lb_stats, sched_energy_enabled, compare_local_busiest_group_type, calculate_imbalance}; + find_busiest_queue -> for_each_sched_group_span -> {fbq_classify_rq, switch_env_migration_type_FBQ}; + detach_tasks -> {can_migrate_task, switch_env_migration_type_DT, detach_task}; + attach_tasks -> {attach_task}; + need_active_balance -> stop_one_cpu_nowait -> active_load_balance_cpu_stop [style=dashed]; + active_load_balance_cpu_stop -> {detach_one_task, attach_one_task}; + + + // update_blocked_averages + update_blocked_averages -> {update_blocked_load_tick, __update_blocked_others, __update_blocked_fair, update_blocked_load_status, cpufreq_update_util}; + __update_blocked_others -> {arch_scale_thermal_pressure, update_rt_rq_load_avg, update_dl_rq_load_avg, update_thermal_load_avg, update_irq_load_avg, others_have_blocked}; + __update_blocked_fair -> {update_cfs_rq_load_avg, cfs_rq_has_blocked}; + + + clear_nohz_idle_balance [label="this_rq->nohz_idle_balance = 0"; style=filled; shape=oval]; + raise_softirq_SCHED_SOFTIRQ [label="raise_softirq(SCHED_SOFTIRQ)"; style=filled; shape=oval]; + + rebalance_domains [label="rebalance_domains"; style=filled; fillcolor=palegreen; shape=oval]; + newidle_balance [label="newidle_balance"; style=filled; fillcolor=palegreen; shape=oval]; + nohz_idle_balance [label="nohz_idle_balance"; style=filled; fillcolor=palegreen; shape=oval]; + + load_balance [label="load_balance"; style=filled; fillcolor=palegreen; shape=oval]; + find_busiest_group [label="find_busiest_group"; style=filled; fillcolor=lightyellow; shape=oval]; + find_busiest_queue [label="find_busiest_queue"; style=filled; fillcolor=lightyellow; shape=oval]; + active_load_balance_cpu_stop [label="active_load_balance_cpu_stop"; style=filled; fillcolor=lightyellow; shape=oval]; + can_migrate_task [label="can_migrate_task"; style=filled; fillcolor=lightyellow; shape=oval]; + +} \ No newline at end of file diff --git a/study/kernel/01-process/05-schedule/07-cfs/10-load_balance/newidle_balance.gv b/study/kernel/01-process/05-schedule/07-cfs/10-load_balance/newidle_balance.gv new file mode 100644 index 0000000..5c9ed3f --- /dev/null +++ b/study/kernel/01-process/05-schedule/07-cfs/10-load_balance/newidle_balance.gv @@ -0,0 +1,27 @@ +digraph newidle_balance { + // IDLE 负载均衡器 + __schedule -> pick_next_task;// -> {__pick_next_task, put_prev_task_balance}; + pick_next_task -> __pick_next_task [label="!sched_core_enabled(rq)"]; + pick_next_task -> put_prev_task_balance [label="sched_core_enabled"]; + + __pick_next_task -> {pick_next_task_fair, pick_next_task_idle, put_prev_task_balance, class_pick_next_task}; + class_pick_next_task -> pick_next_task_fair [style=dashed]; + put_prev_task_balance -> for_class_range_prev_to_idle -> class_balance -> balance_fair -> newidle_balance; + pick_next_task_fair -> newidle_balance + newidle_balance -> {update_misfit_status, update_next_balance, update_blocked_averages, for_each_domain_ILB} -> nohz_newidle_balance; + + for_each_domain_ILB -> {update_next_balance, load_balance, update_newidle_cost}; + nohz_newidle_balance -> set_NOHZ_NEWILB_KICK + + // caller + pick_next_task_fair [label="pick_next_task_fair"; style=filled; fillcolor=lightyellow; shape=oval]; + class_balance [label="class->balance"; style=filled; fillcolor=lightyellow; shape=oval]; + class_pick_next_task [label="class->pick_next_task"; style=filled; fillcolor=lightyellow; shape=oval]; + balance_fair [label="balance_fair"; style=filled; fillcolor=lightyellow; shape=oval]; + + // function + newidle_balance [label="newidle_balance"; style=filled; fillcolor=palegreen; shape=oval]; + + // callee + load_balance [label="load_balance"; style=filled; fillcolor=orange; shape=oval]; +} \ No newline at end of file diff --git a/study/kernel/01-process/05-schedule/07-cfs/10-load_balance/nohz_idle_balance.gv b/study/kernel/01-process/05-schedule/07-cfs/10-load_balance/nohz_idle_balance.gv new file mode 100644 index 0000000..fa060d2 --- /dev/null +++ b/study/kernel/01-process/05-schedule/07-cfs/10-load_balance/nohz_idle_balance.gv @@ -0,0 +1,36 @@ +digraph nohz_idle_balance { + // 周期性负载均衡器 run_rebalance_domains + //run_rebalance_domains -> {nohz_idle_balance, update_blocked_averages, rebalance_domains}; + scheduler_tick -> trigger_load_balance -> {raise_softirq_SCHED_SOFTIRQ, nohz_balancer_kick}; + run_rebalance_domains -> nohz_idle_balance [label="has NOHZ_BALANCE_KICK"]; + run_rebalance_domains -> {/* nohz_idle_balance, */update_blocked_averages, rebalance_domains}; + + // KICK + nohz_balancer_kick -> {set_NOHZ_STATS_KICK, set_NOHZ_BALANCE_KICK, set_NOHZ_NEXT_KICK} -> kick_ilb -> find_new_ilb -> KICK_CPU_BY_IPI -> nohz_csd_func -> raise_softirq_SCHED_SOFTIRQ; + + // NOHZ 负载均衡器 nohz_idle_balance + nohz_idle_balance -> {clear_nohz_idle_balance, _nohz_idle_balance}; + /* + _nohz_idle_balance -> for_each_nohz_idle_cpus_mask -> {has_NOHZ_STATS_KICK, has_NOHZ_BALANCE_KICK}; + has_NOHZ_STATS_KICK -> update_nohz_stats -> update_blocked_averages; + has_NOHZ_BALANCE_KICK -> rebalance_domains; + */ + _nohz_idle_balance -> for_each_nohz_idle_cpus_mask; + for_each_nohz_idle_cpus_mask -> update_nohz_stats -> update_blocked_averages [label="has NOHZ_STATS_KICK"]; + for_each_nohz_idle_cpus_mask -> rebalance_domains [label="has NOHZ_BALANCE_KICK"]; + + // IDLE 负载均衡器 + newidle_balance -> nohz_newidle_balance; + + nohz_newidle_balance -> set_NOHZ_NEWILB_KICK + + raise_softirq_SCHED_SOFTIRQ -> run_rebalance_domains [style=dashed]; + raise_softirq_SCHED_SOFTIRQ [label="raise_softirq(SCHED_SOFTIRQ)"; style=filled; shape=oval] + clear_nohz_idle_balance [label="this_rq->nohz_idle_balance = 0"; style=filled; shape=oval]; + + rebalance_domains [label="rebalance_domains"; style=filled; fillcolor=palegreen; shape=oval]; + newidle_balance [label="newidle_balance"; style=filled; fillcolor=palegreen; shape=oval]; + nohz_idle_balance [label="nohz_idle_balance"; style=filled; fillcolor=palegreen; shape=oval]; + set_NOHZ_NEWILB_KICK [label="atomic_or(NOHZ_NEWILB_KICK, nohz_flags(this_cpu))"; style=filled; fillcolor=palegreen; shape=oval]; + KICK_CPU_BY_IPI [label="smp_call_function_single_async(ilb_cpu, &cpu_rq(ilb_cpu)->nohz_csd)"; style=filled; fillcolor=palegreen; shape=oval]; +} \ No newline at end of file diff --git a/study/kernel/01-process/05-schedule/07-cfs/10-load_balance/tick_load_balance.gv b/study/kernel/01-process/05-schedule/07-cfs/10-load_balance/tick_load_balance.gv new file mode 100644 index 0000000..a3164ac --- /dev/null +++ b/study/kernel/01-process/05-schedule/07-cfs/10-load_balance/tick_load_balance.gv @@ -0,0 +1,18 @@ +digraph rebalance_domains { + + // 周期性负载均衡器 run_rebalance_domains + scheduler_tick -> trigger_load_balance -> {raise_softirq_SCHED_SOFTIRQ, nohz_balancer_kick}; + raise_softirq_SCHED_SOFTIRQ -> run_rebalance_domains [style=dashed]; + run_rebalance_domains -> {nohz_idle_balance, update_blocked_averages, rebalance_domains}; + + rebalance_domains -> for_each_domain_LB -> {update_newidle_cost, get_sd_balance_interval, load_balance, update_next_balance}; + + // caller + raise_softirq_SCHED_SOFTIRQ [label="raise_softirq(SCHED_SOFTIRQ)"; style=filled; fillcolor=lightyellow; shape=oval]; + + // function + rebalance_domains [label="rebalance_domains"; style=filled; fillcolor=palegreen; shape=oval]; + + // callee + load_balance [label="load_balance"; style=filled; fillcolor=orange; shape=oval]; +} \ No newline at end of file