Messages in this thread Patch in this message | | | Date | Tue, 28 Mar 2023 11:26:35 +0200 | From | Peter Zijlstra <> | Subject | [PATCH 13/17] sched: Merge latency_offset into slice |
| |
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> --- include/linux/sched.h | 2 -- kernel/sched/core.c | 17 +++++++---------- kernel/sched/fair.c | 29 ++++++++++++----------------- kernel/sched/sched.h | 2 +- 4 files changed, 20 insertions(+), 30 deletions(-)
--- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -573,8 +573,6 @@ struct sched_entity { /* cached value of my_q->h_nr_running */ unsigned long runnable_weight; #endif - /* preemption offset in ns */ - long latency_offset; #ifdef CONFIG_SMP /* --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -1285,9 +1285,10 @@ static void set_load_weight(struct task_ } } -static void set_latency_offset(struct task_struct *p) +static inline void set_latency_prio(struct task_struct *p, int prio) { - p->se.latency_offset = calc_latency_offset(p->latency_prio - MAX_RT_PRIO); + p->latency_prio = prio; + set_latency_fair(&p->se, prio - MAX_RT_PRIO); } #ifdef CONFIG_UCLAMP_TASK @@ -4442,7 +4443,7 @@ static void __sched_fork(unsigned long c p->se.vlag = 0; INIT_LIST_HEAD(&p->se.group_node); - set_latency_offset(p); + set_latency_prio(p, p->latency_prio); #ifdef CONFIG_FAIR_GROUP_SCHED p->se.cfs_rq = NULL; @@ -4694,9 +4695,7 @@ int sched_fork(unsigned long clone_flags p->prio = p->normal_prio = p->static_prio; set_load_weight(p, false); - - p->latency_prio = NICE_TO_PRIO(0); - set_latency_offset(p); + set_latency_prio(p, NICE_TO_PRIO(0)); /* * We don't need the reset flag anymore after the fork. It has @@ -7469,10 +7468,8 @@ static void __setscheduler_params(struct static void __setscheduler_latency(struct task_struct *p, const struct sched_attr *attr) { - if (attr->sched_flags & SCHED_FLAG_LATENCY_NICE) { - p->latency_prio = NICE_TO_PRIO(attr->sched_latency_nice); - set_latency_offset(p); - } + if (attr->sched_flags & SCHED_FLAG_LATENCY_NICE) + set_latency_prio(p, NICE_TO_PRIO(attr->sched_latency_nice)); } /* --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -919,12 +919,19 @@ int sched_update_scaling(void) } #endif -long calc_latency_offset(int prio) +void set_latency_fair(struct sched_entity *se, int prio) { u32 weight = sched_prio_to_weight[prio]; u64 base = sysctl_sched_base_slice; - return div_u64(base << SCHED_FIXEDPOINT_SHIFT, weight); + /* + * For EEVDF the virtual time slope is determined by w_i (iow. + * nice) while the request time r_i is determined by + * latency-nice. + * + * Smaller request gets better latency. + */ + se->slice = div_u64(base << SCHED_FIXEDPOINT_SHIFT, weight); } /* @@ -937,13 +944,6 @@ static void update_deadline(struct cfs_r return; /* - * For EEVDF the virtual time slope is determined by w_i (iow. - * nice) while the request time r_i is determined by - * latency-nice. - */ - se->slice = se->latency_offset; - - /* * EEVDF: vd_i = ve_i + r_i / w_i */ se->deadline = se->vruntime + calc_delta_fair(se->slice, se); @@ -12231,7 +12231,7 @@ void init_tg_cfs_entry(struct task_group se->my_q = cfs_rq; - se->latency_offset = calc_latency_offset(tg->latency_prio - MAX_RT_PRIO); + set_latency_fair(se, tg->latency_prio - MAX_RT_PRIO); /* guarantee group entities always have weight */ update_load_set(&se->load, NICE_0_LOAD); @@ -12365,7 +12365,6 @@ int sched_group_set_idle(struct task_gro int sched_group_set_latency(struct task_group *tg, int prio) { - long latency_offset; int i; if (tg == &root_task_group) @@ -12379,13 +12378,9 @@ int sched_group_set_latency(struct task_ } tg->latency_prio = prio; - latency_offset = calc_latency_offset(prio - MAX_RT_PRIO); - for_each_possible_cpu(i) { - struct sched_entity *se = tg->se[i]; - - WRITE_ONCE(se->latency_offset, latency_offset); - } + for_each_possible_cpu(i) + set_latency_fair(tg->se[i], prio - MAX_RT_PRIO); mutex_unlock(&shares_mutex); return 0; --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2477,7 +2477,7 @@ extern unsigned int sysctl_numa_balancin extern unsigned int sysctl_numa_balancing_hot_threshold; #endif -extern long calc_latency_offset(int prio); +extern void set_latency_fair(struct sched_entity *se, int prio); #ifdef CONFIG_SCHED_HRTICK
| |