Messages in this thread Patch in this message | | | From | Rik van Riel <> | Subject | [PATCH 04/11] x86,mm: use on_each_cpu_cond for TLB flushes | Date | Wed, 1 Aug 2018 06:02:48 -0400 |
| |
Instead of open coding bitmap magic, use on_each_cpu_cond to determine which CPUs to send TLB flush IPIs to.
This might be a little bit slower than examining the bitmaps, but it should be a lot easier to maintain in the long run.
Suggested-by: Peter Zijlstra <peterz@infradead.org> Signed-off-by: Rik van Riel <riel@surriel.com> --- arch/x86/mm/tlb.c | 75 +++++++++++-------------------------------------------- 1 file changed, 15 insertions(+), 60 deletions(-)
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index 5321e02c4e09..671cc66df801 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -582,12 +582,19 @@ static void flush_tlb_func_remote(void *info) flush_tlb_func_common(f, false, TLB_REMOTE_SHOOTDOWN); } +static bool tlb_is_lazy(int cpu, void *data) +{ + return per_cpu(cpu_tlbstate.is_lazy, cpu); +} + +static bool tlb_is_not_lazy(int cpu, void *data) +{ + return !per_cpu(cpu_tlbstate.is_lazy, cpu); +} + void native_flush_tlb_others(const struct cpumask *cpumask, const struct flush_tlb_info *info) { - cpumask_var_t lazymask; - unsigned int cpu; - count_vm_tlb_event(NR_TLB_REMOTE_FLUSH); if (info->end == TLB_FLUSH_ALL) trace_tlb_flush(TLB_REMOTE_SEND_IPI, TLB_FLUSH_ALL); @@ -596,6 +603,7 @@ void native_flush_tlb_others(const struct cpumask *cpumask, (info->end - info->start) >> PAGE_SHIFT); if (is_uv_system()) { + unsigned int cpu; /* * This whole special case is confused. UV has a "Broadcast * Assist Unit", which seems to be a fancy way to send IPIs. @@ -619,28 +627,8 @@ void native_flush_tlb_others(const struct cpumask *cpumask, return; } - /* - * A temporary cpumask is used in order to skip sending IPIs - * to CPUs in lazy TLB state, while keeping them in mm_cpumask(mm). - * If the allocation fails, simply IPI every CPU in mm_cpumask. - */ - if (!alloc_cpumask_var(&lazymask, GFP_ATOMIC)) { - smp_call_function_many(cpumask, flush_tlb_func_remote, - (void *)info, 1); - return; - } - - cpumask_copy(lazymask, cpumask); - - for_each_cpu(cpu, lazymask) { - if (per_cpu(cpu_tlbstate.is_lazy, cpu)) - cpumask_clear_cpu(cpu, lazymask); - } - - smp_call_function_many(lazymask, flush_tlb_func_remote, - (void *)info, 1); - - free_cpumask_var(lazymask); + on_each_cpu_cond_mask(tlb_is_not_lazy, flush_tlb_func_remote, + (void *)info, 1, GFP_ATOMIC, cpumask); } /* @@ -709,50 +697,17 @@ void tlb_flush_remove_tables_local(void *arg) } } -static void mm_fill_lazy_tlb_cpu_mask(struct mm_struct *mm, - struct cpumask *lazy_cpus) -{ - int cpu; - - for_each_cpu(cpu, mm_cpumask(mm)) { - if (!per_cpu(cpu_tlbstate.is_lazy, cpu)) - cpumask_set_cpu(cpu, lazy_cpus); - } -} - void tlb_flush_remove_tables(struct mm_struct *mm) { int cpu = get_cpu(); - cpumask_var_t lazy_cpus; if (cpumask_any_but(mm_cpumask(mm), cpu) >= nr_cpu_ids) { put_cpu(); return; } - if (!zalloc_cpumask_var(&lazy_cpus, GFP_ATOMIC)) { - /* - * If the cpumask allocation fails, do a brute force flush - * on all the CPUs that have this mm loaded. - */ - smp_call_function_many(mm_cpumask(mm), - tlb_flush_remove_tables_local, (void *)mm, 1); - put_cpu(); - return; - } - - /* - * CPUs with !is_lazy either received a TLB flush IPI while the user - * pages in this address range were unmapped, or have context switched - * and reloaded %CR3 since then. - * - * Shootdown IPIs at page table freeing time only need to be sent to - * CPUs that may have out of date TLB contents. - */ - mm_fill_lazy_tlb_cpu_mask(mm, lazy_cpus); - smp_call_function_many(lazy_cpus, - tlb_flush_remove_tables_local, (void *)mm, 1); - free_cpumask_var(lazy_cpus); + on_each_cpu_cond_mask(tlb_is_lazy, tlb_flush_remove_tables_local, + (void *)mm, 1, GFP_ATOMIC, mm_cpumask(mm)); put_cpu(); } -- 2.14.4
| |