Messages in this thread Patch in this message | | | Date | Tue, 28 Nov 2017 17:39:08 +0100 | From | Peter Zijlstra <> | Subject | Re: [PATCH 15/24] x86/mm: Allow flushing for future ASID switches |
| |
On Mon, Nov 27, 2017 at 09:16:19PM -0800, Andy Lutomirski wrote: > We need to split up __flush_tlb_one() into __flush_tlb_one() and > __flush_tlb_one_kernel(). We've gotten away with having a single > function for both this long because we've never had PCID on and > nonglobal kernel mappings around. So we're busted starting with > "x86/mm/kaiser: Disable global pages by default with KAISER", which > means that we have a potential corruption issue affecting anyone who > tries to bisect the series.
Didn't do that..
> Then we need to make the kernel variant do something sane (presumably > just call __flush_tlb_all if we have PCID && !PGE). And, for the user > variant, we need a straightforward, clean, efficient way to mark a > given address space on a given CPU as needing a usermode PCID flush > when its usermode tables are next loaded. This patch isn't it.
Did give this a try, mostly also to get PCID + !INVPCID working for my IVB.
The below is fairly ugly but it does boot and build a kernel so its not immensely broken.
--- diff --git a/arch/x86/entry/calling.h b/arch/x86/entry/calling.h index 07fa7fdd7b68..d7f1be6ccc97 100644 --- a/arch/x86/entry/calling.h +++ b/arch/x86/entry/calling.h @@ -4,6 +4,7 @@ #include <asm/cpufeatures.h> #include <asm/page_types.h> #include <asm/pgtable_types.h> +#include <asm/percpu.h> /* @@ -220,7 +221,21 @@ For 32-bit we have the following conventions - kernel is built with .macro SWITCH_TO_USER_CR3 scratch_reg:req STATIC_JUMP_IF_FALSE .Lend_\@, kaiser_enabled_key, def=1 mov %cr3, \scratch_reg - ADJUST_USER_CR3 \scratch_reg + push \scratch_reg + andq $(0x7FF), \scratch_reg + bt \scratch_reg, PER_CPU_VAR(__asid_flush) + jnc .Lnoflush_\@ + + btr \scratch_reg, PER_CPU_VAR(__asid_flush) + pop \scratch_reg + jmp .Ldo_\@ + +.Lnoflush_\@: + pop \scratch_reg + ALTERNATIVE "", "bts $63, \scratch_reg", X86_FEATURE_PCID + +.Ldo_\@: + orq $(KAISER_SWITCH_MASK), \scratch_reg mov \scratch_reg, %cr3 .Lend_\@: .endm diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h index 27eb7e8c5e84..1fb137da4c9f 100644 --- a/arch/x86/include/asm/tlbflush.h +++ b/arch/x86/include/asm/tlbflush.h @@ -9,6 +9,7 @@ #include <asm/cpufeature.h> #include <asm/special_insns.h> #include <asm/smp.h> +#include <asm/kaiser.h> static inline void __invpcid(unsigned long pcid, unsigned long addr, unsigned long type) @@ -347,9 +348,33 @@ static inline void cr4_set_bits_and_update_boot(unsigned long mask) extern void initialize_tlbstate_and_flush(void); +DECLARE_PER_CPU(unsigned long, __asid_flush); + +/* + * Given an asid, flush the corresponding KAISER user ASID. + */ +static inline void flush_user_asid(u16 asid) +{ + /* There is no user ASID if KAISER is off */ + if (!IS_ENABLED(CONFIG_KAISER)) + return; + /* + * We only have a single ASID if PCID is off and the CR3 + * write will have flushed it. + */ + if (!cpu_feature_enabled(X86_FEATURE_PCID)) + return; + + if (!kaiser_enabled) + return; + + __set_bit(kern_asid(asid), this_cpu_ptr(&__asid_flush)); +} + static inline void __native_flush_tlb(void) { if (!cpu_feature_enabled(X86_FEATURE_INVPCID)) { +#if 0 /* * native_write_cr3() only clears the current PCID if * CR4 has X86_CR4_PCIDE set. In other words, this does @@ -358,9 +383,10 @@ static inline void __native_flush_tlb(void) * With KAISER and PCIDs, the means that we did not * flush the user PCID. Warn if it gets called. */ - if (IS_ENABLED(CONFIG_KAISER)) - WARN_ON_ONCE(this_cpu_read(cpu_tlbstate.cr4) & - X86_CR4_PCIDE); + if (IS_ENABLED(CONFIG_KAISER) && kaiser_enabled) + WARN_ON_ONCE(this_cpu_read(cpu_tlbstate.cr4) & X86_CR4_PCIDE); +#endif + flush_user_asid(this_cpu_read(cpu_tlbstate.loaded_mm_asid)); /* * If current->mm == NULL then we borrow a mm * which may change during a task switch and @@ -435,6 +461,8 @@ static inline void __native_flush_tlb_single(unsigned long addr) * early. */ if (!this_cpu_has(X86_FEATURE_INVPCID_SINGLE)) { + flush_user_asid(loaded_mm_asid); + asm volatile("invlpg (%0)" ::"r" (addr) : "memory"); return; } diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c index 72f115178d14..2dcd01615772 100644 --- a/arch/x86/mm/init.c +++ b/arch/x86/mm/init.c @@ -218,11 +218,13 @@ static void setup_pcid(void) * INVPCID. Just avoid using PCIDs at all if we * have KAISER and do not have INVPCID. */ +#if 0 if (!IS_ENABLED(CONFIG_X86_GLOBAL_PAGES) && - !boot_cpu_has(X86_FEATURE_INVPCID)) { + kaiser_enabled && !boot_cpu_has(X86_FEATURE_INVPCID)) { setup_clear_cpu_cap(X86_FEATURE_PCID); return; } +#endif /* * This can't be cr4_set_bits_and_update_boot() -- * the trampoline code can't handle CR4.PCIDE and diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index f75b6eb47a6d..4ed1d0dfd54f 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -100,55 +100,14 @@ static void choose_new_asid(struct mm_struct *next, u64 next_tlb_gen, *need_flush = true; } -/* - * Given a kernel asid, flush the corresponding KAISER - * user ASID. - */ -static void flush_user_asid(pgd_t *pgd, u16 kern_asid) -{ - /* There is no user ASID if KAISER is off */ - if (!IS_ENABLED(CONFIG_KAISER)) - return; - /* - * We only have a single ASID if PCID is off and the CR3 - * write will have flushed it. - */ - if (!cpu_feature_enabled(X86_FEATURE_PCID)) - return; - /* - * With PCIDs enabled, write_cr3() only flushes TLB - * entries for the current (kernel) ASID. This leaves - * old TLB entries for the user ASID in place and we must - * flush that context separately. We can theoretically - * delay doing this until we actually load up the - * userspace CR3, but do it here for simplicity. - */ - if (cpu_feature_enabled(X86_FEATURE_INVPCID)) { - invpcid_flush_single_context(user_asid(kern_asid)); - } else { - /* - * On systems with PCIDs, but no INVPCID, the only - * way to flush a PCID is a CR3 write. Note that - * we use the kernel page tables with the *user* - * ASID here. - */ - unsigned long user_asid_flush_cr3; - user_asid_flush_cr3 = build_cr3(pgd, user_asid(kern_asid)); - write_cr3(user_asid_flush_cr3); - /* - * We do not use PCIDs with KAISER unless we also - * have INVPCID. Getting here is unexpected. - */ - WARN_ON_ONCE(1); - } -} +__visible DEFINE_PER_CPU(unsigned long, __asid_flush); static void load_new_mm_cr3(pgd_t *pgdir, u16 new_asid, bool need_flush) { unsigned long new_mm_cr3; if (need_flush) { - flush_user_asid(pgdir, new_asid); + flush_user_asid(new_asid); new_mm_cr3 = build_cr3(pgdir, new_asid); } else { new_mm_cr3 = build_cr3_noflush(pgdir, new_asid);
| |