lkml.org 
[lkml]   [2020]   [Apr]   [26]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    Date
    From
    Subject[tip: x86/mm] x86/tlb: Move __flush_tlb_global() out of line
    The following commit has been merged into the x86/mm branch of tip:

    Commit-ID: cd30d26cf307b45159cd629d60b989e582372afe
    Gitweb: https://git.kernel.org/tip/cd30d26cf307b45159cd629d60b989e582372afe
    Author: Thomas Gleixner <tglx@linutronix.de>
    AuthorDate: Tue, 21 Apr 2020 11:20:33 +02:00
    Committer: Borislav Petkov <bp@suse.de>
    CommitterDate: Sun, 26 Apr 2020 11:00:27 +02:00

    x86/tlb: Move __flush_tlb_global() out of line

    cpu_tlbstate is exported because various TLB-related functions need
    access to it, but cpu_tlbstate is sensitive information which should
    only be accessed by well-contained kernel functions and not be directly
    exposed to modules.

    As a second step, move __flush_tlb_global() out of line and hide the
    native function. The latter can be static when CONFIG_PARAVIRT is
    disabled.

    Consolidate the namespace while at it and remove the pointless extra
    wrapper in the paravirt code.

    No functional change.

    Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
    Signed-off-by: Borislav Petkov <bp@suse.de>
    Reviewed-by: Alexandre Chartre <alexandre.chartre@oracle.com>
    Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
    Link: https://lkml.kernel.org/r/20200421092559.336916818@linutronix.de
    ---
    arch/x86/include/asm/paravirt.h | 1 +-
    arch/x86/include/asm/tlbflush.h | 38 +-----------------------------
    arch/x86/kernel/paravirt.c | 9 +-------
    arch/x86/mm/tlb.c | 41 ++++++++++++++++++++++++++++++++-
    4 files changed, 44 insertions(+), 45 deletions(-)

    diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
    index f412450..712e059 100644
    --- a/arch/x86/include/asm/paravirt.h
    +++ b/arch/x86/include/asm/paravirt.h
    @@ -48,6 +48,7 @@ static inline void slow_down_io(void)
    }

    void native_flush_tlb_local(void);
    +void native_flush_tlb_global(void);

    static inline void __flush_tlb_local(void)
    {
    diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
    index fe1fd02..d66d16e 100644
    --- a/arch/x86/include/asm/tlbflush.h
    +++ b/arch/x86/include/asm/tlbflush.h
    @@ -141,11 +141,11 @@ static inline unsigned long build_cr3_noflush(pgd_t *pgd, u16 asid)
    }

    void flush_tlb_local(void);
    +void flush_tlb_global(void);

    #ifdef CONFIG_PARAVIRT
    #include <asm/paravirt.h>
    #else
    -#define __flush_tlb_global() __native_flush_tlb_global()
    #define __flush_tlb_one_user(addr) __native_flush_tlb_one_user(addr)
    #endif

    @@ -372,40 +372,6 @@ static inline void invalidate_user_asid(u16 asid)
    }

    /*
    - * flush everything
    - */
    -static inline void __native_flush_tlb_global(void)
    -{
    - unsigned long cr4, flags;
    -
    - if (static_cpu_has(X86_FEATURE_INVPCID)) {
    - /*
    - * Using INVPCID is considerably faster than a pair of writes
    - * to CR4 sandwiched inside an IRQ flag save/restore.
    - *
    - * Note, this works with CR4.PCIDE=0 or 1.
    - */
    - invpcid_flush_all();
    - return;
    - }
    -
    - /*
    - * Read-modify-write to CR4 - protect it from preemption and
    - * from interrupts. (Use the raw variant because this code can
    - * be called from deep inside debugging code.)
    - */
    - raw_local_irq_save(flags);
    -
    - cr4 = this_cpu_read(cpu_tlbstate.cr4);
    - /* toggle PGE */
    - native_write_cr4(cr4 ^ X86_CR4_PGE);
    - /* write old PGE again and flush TLBs */
    - native_write_cr4(cr4);
    -
    - raw_local_irq_restore(flags);
    -}
    -
    -/*
    * flush one page in the user mapping
    */
    static inline void __native_flush_tlb_one_user(unsigned long addr)
    @@ -439,7 +405,7 @@ static inline void __flush_tlb_all(void)
    VM_WARN_ON_ONCE(preemptible());

    if (boot_cpu_has(X86_FEATURE_PGE)) {
    - __flush_tlb_global();
    + flush_tlb_global();
    } else {
    /*
    * !PGE -> !PCID (setup_pcid()), thus every flush is total.
    diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
    index 4cb3d82..6094b00 100644
    --- a/arch/x86/kernel/paravirt.c
    +++ b/arch/x86/kernel/paravirt.c
    @@ -160,15 +160,6 @@ unsigned paravirt_patch_insns(void *insn_buff, unsigned len,
    return insn_len;
    }

    -/*
    - * Global pages have to be flushed a bit differently. Not a real
    - * performance problem because this does not happen often.
    - */
    -static void native_flush_tlb_global(void)
    -{
    - __native_flush_tlb_global();
    -}
    -
    static void native_flush_tlb_one_user(unsigned long addr)
    {
    __native_flush_tlb_one_user(addr);
    diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
    index 0611648..d548b98 100644
    --- a/arch/x86/mm/tlb.c
    +++ b/arch/x86/mm/tlb.c
    @@ -23,6 +23,7 @@
    #else
    # define STATIC_NOPV static
    # define __flush_tlb_local native_flush_tlb_local
    +# define __flush_tlb_global native_flush_tlb_global
    #endif

    /*
    @@ -891,6 +892,46 @@ unsigned long __get_current_cr3_fast(void)
    EXPORT_SYMBOL_GPL(__get_current_cr3_fast);

    /*
    + * Flush everything
    + */
    +STATIC_NOPV void native_flush_tlb_global(void)
    +{
    + unsigned long cr4, flags;
    +
    + if (static_cpu_has(X86_FEATURE_INVPCID)) {
    + /*
    + * Using INVPCID is considerably faster than a pair of writes
    + * to CR4 sandwiched inside an IRQ flag save/restore.
    + *
    + * Note, this works with CR4.PCIDE=0 or 1.
    + */
    + invpcid_flush_all();
    + return;
    + }
    +
    + /*
    + * Read-modify-write to CR4 - protect it from preemption and
    + * from interrupts. (Use the raw variant because this code can
    + * be called from deep inside debugging code.)
    + */
    + raw_local_irq_save(flags);
    +
    + cr4 = this_cpu_read(cpu_tlbstate.cr4);
    + /* toggle PGE */
    + native_write_cr4(cr4 ^ X86_CR4_PGE);
    + /* write old PGE again and flush TLBs */
    + native_write_cr4(cr4);
    +
    + raw_local_irq_restore(flags);
    +}
    +
    +void flush_tlb_global(void)
    +{
    + __flush_tlb_global();
    +}
    +EXPORT_SYMBOL_GPL(flush_tlb_global);
    +
    +/*
    * Flush the entire current user mapping
    */
    STATIC_NOPV void native_flush_tlb_local(void)
    \
     
     \ /
      Last update: 2020-04-26 20:44    [W:3.996 / U:0.060 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site