lkml.org 
[lkml]   [2020]   [Mar]   [25]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    SubjectRe: [RFC PATCH v2 4/4] arch/x86: L1D flush, optimize the context switch
    Date
    On Wed, 2020-03-25 at 18:11 +1100, Balbir Singh wrote:
    > Use a static branch/jump label to optimize the code. Right now
    > we don't ref count the users, but that could be done if needed
    > in the future.
    >
    > Signed-off-by: Balbir Singh <sblbir@amazon.com>
    >

    I sent an older version of the patch, here is the updated version

    Balbir Singh


    From fa80e9691202183a2879a013e497f221b22305a6 Mon Sep 17 00:00:00 2001
    From: Balbir Singh <sblbir@amazon.com>
    Date: Wed, 25 Mar 2020 16:41:18 +1100
    Subject: [RFC PATCH v2 4/4] arch/x86: L1D flush, optimize the context switch

    Use a static branch/jump label to optimize the code. Right now
    we don't ref count the users, but that could be done if needed
    in the future.

    Signed-off-by: Balbir Singh <sblbir@amazon.com>
    ---
    arch/x86/include/asm/nospec-branch.h | 2 ++
    arch/x86/mm/tlb.c | 52 +++++++++++++++++-----------
    2 files changed, 34 insertions(+), 20 deletions(-)

    diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
    index 07e95dcb40ad..371e28cea1f4 100644
    --- a/arch/x86/include/asm/nospec-branch.h
    +++ b/arch/x86/include/asm/nospec-branch.h
    @@ -310,6 +310,8 @@ DECLARE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
    DECLARE_STATIC_KEY_FALSE(mds_user_clear);
    DECLARE_STATIC_KEY_FALSE(mds_idle_clear);

    +DECLARE_STATIC_KEY_FALSE(switch_mm_l1d_flush);
    +
    #include <asm/segment.h>

    /**
    diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
    index 22f96c5f74f0..8f272e5921ce 100644
    --- a/arch/x86/mm/tlb.c
    +++ b/arch/x86/mm/tlb.c
    @@ -155,6 +155,11 @@ EXPORT_SYMBOL_GPL(leave_mm);
    static void *l1d_flush_pages;
    static DEFINE_MUTEX(l1d_flush_mutex);

    +/* Flush L1D on switch_mm() */
    +DEFINE_STATIC_KEY_FALSE(switch_mm_l1d_flush);
    +EXPORT_SYMBOL_GPL(switch_mm_l1d_flush);
    +
    +
    int enable_l1d_flush_for_task(struct task_struct *tsk)
    {
    struct page *page;
    @@ -170,6 +175,11 @@ int enable_l1d_flush_for_task(struct task_struct *tsk)
    l1d_flush_pages = alloc_l1d_flush_pages();
    if (!l1d_flush_pages)
    return -ENOMEM;
    + /*
    + * We could do more accurate ref counting
    + * if needed
    + */
    + static_branch_enable(&switch_mm_l1d_flush);
    }
    mutex_unlock(&l1d_flush_mutex);
    }
    @@ -195,29 +205,31 @@ static void l1d_flush(struct mm_struct *next, struct task_struct *tsk)
    {
    struct mm_struct *real_prev = this_cpu_read(cpu_tlbstate.loaded_mm);

    - /*
    - * If we are not really switching mm's, we can just return
    - */
    - if (real_prev == next)
    - return;
    + if (static_branch_unlikely(&switch_mm_l1d_flush)) {
    + /*
    + * If we are not really switching mm's, we can just return
    + */
    + if (real_prev == next)
    + return;

    - /*
    - * Do we need flushing for by the previous task
    - */
    - if (this_cpu_read(cpu_tlbstate.last_user_mm_l1d_flush) != 0) {
    - if (!flush_l1d_cache_hw())
    - flush_l1d_cache_sw(l1d_flush_pages);
    - this_cpu_write(cpu_tlbstate.last_user_mm_l1d_flush, 0);
    - /* Make sure we clear the values before we set it again */
    - barrier();
    - }
    + /*
    + * Do we need flushing for by the previous task
    + */
    + if (this_cpu_read(cpu_tlbstate.last_user_mm_l1d_flush) != 0) {
    + if (!flush_l1d_cache_hw())
    + flush_l1d_cache_sw(l1d_flush_pages);
    + this_cpu_write(cpu_tlbstate.last_user_mm_l1d_flush, 0);
    + /* Make sure we clear the values before we set it again */
    + barrier();
    + }

    - if (tsk == NULL)
    - return;
    + if (tsk == NULL)
    + return;

    - /* We don't need stringent checks as we opt-in/opt-out */
    - if (test_ti_thread_flag(&tsk->thread_info, TIF_L1D_FLUSH))
    - this_cpu_write(cpu_tlbstate.last_user_mm_l1d_flush, 1);
    + /* We don't need stringent checks as we opt-in/opt-out */
    + if (test_ti_thread_flag(&tsk->thread_info, TIF_L1D_FLUSH))
    + this_cpu_write(cpu_tlbstate.last_user_mm_l1d_flush, 1);
    + }
    }

    void switch_mm(struct mm_struct *prev, struct mm_struct *next,
    --
    2.17.1
    \
     
     \ /
      Last update: 2020-03-25 08:15    [W:7.777 / U:0.868 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site