lkml.org 
[lkml]   [2023]   [Jan]   [9]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    Date
    Subject[PATCH 34/41] x86/mm: try VMA lock-based page fault handling first
    From
    Attempt VMA lock-based page fault handling first, and fall back to the
    existing mmap_lock-based handling if that fails.

    Signed-off-by: Suren Baghdasaryan <surenb@google.com>
    ---
    arch/x86/Kconfig | 1 +
    arch/x86/mm/fault.c | 36 ++++++++++++++++++++++++++++++++++++
    2 files changed, 37 insertions(+)

    diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
    index 3604074a878b..3647f7bdb110 100644
    --- a/arch/x86/Kconfig
    +++ b/arch/x86/Kconfig
    @@ -27,6 +27,7 @@ config X86_64
    # Options that are inherently 64-bit kernel only:
    select ARCH_HAS_GIGANTIC_PAGE
    select ARCH_SUPPORTS_INT128 if CC_HAS_INT128
    + select ARCH_SUPPORTS_PER_VMA_LOCK
    select ARCH_USE_CMPXCHG_LOCKREF
    select HAVE_ARCH_SOFT_DIRTY
    select MODULES_USE_ELF_RELA
    diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
    index 7b0d4ab894c8..983266e7c49b 100644
    --- a/arch/x86/mm/fault.c
    +++ b/arch/x86/mm/fault.c
    @@ -19,6 +19,7 @@
    #include <linux/uaccess.h> /* faulthandler_disabled() */
    #include <linux/efi.h> /* efi_crash_gracefully_on_page_fault()*/
    #include <linux/mm_types.h>
    +#include <linux/mm.h> /* find_and_lock_vma() */

    #include <asm/cpufeature.h> /* boot_cpu_has, ... */
    #include <asm/traps.h> /* dotraplinkage, ... */
    @@ -1354,6 +1355,38 @@ void do_user_addr_fault(struct pt_regs *regs,
    }
    #endif

    +#ifdef CONFIG_PER_VMA_LOCK
    + if (!(flags & FAULT_FLAG_USER) || atomic_read(&mm->mm_users) == 1)
    + goto lock_mmap;
    +
    + vma = lock_vma_under_rcu(mm, address);
    + if (!vma)
    + goto lock_mmap;
    +
    + if (unlikely(access_error(error_code, vma))) {
    + vma_read_unlock(vma);
    + goto lock_mmap;
    + }
    + fault = handle_mm_fault(vma, address, flags | FAULT_FLAG_VMA_LOCK, regs);
    + vma_read_unlock(vma);
    +
    + if (!(fault & VM_FAULT_RETRY)) {
    + count_vm_vma_lock_event(VMA_LOCK_SUCCESS);
    + goto done;
    + }
    + count_vm_vma_lock_event(VMA_LOCK_RETRY);
    +
    + /* Quick path to respond to signals */
    + if (fault_signal_pending(fault, regs)) {
    + if (!user_mode(regs))
    + kernelmode_fixup_or_oops(regs, error_code, address,
    + SIGBUS, BUS_ADRERR,
    + ARCH_DEFAULT_PKEY);
    + return;
    + }
    +lock_mmap:
    +#endif /* CONFIG_PER_VMA_LOCK */
    +
    /*
    * Kernel-mode access to the user address space should only occur
    * on well-defined single instructions listed in the exception
    @@ -1454,6 +1487,9 @@ void do_user_addr_fault(struct pt_regs *regs,
    }

    mmap_read_unlock(mm);
    +#ifdef CONFIG_PER_VMA_LOCK
    +done:
    +#endif
    if (likely(!(fault & VM_FAULT_ERROR)))
    return;

    --
    2.39.0
    \
     
     \ /
      Last update: 2023-03-26 23:33    [W:4.320 / U:0.016 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site