lkml.org 
[lkml]   [2019]   [May]   [9]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[PATCH 5.1 30/30] arm64: futex: Bound number of LDXR/STXR loops in FUTEX_WAKE_OP
    Date
    From: Will Deacon <will.deacon@arm.com>

    commit 03110a5cb2161690ae5ac04994d47ed0cd6cef75 upstream.

    Our futex implementation makes use of LDXR/STXR loops to perform atomic
    updates to user memory from atomic context. This can lead to latency
    problems if we end up spinning around the LL/SC sequence at the expense
    of doing something useful.

    Rework our futex atomic operations so that we return -EAGAIN if we fail
    to update the futex word after 128 attempts. The core futex code will
    reschedule if necessary and we'll try again later.

    Cc: <stable@kernel.org>
    Fixes: 6170a97460db ("arm64: Atomic operations")
    Signed-off-by: Will Deacon <will.deacon@arm.com>
    Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

    ---
    arch/arm64/include/asm/futex.h | 55 +++++++++++++++++++++++++----------------
    1 file changed, 34 insertions(+), 21 deletions(-)

    --- a/arch/arm64/include/asm/futex.h
    +++ b/arch/arm64/include/asm/futex.h
    @@ -23,26 +23,34 @@

    #include <asm/errno.h>

    +#define FUTEX_MAX_LOOPS 128 /* What's the largest number you can think of? */
    +
    #define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg) \
    do { \
    + unsigned int loops = FUTEX_MAX_LOOPS; \
    + \
    uaccess_enable(); \
    asm volatile( \
    " prfm pstl1strm, %2\n" \
    "1: ldxr %w1, %2\n" \
    insn "\n" \
    "2: stlxr %w0, %w3, %2\n" \
    -" cbnz %w0, 1b\n" \
    -" dmb ish\n" \
    +" cbz %w0, 3f\n" \
    +" sub %w4, %w4, %w0\n" \
    +" cbnz %w4, 1b\n" \
    +" mov %w0, %w7\n" \
    "3:\n" \
    +" dmb ish\n" \
    " .pushsection .fixup,\"ax\"\n" \
    " .align 2\n" \
    -"4: mov %w0, %w5\n" \
    +"4: mov %w0, %w6\n" \
    " b 3b\n" \
    " .popsection\n" \
    _ASM_EXTABLE(1b, 4b) \
    _ASM_EXTABLE(2b, 4b) \
    - : "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp) \
    - : "r" (oparg), "Ir" (-EFAULT) \
    + : "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp), \
    + "+r" (loops) \
    + : "r" (oparg), "Ir" (-EFAULT), "Ir" (-EAGAIN) \
    : "memory"); \
    uaccess_disable(); \
    } while (0)
    @@ -57,23 +65,23 @@ arch_futex_atomic_op_inuser(int op, int

    switch (op) {
    case FUTEX_OP_SET:
    - __futex_atomic_op("mov %w3, %w4",
    + __futex_atomic_op("mov %w3, %w5",
    ret, oldval, uaddr, tmp, oparg);
    break;
    case FUTEX_OP_ADD:
    - __futex_atomic_op("add %w3, %w1, %w4",
    + __futex_atomic_op("add %w3, %w1, %w5",
    ret, oldval, uaddr, tmp, oparg);
    break;
    case FUTEX_OP_OR:
    - __futex_atomic_op("orr %w3, %w1, %w4",
    + __futex_atomic_op("orr %w3, %w1, %w5",
    ret, oldval, uaddr, tmp, oparg);
    break;
    case FUTEX_OP_ANDN:
    - __futex_atomic_op("and %w3, %w1, %w4",
    + __futex_atomic_op("and %w3, %w1, %w5",
    ret, oldval, uaddr, tmp, ~oparg);
    break;
    case FUTEX_OP_XOR:
    - __futex_atomic_op("eor %w3, %w1, %w4",
    + __futex_atomic_op("eor %w3, %w1, %w5",
    ret, oldval, uaddr, tmp, oparg);
    break;
    default:
    @@ -93,6 +101,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval,
    u32 oldval, u32 newval)
    {
    int ret = 0;
    + unsigned int loops = FUTEX_MAX_LOOPS;
    u32 val, tmp;
    u32 __user *uaddr;

    @@ -104,20 +113,24 @@ futex_atomic_cmpxchg_inatomic(u32 *uval,
    asm volatile("// futex_atomic_cmpxchg_inatomic\n"
    " prfm pstl1strm, %2\n"
    "1: ldxr %w1, %2\n"
    -" sub %w3, %w1, %w4\n"
    -" cbnz %w3, 3f\n"
    -"2: stlxr %w3, %w5, %2\n"
    -" cbnz %w3, 1b\n"
    -" dmb ish\n"
    +" sub %w3, %w1, %w5\n"
    +" cbnz %w3, 4f\n"
    +"2: stlxr %w3, %w6, %2\n"
    +" cbz %w3, 3f\n"
    +" sub %w4, %w4, %w3\n"
    +" cbnz %w4, 1b\n"
    +" mov %w0, %w8\n"
    "3:\n"
    +" dmb ish\n"
    +"4:\n"
    " .pushsection .fixup,\"ax\"\n"
    -"4: mov %w0, %w6\n"
    -" b 3b\n"
    +"5: mov %w0, %w7\n"
    +" b 4b\n"
    " .popsection\n"
    - _ASM_EXTABLE(1b, 4b)
    - _ASM_EXTABLE(2b, 4b)
    - : "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp)
    - : "r" (oldval), "r" (newval), "Ir" (-EFAULT)
    + _ASM_EXTABLE(1b, 5b)
    + _ASM_EXTABLE(2b, 5b)
    + : "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp), "+r" (loops)
    + : "r" (oldval), "r" (newval), "Ir" (-EFAULT), "Ir" (-EAGAIN)
    : "memory");
    uaccess_disable();


    \
     
     \ /
      Last update: 2019-05-09 20:57    [W:4.066 / U:0.812 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site