lkml.org 
[lkml]   [2010]   [May]   [17]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[PATCH 09/10] x86 rwsem: down_read_critical implementation
    Date
    The basic properties that have been true so far still hold:
    - RWSEM_ACTIVE_READ_BIAS & RWSEM_ACTIVE_MASK == 1
    - RWSEM_ACTIVE_WRITE_BIAS & RWSEM_ACTIVE_MASK == 1
    - RWSEM_WAITING_BIAS & RWSEM_ACTIVE_MASK == 0
    - RWSEM_ACTIVE_WRITE_BIAS < 0 and RWSEM_WAITING_BIAS < 0

    A new property is introduced: RWSEM_ACTIVE_WRITE_BIAS is set to be
    'more negative' than RWSEM_WAITING_BIAS. This way, even the first writer
    will set the rwsem count to be < RWSEM_WAITING_BIAS and down_read_critical()
    can make use of this to determine if there are any active writers.

    Signed-off-by: Michel Lespinasse <walken@google.com>
    ---
    arch/x86/include/asm/rwsem.h | 53 ++++++++++++++++++++++++++++++++---------
    arch/x86/lib/rwsem_64.S | 14 +++++++++-
    arch/x86/lib/semaphore_32.S | 21 +++++++++++++++-
    3 files changed, 72 insertions(+), 16 deletions(-)

    diff --git a/arch/x86/include/asm/rwsem.h b/arch/x86/include/asm/rwsem.h
    index 86e0473..fbb068c 100644
    --- a/arch/x86/include/asm/rwsem.h
    +++ b/arch/x86/include/asm/rwsem.h
    @@ -16,11 +16,10 @@
    * if there are writers (and maybe) readers waiting (in which case it goes to
    * sleep).
    *
    - * The value of WAITING_BIAS supports up to 32766 waiting processes. This can
    - * be extended to 65534 by manually checking the whole MSW rather than relying
    - * on the S flag.
    + * The WRITE_BIAS value supports up to 32767 processes simultaneously
    + * trying to acquire a write lock.
    *
    - * The value of ACTIVE_BIAS supports up to 65535 active processes.
    + * The value of ACTIVE_BIAS supports up to 32767 active processes.
    *
    * This should be totally fair - if anything is waiting, a process that wants a
    * lock will go to the back of the queue. When the currently active lock is
    @@ -48,6 +47,8 @@ struct rwsem_waiter;
    extern asmregparm struct rw_semaphore *
    rwsem_down_read_failed(struct rw_semaphore *sem);
    extern asmregparm struct rw_semaphore *
    + rwsem_down_read_unfair_failed(struct rw_semaphore *sem);
    +extern asmregparm struct rw_semaphore *
    rwsem_down_write_failed(struct rw_semaphore *sem);
    extern asmregparm struct rw_semaphore *
    rwsem_wake(struct rw_semaphore *);
    @@ -62,17 +63,22 @@ extern asmregparm struct rw_semaphore *
    * for 64 bits.
    */

    +
    #ifdef CONFIG_X86_64
    -# define RWSEM_ACTIVE_MASK 0xffffffffL
    +# define RWSEM_UNLOCKED_VALUE 0x0000000000000000L
    +# define RWSEM_ACTIVE_MASK 0x000000007fffffffL
    +# define RWSEM_ACTIVE_READ_BIAS 0x0000000000000001L
    +# define RWSEM_ACTIVE_WRITE_BIAS 0xffffffff00000001L
    +# define RWSEM_WAITING_BIAS 0xffffffff80000000L
    #else
    -# define RWSEM_ACTIVE_MASK 0x0000ffffL
    +# define RWSEM_UNLOCKED_VALUE 0x00000000L
    +# define RWSEM_ACTIVE_MASK 0x00007fffL
    +# define RWSEM_ACTIVE_READ_BIAS 0x00000001L
    +# define RWSEM_ACTIVE_WRITE_BIAS 0xffff0001L
    +# define RWSEM_WAITING_BIAS 0xffff8000L
    #endif

    -#define RWSEM_UNLOCKED_VALUE 0x00000000L
    -#define RWSEM_ACTIVE_BIAS 0x00000001L
    -#define RWSEM_WAITING_BIAS (-RWSEM_ACTIVE_MASK-1)
    -#define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS
    -#define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)
    +#define __HAVE_DOWN_READ_UNFAIR

    typedef signed long rwsem_count_t;

    @@ -129,6 +135,28 @@ static inline void __down_read(struct rw_semaphore *sem)
    }

    /*
    + * lock for reading - skip waiting threads
    + */
    +static inline void __down_read_unfair(struct rw_semaphore *sem)
    +{
    + rwsem_count_t tmp;
    +
    + tmp = RWSEM_ACTIVE_READ_BIAS;
    + asm volatile("# beginning down_read_unfair\n\t"
    + LOCK_PREFIX " xadd %1,(%2)\n\t"
    + /* adds 0x00000001, returns the old value */
    + " cmp %4,%1\n\t"
    + /* was the count >= RWSEM_WAITING_BIAS before? */
    + " jge 1f\n"
    + " call call_rwsem_down_read_unfair_failed\n"
    + "1:\n"
    + "# ending down_read_unfair"
    + : "+m" (sem->count), "=r" (tmp)
    + : "a" (sem), "1" (tmp), "re" (RWSEM_WAITING_BIAS)
    + : "memory", "cc");
    +}
    +
    +/*
    * trylock for reading -- returns 1 if successful, 0 if contention
    */
    static inline int __down_read_trylock(struct rw_semaphore *sem)
    @@ -242,7 +270,8 @@ static inline void __downgrade_write(struct rw_semaphore *sem)
    "1:\n\t"
    "# ending __downgrade_write\n"
    : "+m" (sem->count)
    - : "a" (sem), "er" (-RWSEM_WAITING_BIAS)
    + : "a" (sem),
    + "er" (RWSEM_ACTIVE_READ_BIAS - RWSEM_ACTIVE_WRITE_BIAS)
    : "memory", "cc");
    }

    diff --git a/arch/x86/lib/rwsem_64.S b/arch/x86/lib/rwsem_64.S
    index 41fcf00..e99d187 100644
    --- a/arch/x86/lib/rwsem_64.S
    +++ b/arch/x86/lib/rwsem_64.S
    @@ -51,6 +51,16 @@ ENTRY(call_rwsem_down_read_failed)
    ret
    ENDPROC(call_rwsem_down_read_failed)

    +ENTRY(call_rwsem_down_read_unfair_failed)
    + save_common_regs
    + pushq %rdx
    + movq %rax,%rdi
    + call rwsem_down_read_unfair_failed
    + popq %rdx
    + restore_common_regs
    + ret
    + ENDPROC(call_rwsem_down_read_failed)
    +
    ENTRY(call_rwsem_down_write_failed)
    save_common_regs
    movq %rax,%rdi
    @@ -60,8 +70,8 @@ ENTRY(call_rwsem_down_write_failed)
    ENDPROC(call_rwsem_down_write_failed)

    ENTRY(call_rwsem_wake)
    - decl %edx /* do nothing if still outstanding active readers */
    - jnz 1f
    + cmpl $0x80000001, %edx
    + jne 1f /* do nothing unless there are waiters and no active threads */
    save_common_regs
    movq %rax,%rdi
    call rwsem_wake
    diff --git a/arch/x86/lib/semaphore_32.S b/arch/x86/lib/semaphore_32.S
    index 648fe47..fea9a53 100644
    --- a/arch/x86/lib/semaphore_32.S
    +++ b/arch/x86/lib/semaphore_32.S
    @@ -89,6 +89,23 @@ ENTRY(call_rwsem_down_read_failed)
    CFI_ENDPROC
    ENDPROC(call_rwsem_down_read_failed)

    +ENTRY(call_rwsem_down_read_unfair_failed)
    + CFI_STARTPROC
    + push %ecx
    + CFI_ADJUST_CFA_OFFSET 4
    + CFI_REL_OFFSET ecx,0
    + push %edx
    + CFI_ADJUST_CFA_OFFSET 4
    + CFI_REL_OFFSET edx,0
    + call rwsem_down_read_unfair_failed
    + pop %edx
    + CFI_ADJUST_CFA_OFFSET -4
    + pop %ecx
    + CFI_ADJUST_CFA_OFFSET -4
    + ret
    + CFI_ENDPROC
    + ENDPROC(call_rwsem_down_read_failed)
    +
    ENTRY(call_rwsem_down_write_failed)
    CFI_STARTPROC
    push %ecx
    @@ -103,8 +120,8 @@ ENTRY(call_rwsem_down_write_failed)

    ENTRY(call_rwsem_wake)
    CFI_STARTPROC
    - decw %dx /* do nothing if still outstanding active readers */
    - jnz 1f
    + cmpw $0x8001, %dx
    + jne 1f /* do nothing unless there are waiters and no active threads */
    push %ecx
    CFI_ADJUST_CFA_OFFSET 4
    CFI_REL_OFFSET ecx,0
    --
    1.7.0.1


    \
     
     \ /
      Last update: 2010-05-18 00:31    [W:0.028 / U:0.128 seconds]
    ©2003-2016 Jasper Spaans. hosted at Digital OceanAdvertise on this site