lkml.org 
[lkml]   [2010]   [Oct]   [18]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    Date
    From
    Subject[PATCH 20/20] mm: Optimize page_lock_anon_vma() fast-path
    Optimize the page_lock_anon_vma() fast path to be one LOCKed op,
    instead of two.

    Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
    ---
    mm/rmap.c | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----
    1 file changed, 67 insertions(+), 4 deletions(-)

    Index: linux-2.6/mm/rmap.c
    ===================================================================
    --- linux-2.6.orig/mm/rmap.c
    +++ linux-2.6/mm/rmap.c
    @@ -357,20 +357,75 @@ out:
    return anon_vma;
    }

    +/*
    + * Similar to page_get_anon_vma() except it locks the anon_vma.
    + *
    + * Its a little more complex as it tries to keep the fast path to a single
    + * atomic op -- the trylock. If we fail the trylock, we fall back to getting a
    + * reference like with page_get_anon_vma() and then block on the mutex.
    + */
    struct anon_vma *page_lock_anon_vma(struct page *page)
    {
    - struct anon_vma *anon_vma = page_get_anon_vma(page);
    + struct anon_vma *anon_vma = NULL;
    + unsigned long anon_mapping;
    +
    + rcu_read_lock();
    + anon_mapping = (unsigned long) ACCESS_ONCE(page->mapping);
    + if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
    + goto out;
    + if (!page_mapped(page))
    + goto out;
    +
    + anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);
    + if (mutex_trylock(&anon_vma->root->lock)) {
    + /*
    + * If we observe a !0 refcount, then holding the lock ensures
    + * the anon_vma will not go away, see __put_anon_vma().
    + */
    + if (!atomic_read(&anon_vma->refcount)) {
    + anon_vma_unlock(anon_vma);
    + anon_vma = NULL;
    + }
    + goto out;
    + }

    - if (anon_vma)
    - anon_vma_lock(anon_vma);
    + /* trylock failed, we got to sleep */
    + if (!atomic_inc_not_zero(&anon_vma->refcount)) {
    + anon_vma = NULL;
    + goto out;
    + }
    +
    + if (!page_mapped(page)) {
    + put_anon_vma(anon_vma);
    + anon_vma = NULL;
    + goto out;
    + }
    +
    + /* we pinned the anon_vma, its safe to sleep */
    + rcu_read_unlock();
    + anon_vma_lock(anon_vma);
    +
    + if (atomic_dec_and_test(&anon_vma->refcount)) {
    + /*
    + * Oops, we held the last refcount, release the lock
    + * and bail -- can't simply use put_anon_vma() because
    + * we'll deadlock on the anon_vma_lock() recursion.
    + */
    + anon_vma_unlock(anon_vma);
    + __put_anon_vma(anon_vma);
    + anon_vma = NULL;
    + }

    return anon_vma;
    +
    +out:
    + rcu_read_unlock();
    + return anon_vma;
    }

    void page_unlock_anon_vma(struct anon_vma *anon_vma)
    {
    anon_vma_unlock(anon_vma);
    - put_anon_vma(anon_vma);
    }

    /*
    @@ -1462,6 +1517,14 @@ int try_to_munlock(struct page *page)

    void __put_anon_vma(struct anon_vma *anon_vma)
    {
    + /*
    + * Synchronize against page_lock_anon_vma() such that
    + * we can safely hold the lock without the anon_vma getting
    + * freed.
    + */
    + anon_vma_lock(anon_vma);
    + anon_vma_unlock(anon_vma);
    +
    if (anon_vma->root != anon_vma)
    put_anon_vma(anon_vma->root);
    anon_vma_free(anon_vma);



    \
     
     \ /
      Last update: 2010-10-18 13:43    [W:0.023 / U:177.600 seconds]
    ©2003-2016 Jasper Spaans. hosted at Digital OceanAdvertise on this site