lkml.org 
[lkml]   [2007]   [Oct]   [31]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    Date
    From
    Subject[patch 7/7] SLUB: Optimize per cpu access on the local cpu using this_cpu_ptr()
    Use this_cpu_ptr to optimize access to the per cpu area in the fastpaths.

    Signed-off-by: Christoph Lameter <clameter@sgi.com>

    ---
    mm/slub.c | 27 +++++++++++++++++++--------
    1 file changed, 19 insertions(+), 8 deletions(-)

    Index: linux-2.6/mm/slub.c
    ===================================================================
    --- linux-2.6.orig/mm/slub.c 2007-10-31 14:00:28.635673087 -0700
    +++ linux-2.6/mm/slub.c 2007-10-31 14:01:29.803422492 -0700
    @@ -248,6 +248,15 @@ static inline struct kmem_cache_cpu *get
    #endif
    }

    +static inline struct kmem_cache_cpu *this_cpu_slab(struct kmem_cache *s)
    +{
    +#ifdef CONFIG_SMP
    + return this_cpu_ptr(s->cpu_slab);
    +#else
    + return &s->cpu_slab;
    +#endif
    +}
    +
    /*
    * The end pointer in a slab is special. It points to the first object in the
    * slab but has bit 0 set to mark it.
    @@ -1521,7 +1530,7 @@ static noinline unsigned long get_new_sl
    if (!page)
    return 0;

    - *pc = c = get_cpu_slab(s, smp_processor_id());
    + *pc = c = this_cpu_slab(s);
    if (c->page) {
    /*
    * Someone else populated the cpu_slab while we
    @@ -1650,25 +1659,26 @@ static void __always_inline *slab_alloc(
    struct kmem_cache_cpu *c;

    #ifdef CONFIG_FAST_CMPXCHG_LOCAL
    - c = get_cpu_slab(s, get_cpu());
    + preempt_disable();
    + c = this_cpu_slab(s);
    do {
    object = c->freelist;
    if (unlikely(is_end(object) || !node_match(c, node))) {
    object = __slab_alloc(s, gfpflags, node, addr, c);
    if (unlikely(!object)) {
    - put_cpu();
    + preempt_enable();
    goto out;
    }
    break;
    }
    } while (cmpxchg_local(&c->freelist, object,
    get_freepointer(s, object)) != object);
    - put_cpu();
    + preempt_enable();
    #else
    unsigned long flags;

    local_irq_save(flags);
    - c = get_cpu_slab(s, smp_processor_id());
    + c = this_cpu_slab(s);
    if (unlikely((is_end(c->freelist)) || !node_match(c, node))) {

    object = __slab_alloc(s, gfpflags, node, addr, c);
    @@ -1794,7 +1804,8 @@ static void __always_inline slab_free(st
    #ifdef CONFIG_FAST_CMPXCHG_LOCAL
    void **freelist;

    - c = get_cpu_slab(s, get_cpu());
    + preempt_disable();
    + c = this_cpu_slab(s);
    debug_check_no_locks_freed(object, s->objsize);
    do {
    freelist = c->freelist;
    @@ -1816,13 +1827,13 @@ static void __always_inline slab_free(st
    }
    set_freepointer(s, object, freelist);
    } while (cmpxchg_local(&c->freelist, freelist, object) != freelist);
    - put_cpu();
    + preempt_enable();
    #else
    unsigned long flags;

    local_irq_save(flags);
    debug_check_no_locks_freed(object, s->objsize);
    - c = get_cpu_slab(s, smp_processor_id());
    + c = this_cpu_slab(s);
    if (likely(page == c->page && c->node >= 0)) {
    set_freepointer(s, object, c->freelist);
    c->freelist = object;
    --
    -
    To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
    the body of a message to majordomo@vger.kernel.org
    More majordomo info at http://vger.kernel.org/majordomo-info.html
    Please read the FAQ at http://www.tux.org/lkml/

    \
     
     \ /
      Last update: 2007-11-01 01:09    [W:2.956 / U:0.040 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site