lkml.org 
[lkml]   [2002]   [Mar]   [1]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
Patches in this message
/
Date
From
Subject[Patch] Performance improvements with scalable statistics counters
Hello,
We did some measurements by modifying the loopback driver to use scalable
statistics counters, and the results follow...

Recall that this approach uses per-cpu versions for counters instead of a
shared global location; reducing counter cacheline bouncing thus
improving SMP performance. Particularly useful for frequently updated
statistics counters used in the kernel. The implementation uses a per-cpu
dynamic word allocator. For the UP case, the interfaces wrap onto the
usual global shared counter.

Results:
-------

1. Running tbench with 40 clients on loopback interface on 8 way smp.
(Throughput in MB/sec averaged for 10 runs)

Cpus Vanilla kernel Statctr mods to lo % improvement
---- ------------- ------------------ -------------
8 72.69243 78.45725 7.93
7 71.88495 73.35581 2.04
6 77.65202 78.95159 1.67
5 83.02133 90.67332 9.21
4 83.77254 84.41258 0.76
3 73.32989 74.73163 1.91
2 56.27168 57.20956 1.67

2. Kernprof PC sample profile count in loopback_xmit (This is the lo routine
which is modified to use scalable statistics counters). This was for 4 runs
of tbench 40 on the loopback interface
(kernprof -b -c all -d time -f 20000 -t pc)

Cpus Vanilla Kernel Statctr mods to lo % less time spent
---- ------------- ------------------ -----------------
8 217844 66648 69.40
7 136920 65836 51.91
6 134528 64739 51.87
5 131168 62214 52.56
4 119877 65441 45.40
3 123957 63022 49.15
2 125826 59127 53.00


You can find microbenchmark comparisons and interface/implementation details on
http://lse.sourceforge.net/counters/statctr.html

Measurements above were made on a 2.4.16 kernel. Find the statctr and lo
modification patches for 2.5.5 below.

Comments, suggestions welcome.

Regards,
Kiran

PS: Note that you cannot compile any other network driver if lo patch is
applied too..

Statctr Patch:
--------------

diff -ruN linux-2.5.5/include/linux/pcpu_alloc.h statctr-2.5.5/include/linux/pcpu_alloc.h
--- linux-2.5.5/include/linux/pcpu_alloc.h Thu Jan 1 05:30:00 1970
+++ statctr-2.5.5/include/linux/pcpu_alloc.h Fri Mar 1 11:38:16 2002
@@ -0,0 +1,75 @@
+/*
+ * Per-CPU Word allocator.
+ *
+ * Inspired by the freelist maintanance of the slab allocator implementation
+ * (in mm/slab.c)
+ *
+ * Copyright (c) International Business Machines Corp., 2001
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Author: Ravikiran Thirumalai <kiran@in.ibm.com>
+ *
+ * include/linux/pcpu_alloc.h
+ *
+ */
+#ifndef _LINUX_PCPU_ALLOC_H
+#define _LINUX_PCPU_ALLOC_H
+
+#ifdef __KERNEL__
+
+#include <linux/config.h>
+#include <linux/init.h>
+#include <linux/threads.h>
+#include <linux/sched.h>
+
+/* Prototypes */
+#ifdef CONFIG_SMP
+
+typedef struct pcpu_ctr_s {
+ void *arr[NR_CPUS]; /* Pcpu counter array */
+ void *blkp; /* Pointer to block from which ctr was
+ allocated from (for use with free code) */
+} pcpu_ctr_t;
+
+extern pcpu_ctr_t *pcpu_ctr_alloc(int);
+extern void pcpu_ctr_free(pcpu_ctr_t *);
+extern void __init pcpu_ctr_sys_init(void);
+
+#define PCPU_CTR(ctr, cpuid) ((unsigned long *)ctr->arr[cpuid])
+
+#else /* CONFIG_SMP */
+
+#include <linux/slab.h>
+
+typedef unsigned long pcpu_ctr_t;
+
+static inline pcpu_ctr_t *pcpu_ctr_alloc(int flags)
+{
+ return(kmalloc(sizeof(pcpu_ctr_t), flags));
+}
+
+static inline void pcpu_ctr_free(pcpu_ctr_t *ptr)
+{
+ kfree(ptr);
+}
+
+#define pcpu_ctr_sys_init() do { } while (0)
+
+#endif /* CONFIG_SMP */
+
+#endif /* __KERNEL__ */
+
+#endif /* _LINUX_PCPU_ALLOC_H */
diff -ruN linux-2.5.5/include/linux/statctr.h statctr-2.5.5/include/linux/statctr.h
--- linux-2.5.5/include/linux/statctr.h Thu Jan 1 05:30:00 1970
+++ statctr-2.5.5/include/linux/statctr.h Fri Mar 1 12:22:27 2002
@@ -0,0 +1,171 @@
+/*
+ * Scalable Statistics Counters.
+ *
+ * Visit http://lse.sourceforge.net/counters for detailed explanation of
+ * Scalable Statistic Counters
+ *
+ * Copyright (c) International Business Machines Corp., 2001
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Author: Ravikiran Thirumalai <kiran@in.ibm.com>
+ *
+ * include/linux/statctr.h
+ *
+ */
+
+#if !defined(_LINUX_STATCTR_H)
+#define _LINUX_STATCTR_H
+
+#if defined(__KERNEL__)
+
+#include <linux/config.h>
+#include <linux/init.h>
+#include <linux/sched.h>
+#include <linux/pcpu_alloc.h>
+
+#ifdef CONFIG_SMP
+typedef struct {
+ pcpu_ctr_t *ctr;
+} statctr_t;
+
+#else
+typedef unsigned long statctr_t;
+#endif
+
+
+/* prototypes */
+#ifdef CONFIG_SMP
+extern int statctr_init(statctr_t *, unsigned long);
+extern void statctr_cleanup(statctr_t *);
+extern int statctr_ninit(statctr_t *, unsigned long, int);
+extern void statctr_ncleanup(statctr_t *, int);
+#else
+#define statctr_init(stctr, val) ({*(stctr) = (val); 0;})
+#define statctr_cleanup(stctr) do { } while (0)
+
+static inline int
+statctr_ninit(statctr_t *stctr, unsigned long val, int no)
+{
+ int i;
+ for(i=0; i < no; i++) {
+ *(stctr) = val;
+ stctr++;
+ }
+ return 0;
+}
+
+#define statctr_ncleanup(st, bn) do { } while (0)
+
+#endif /* CONFIG_SMP */
+
+/* inlines */
+#ifdef CONFIG_SMP
+/**
+ * statctr_inc - Increment the statistics counter by one.
+ * @stctr: Statistics counter
+ *
+ * Increments the counter by one. Internally only the per-cpu counter is
+ * incremented.
+ */
+
+static inline void statctr_inc(statctr_t *stctr)
+{
+ (*PCPU_CTR(stctr->ctr, smp_processor_id()))++;
+}
+
+/**
+ * statctr_dec - Deccrement the statistics counter by one.
+ * @stctr: Statistics counter
+ *
+ * Decrements the counter by one. Internally only the per-cpu counter is
+ * incremented.
+ */
+
+static inline void statctr_dec(statctr_t *stctr)
+{
+ (*PCPU_CTR(stctr->ctr, smp_processor_id()))--;
+}
+
+/**
+ * statctr_set - Set the statistics counter to value passed.
+ * @stctr: Statistcs counter
+ * @val: Value to be set..
+ *
+ * Sets the statistics counter. If statctr_read() is invoked after a counter
+ * is set, return value of statctr_read shud reflect the value set.
+ */
+
+static inline void statctr_set(statctr_t *stctr, unsigned long val)
+{
+ int i;
+ *PCPU_CTR(stctr->ctr, 0) = val;
+ for (i=1; i < smp_num_cpus; i++) {
+ *PCPU_CTR(stctr->ctr, i) = 0;
+ }
+}
+
+/**
+ * statctr_read - Returns the counter value.
+ * @stctr: Statistics counter
+ *
+ * Reads all of the other per-cpu versions of this counter, consolidates them
+ * and returns to the caller.
+ */
+
+static inline long statctr_read(statctr_t *stctr)
+{
+ int i;
+ unsigned long res = 0;
+ for( i=0; i < smp_num_cpus; i++ )
+ res += *PCPU_CTR(stctr->ctr, i);
+ return res;
+}
+
+/**
+ * statctr_add - Adds the passed val to the counter value.
+ * @stctr: Statistics counter
+ * @val: Addend
+ *
+ */
+
+static inline void statctr_add(statctr_t *stctr, unsigned long val)
+{
+ *PCPU_CTR(stctr->ctr, smp_processor_id()) += val;
+}
+
+/**
+ * statctr_sub - Subtracts the passed val from the counter value.
+ * @stctr: Statistics counter
+ * @val: Subtrahend
+ *
+ */
+
+static inline void statctr_sub(statctr_t *stctr, unsigned long val)
+{
+ *PCPU_CTR(stctr->ctr, smp_processor_id()) -= val;
+}
+#else /* CONFIG_SMP */
+#define statctr_inc(stctr) ((*(stctr))++)
+#define statctr_dec(stctr) ((*(stctr))--)
+#define statctr_read(stctr) (*stctr)
+#define statctr_set(stctr,val) (*(stctr) = (val))
+#define statctr_add(stctr,val) ((*(stctr))+=(val))
+#define statctr_sub(stctr,val) ((*(stctr))-=(val))
+#endif
+
+#endif /* __KERNEL__ */
+
+#endif /* _LINUX_STATCTR_H */
diff -ruN linux-2.5.5/init/main.c statctr-2.5.5/init/main.c
--- linux-2.5.5/init/main.c Wed Feb 20 07:40:56 2002
+++ statctr-2.5.5/init/main.c Fri Mar 1 11:38:16 2002
@@ -27,6 +27,7 @@
#include <linux/iobuf.h>
#include <linux/bootmem.h>
#include <linux/tty.h>
+#include <linux/pcpu_alloc.h>

#include <asm/io.h>
#include <asm/bugs.h>
@@ -383,6 +384,7 @@
* make syscalls (and thus be locked).
*/
smp_init();
+ pcpu_ctr_sys_init();

/* Do the rest non-__init'ed, we're now alive */
rest_init();
diff -ruN linux-2.5.5/kernel/Makefile statctr-2.5.5/kernel/Makefile
--- linux-2.5.5/kernel/Makefile Wed Feb 20 07:40:57 2002
+++ statctr-2.5.5/kernel/Makefile Fri Mar 1 11:38:16 2002
@@ -10,7 +10,7 @@
O_TARGET := kernel.o

export-objs = signal.o sys.o kmod.o context.o ksyms.o pm.o exec_domain.o \
- printk.o
+ printk.o statctr.o

obj-y = sched.o dma.o fork.o exec_domain.o panic.o printk.o \
module.o exit.o itimer.o info.o time.o softirq.o resource.o \
@@ -20,6 +20,7 @@
obj-$(CONFIG_UID16) += uid16.o
obj-$(CONFIG_MODULES) += ksyms.o
obj-$(CONFIG_PM) += pm.o
+obj-$(CONFIG_SMP) += statctr.o

ifneq ($(CONFIG_IA64),y)
# According to Alan Modra <alan@linuxcare.com.au>, the -fno-omit-frame-pointer is
diff -ruN linux-2.5.5/kernel/statctr.c statctr-2.5.5/kernel/statctr.c
--- linux-2.5.5/kernel/statctr.c Thu Jan 1 05:30:00 1970
+++ statctr-2.5.5/kernel/statctr.c Fri Mar 1 12:13:36 2002
@@ -0,0 +1,104 @@
+/*
+ * Scalable Statistics Counters.
+ *
+ * Visit http://lse.sourceforge.net/counters for detailed explanation of
+ * Scalable Statistic Counters
+ *
+ * Copyright (c) International Business Machines Corp., 2001
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Author: Ravikiran Thirumalai <kiran@in.ibm.com>
+ *
+ * kernel/statctr.c
+ *
+ */
+
+#include <linux/pcpu_alloc.h>
+#include <linux/statctr.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+
+/**
+ * statctr_init - Sets up the statistics counter
+ * @ctr: Pointer to statctr_t type; counter to be initialised
+ * @val: Initialization value.
+ *
+ * Allocates memory to a statistics counter. Returns 0 on successful
+ * allocation and non zero otherwise
+ */
+int statctr_init(statctr_t *stctr, unsigned long val)
+{
+ stctr->ctr = pcpu_ctr_alloc(GFP_ATOMIC);
+ if(!stctr->ctr)
+ return -1;
+ statctr_set(stctr, val);
+ return 0;
+}
+
+/**
+ * statctr_cleanup - Perform cleanup functions on a statctr counter
+ * @ctr: Pointer to statctr_t type;
+ */
+void statctr_cleanup(statctr_t *stctr)
+{
+ pcpu_ctr_free(stctr->ctr);
+}
+
+/**
+ * statctr_ninit - Inits a bunch of contiguos statctrs
+ * @ctr: Ptr to first ctr to be inited in the bunch
+ * @val: Val to be set
+ * @no : No of ctrs to be inited
+ *
+ * Inits a bunch of contigious statctrs. Useful when statctrs
+ * are used as arrays or in "packed" structures.
+ * Returns non zero if inits fail
+ */
+int statctr_ninit(statctr_t *stctr, unsigned long val, int no)
+{
+ int i;
+ for(i=0; i < no; i++) {
+ if(statctr_init(&stctr[i], val))
+ goto cleanup;
+ }
+ return 0;
+
+cleanup:
+ i--;
+ for(; i >= 0; i--) {
+ statctr_cleanup(&stctr[i]);
+ }
+ return -1;
+}
+
+/**
+ * statctr_ncleanup - cleans up a bunch of contiguos statctrs
+ * @ctr: Ptr to first ctr to be inited in the bunch
+ * @no : No of ctrs to be inited
+ *
+ */
+void statctr_ncleanup(statctr_t *stctr, int no)
+{
+ int i;
+ for(i=0; i < no; i++)
+ statctr_cleanup(&stctr[i]);
+}
+
+EXPORT_SYMBOL(statctr_init);
+EXPORT_SYMBOL(statctr_cleanup);
+EXPORT_SYMBOL(statctr_ninit);
+EXPORT_SYMBOL(statctr_ncleanup);
+
diff -ruN linux-2.5.5/mm/Makefile statctr-2.5.5/mm/Makefile
--- linux-2.5.5/mm/Makefile Wed Feb 20 07:41:04 2002
+++ statctr-2.5.5/mm/Makefile Fri Mar 1 11:38:16 2002
@@ -9,11 +9,12 @@

O_TARGET := mm.o

-export-objs := shmem.o filemap.o mempool.o page_alloc.o
+export-objs := shmem.o filemap.o mempool.o page_alloc.o pcpu_alloc.o

obj-y := memory.o mmap.o filemap.o mprotect.o mlock.o mremap.o \
vmalloc.o slab.o bootmem.o swap.o vmscan.o page_io.o \
page_alloc.o swap_state.o swapfile.o numa.o oom_kill.o \
shmem.o highmem.o mempool.o

+obj-$(CONFIG_SMP) += pcpu_alloc.o
include $(TOPDIR)/Rules.make
diff -ruN linux-2.5.5/mm/pcpu_alloc.c statctr-2.5.5/mm/pcpu_alloc.c
--- linux-2.5.5/mm/pcpu_alloc.c Thu Jan 1 05:30:00 1970
+++ statctr-2.5.5/mm/pcpu_alloc.c Fri Mar 1 11:38:16 2002
@@ -0,0 +1,346 @@
+/*
+ * Per-CPU Counter allocator.
+ *
+ * Inspired by the freelist maintanance of the slab allocator implementation
+ * (in mm/slab.c)
+ *
+ * Copyright (c) International Business Machines Corp., 2001
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Author: Ravikiran Thirumalai <kiran@in.ibm.com>
+ *
+ * mm/pcpu_alloc.c
+ *
+ */
+
+#include <linux/pcpu_alloc.h>
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/cache.h>
+
+#include <asm/hardirq.h>
+
+#define PCPU_CTR_SIZE (sizeof(unsigned long))
+#define PCPU_CTR_LINE_SIZE (SMP_CACHE_BYTES)
+#define PCPU_CTR_OBJS_IN_BLK (PCPU_CTR_LINE_SIZE / PCPU_CTR_SIZE)
+
+#define PCPU_CTR_CACHE_NAME "pcpuctr"
+
+/*
+ * The block control structure for pcpu_ctr_t objects ...
+ */
+struct pcpu_ctr_blk {
+ void *lineaddr[NR_CPUS]; /* Array of pointers to
+ per cpu cachelines */
+ struct list_head linkage; /* Linkage for pcpu_ctr_blk s */
+ unsigned int usecount; /* To decide when to free this
+ block */
+
+ /* Free list maintanance for objs on blk */
+ int freearr[PCPU_CTR_OBJS_IN_BLK];
+ /* Array linked list as a per
+ block freelist.. */
+ int freehead; /* Start of list ..*/
+};
+
+/*
+ * Main Control structure for pcpu_ctr_t objects ...
+ */
+struct pcpu_ctr_ctl {
+ struct list_head blks; /* Head ptr of the list of all
+ blocks in the pcpu_ctr
+ "cache".
+ Full, partial first then
+ free..*/
+ struct list_head *firstnotfull;
+ spinlock_t lock;
+ kmem_cache_t *cachep; /* Cache to alloc per cpu
+ cachelines from */
+};
+
+static struct pcpu_ctr_ctl pcpu_ctr_ctl = {
+ blks: LIST_HEAD_INIT(pcpu_ctr_ctl.blks),
+ firstnotfull: &pcpu_ctr_ctl.blks,
+ lock: SPIN_LOCK_UNLOCKED,
+};
+
+/**
+ * Allocate a block descriptor structure and initialize it. Returns addr of
+ * block descriptor on success
+ */
+static struct pcpu_ctr_blk
+*pcpu_ctr_blkctl_alloc(struct pcpu_ctr_ctl *ctl, int flags)
+{
+ struct pcpu_ctr_blk *blkp;
+ int i;
+ if (!(blkp = (struct pcpu_ctr_blk *)
+ kmalloc(sizeof(struct pcpu_ctr_blk), flags)))
+ return blkp;
+ blkp->usecount = 0;
+
+ blkp->freehead = 0;
+ for (i=0; i < PCPU_CTR_OBJS_IN_BLK; i++)
+ blkp->freearr[i] = i+1;
+ blkp->freearr[i-1] = -1; /* Marks the end of the array */
+
+ return blkp;
+}
+
+/**
+ * Frees the block descriptor structure
+ */
+static void pcpu_ctr_blkctl_free(struct pcpu_ctr_blk *blkp)
+{
+ kfree(blkp);
+}
+
+/**
+ * Add a "block" to the pcpu_ctr object memory pool. Returns 0 on failure and
+ * 1 on success
+ */
+static int pcpu_ctr_mem_grow(struct pcpu_ctr_ctl *ctl, int flags)
+{
+ void *addr;
+ struct pcpu_ctr_blk *blkp;
+ unsigned int save_flags;
+ int i;
+
+ if (!(blkp = pcpu_ctr_blkctl_alloc(ctl, flags)))
+ return 0;
+
+ /* Get per cpu cache lines for the block */
+ for (i=0; i < smp_num_cpus; i++) {
+ blkp->lineaddr[i] = kmem_cache_alloc(ctl->cachep, flags);
+ if(!(blkp->lineaddr[i]))
+ goto exit1;
+ memset(blkp->lineaddr[i], 0, PCPU_CTR_LINE_SIZE);
+ }
+
+ /* Now that we have the block successfully allocated and instantiated..
+ add it.....*/
+ spin_lock_irqsave(&ctl->lock, save_flags);
+ list_add_tail(&blkp->linkage, &ctl->blks);
+ if (ctl->firstnotfull == &ctl->blks)
+ ctl->firstnotfull = &blkp->linkage;
+ spin_unlock_irqrestore(&ctl->lock, save_flags);
+ return 1;
+
+exit1:
+ /* Free cachelines allocated to this block, and the blk struct..*/
+ i--;
+ for (i; i >= 0; i--)
+ kmem_cache_free(ctl->cachep, blkp->lineaddr[i]);
+ pcpu_ctr_blkctl_free(blkp);
+ return 0;
+}
+
+/**
+ * Initialise the main pcpu_ctr_ctl control structure.
+ */
+static void pcpu_ctr_init(struct pcpu_ctr_ctl * ctl)
+{
+ /*
+ * Create cache to serve out cache line sized, cache line
+ * aligned chunks..
+ */
+ ctl->cachep = kmem_cache_create(PCPU_CTR_CACHE_NAME,
+ PCPU_CTR_LINE_SIZE, 0, SLAB_HWCACHE_ALIGN,
+ NULL, NULL);
+ if(!ctl->cachep)
+ BUG();
+}
+
+/**
+ * Initialisation - setup the pcpu_ctr main control structures (pcpu_ctr_ctl)
+ */
+void __init pcpu_ctr_sys_init(void)
+{
+ pcpu_ctr_init(&pcpu_ctr_ctl);
+ if( !pcpu_ctr_mem_grow(&pcpu_ctr_ctl, GFP_ATOMIC))
+ BUG();
+}
+
+/**
+ * Allocs an object from the block. Returns back the object index.
+ */
+static unsigned int
+__pcpu_ctr_alloc_one(struct pcpu_ctr_ctl *ctl, struct pcpu_ctr_blk *blkp)
+{
+ void *objp;
+ unsigned int objidx;
+
+ objidx = blkp->freehead;
+ blkp->freehead = blkp->freearr[objidx];
+ blkp->usecount++;
+ if(blkp->freehead < 0) {
+ /* Last obj allocated. So if this is still first not full,
+ change it */
+ ctl->firstnotfull = blkp->linkage.next;
+ }
+
+ return objidx;
+}
+
+/**
+ * __pcpu_ctr_alloc - Allocate a Per CPU counter
+ */
+static pcpu_ctr_t *__pcpu_ctr_alloc(struct pcpu_ctr_ctl *ctl, int flags)
+{
+ pcpu_ctr_t *ctr;
+ struct pcpu_ctr_blk *blkp;
+ unsigned int save_flags;
+ struct list_head *l;
+ unsigned int objidx;
+ int i;
+
+ ctr = kmalloc(sizeof(pcpu_ctr_t), flags);
+ if (ctr == NULL)
+ return NULL;
+
+tryagain:
+
+ /* Get the block to allocate pcpu_ctr from.. */
+ spin_lock_irqsave(&ctl->lock, save_flags);
+ l = ctl->firstnotfull;
+ if (l == &ctl->blks)
+ goto unlock_and_get_mem;
+ blkp = list_entry(l, struct pcpu_ctr_blk, linkage);
+
+ /* Get the object index from the block */
+ objidx = __pcpu_ctr_alloc_one(ctl, blkp);
+ spin_unlock_irqrestore(&ctl->lock, save_flags);
+ /* Since we hold the lock and firstnotfull is not the
+ head list, we shud be getting an object alloc here.firstnotfull
+ can be pointing to head of the list when all the blks are
+ full or when there're no blocks left */
+ for (i=0; i < smp_num_cpus; i++)
+ ctr->arr[i] = blkp->lineaddr[i] + objidx*PCPU_CTR_SIZE;
+ ctr->blkp = (void *)blkp;
+ return ctr;
+
+unlock_and_get_mem:
+
+ spin_unlock_irqrestore(&ctl->lock, save_flags);
+ if(pcpu_ctr_mem_grow(ctl, flags))
+ goto tryagain; /* added another block..try allocing obj ..*/
+
+ /* Unable to get mem for blocks so ret after cleanup.. */
+ kfree(ctr);
+ return NULL;
+}
+
+/**
+ * pcpu_ctr_alloc - Allocate a PCPU ctr.
+ * @flags: Type of memory to allocate
+ * Can take in all flags kmem_cache_alloc takes in as args.
+ *
+ * Allocates memory to a PCPU counter. Returns NULL on failure.
+ */
+pcpu_ctr_t *pcpu_ctr_alloc(int flags)
+{
+ return( __pcpu_ctr_alloc(&pcpu_ctr_ctl, flags));
+}
+
+/**
+ * pcpu_ctr_blk_destroy - Frees memory associated with a pcpu_ctr block
+ */
+static void
+pcpu_ctr_blk_destroy(struct pcpu_ctr_ctl *ctl, struct pcpu_ctr_blk *blkp)
+{
+ int i;
+
+ /* Return cache line aligned ctr lines back to slab cache */
+ for (i=0; i< smp_num_cpus; i++)
+ kmem_cache_free(ctl->cachep, blkp->lineaddr[i]);
+
+ pcpu_ctr_blkctl_free(blkp);
+}
+
+/**
+ * Frees an object from a block and fixes the freelist accdly.
+ * Frees the slab cache memory if a block gets empty during free.
+ */
+static void __pcpu_ctr_free(struct pcpu_ctr_ctl *ctl, pcpu_ctr_t *ptr)
+{
+ struct pcpu_ctr_blk *blkp;
+ unsigned int objidx;
+ unsigned int objoffset;
+ struct list_head *t;
+ int cpuid = smp_processor_id();
+ unsigned int save_flags;
+
+ spin_lock_irqsave(&ctl->lock, save_flags);
+
+ blkp = (struct pcpu_ctr_blk *)ptr->blkp;
+ objoffset = (unsigned int)(ptr->arr[cpuid] - blkp->lineaddr[cpuid]);
+ objidx = objoffset / PCPU_CTR_SIZE;
+
+ kfree(ptr);
+
+ /* Update the block freelist */
+ blkp->freearr[objidx] = blkp->freehead;
+ blkp->freehead = objidx;
+
+ /* Update usage count */
+ blkp->usecount--;
+
+ /* Fix block freelist chain */
+ if (blkp->freearr[objidx] < 0) {
+ /* block was previously full and is now just partially full ..
+ so make firstnotfull pt to this block and fix list accdly */
+ t = ctl->firstnotfull;
+ ctl->firstnotfull = &blkp->linkage;
+ if (blkp->linkage.next == t) {
+ spin_unlock_irqrestore(&ctl->lock, save_flags);
+ return;
+ }
+ list_del(&blkp->linkage);
+ list_add_tail(&blkp->linkage, t);
+
+ spin_unlock_irqrestore(&ctl->lock, save_flags);
+ return;
+ }
+
+ if (blkp->usecount == 0) {
+ /* Block now empty so give mem back to the slab cache */
+ t = ctl->firstnotfull->prev;
+
+ list_del(&blkp->linkage);
+ if (ctl->firstnotfull == &blkp->linkage)
+ ctl->firstnotfull = t->next;
+
+ spin_unlock_irqrestore(&ctl->lock, save_flags);
+ pcpu_ctr_blk_destroy(ctl, blkp);
+ return;
+ }
+
+ spin_unlock_irqrestore(&ctl->lock, save_flags);
+ return;
+}
+
+/**
+ * pcpu_ctr_free - Frees up a PCPU ctr from memory
+ * @ptr: Pointer to pcpu_ctr_t;
+ */
+void pcpu_ctr_free(pcpu_ctr_t *ptr)
+{
+ __pcpu_ctr_free(&pcpu_ctr_ctl, ptr);
+}
+
+EXPORT_SYMBOL(pcpu_ctr_alloc);
+EXPORT_SYMBOL(pcpu_ctr_free);

lo driver modifications:
------------------------
diff -ruN linux-2.5.5/drivers/net/loopback.c statctrusagelo/drivers/net/loopback.c
--- linux-2.5.5/drivers/net/loopback.c Wed Feb 20 07:40:56 2002
+++ statctrusagelo/drivers/net/loopback.c Fri Mar 1 16:11:35 2002
@@ -87,10 +87,10 @@
#endif

dev->last_rx = jiffies;
- stats->rx_bytes+=skb->len;
- stats->tx_bytes+=skb->len;
- stats->rx_packets++;
- stats->tx_packets++;
+ statctr_add(&stats->rx_bytes, skb->len);
+ statctr_add(&stats->tx_bytes, skb->len);
+ statctr_inc(&stats->rx_packets);
+ statctr_inc(&stats->tx_packets);

netif_rx(skb);

@@ -120,7 +120,11 @@
dev->priv = kmalloc(sizeof(struct net_device_stats), GFP_KERNEL);
if (dev->priv == NULL)
return -ENOMEM;
- memset(dev->priv, 0, sizeof(struct net_device_stats));
+ if (statctr_ninit((statctr_t *)dev->priv, 0, 23)) {
+ /* Init failed cleanup.. */
+ kfree(dev->priv);
+ return -ENOMEM;
+ }
dev->get_stats = get_stats;

/*
diff -ruN linux-2.5.5/include/linux/netdevice.h statctrusagelo/include/linux/netdevice.h
--- linux-2.5.5/include/linux/netdevice.h Wed Feb 20 07:41:05 2002
+++ statctrusagelo/include/linux/netdevice.h Fri Mar 1 16:11:35 2002
@@ -28,6 +28,7 @@
#include <linux/if.h>
#include <linux/if_ether.h>
#include <linux/if_packet.h>
+#include <linux/statctr.h>

#include <asm/atomic.h>
#include <asm/cache.h>
@@ -95,35 +96,35 @@

struct net_device_stats
{
- unsigned long rx_packets; /* total packets received */
- unsigned long tx_packets; /* total packets transmitted */
- unsigned long rx_bytes; /* total bytes received */
- unsigned long tx_bytes; /* total bytes transmitted */
- unsigned long rx_errors; /* bad packets received */
- unsigned long tx_errors; /* packet transmit problems */
- unsigned long rx_dropped; /* no space in linux buffers */
- unsigned long tx_dropped; /* no space available in linux */
- unsigned long multicast; /* multicast packets received */
- unsigned long collisions;
+ statctr_t rx_packets; /* total packets received */
+ statctr_t tx_packets; /* total packets transmitted */
+ statctr_t rx_bytes; /* total bytes received */
+ statctr_t tx_bytes; /* total bytes transmitted */
+ statctr_t rx_errors; /* bad packets received */
+ statctr_t tx_errors; /* packet transmit problems */
+ statctr_t rx_dropped; /* no space in linux buffers */
+ statctr_t tx_dropped; /* no space available in linux */
+ statctr_t multicast; /* multicast packets received */
+ statctr_t collisions;

/* detailed rx_errors: */
- unsigned long rx_length_errors;
- unsigned long rx_over_errors; /* receiver ring buff overflow */
- unsigned long rx_crc_errors; /* recved pkt with crc error */
- unsigned long rx_frame_errors; /* recv'd frame alignment error */
- unsigned long rx_fifo_errors; /* recv'r fifo overrun */
- unsigned long rx_missed_errors; /* receiver missed packet */
+ statctr_t rx_length_errors;
+ statctr_t rx_over_errors; /* receiver ring buff overflow */
+ statctr_t rx_crc_errors; /* recved pkt with crc error */
+ statctr_t rx_frame_errors; /* recv'd frame alignment error */
+ statctr_t rx_fifo_errors; /* recv'r fifo overrun */
+ statctr_t rx_missed_errors; /* receiver missed packet */

/* detailed tx_errors */
- unsigned long tx_aborted_errors;
- unsigned long tx_carrier_errors;
- unsigned long tx_fifo_errors;
- unsigned long tx_heartbeat_errors;
- unsigned long tx_window_errors;
+ statctr_t tx_aborted_errors;
+ statctr_t tx_carrier_errors;
+ statctr_t tx_fifo_errors;
+ statctr_t tx_heartbeat_errors;
+ statctr_t tx_window_errors;

/* for cslip etc */
- unsigned long rx_compressed;
- unsigned long tx_compressed;
+ statctr_t rx_compressed;
+ statctr_t tx_compressed;
};


diff -ruN linux-2.5.5/net/core/dev.c statctrusagelo/net/core/dev.c
--- linux-2.5.5/net/core/dev.c Wed Feb 20 07:41:03 2002
+++ statctrusagelo/net/core/dev.c Fri Mar 1 16:11:35 2002
@@ -1691,19 +1691,19 @@
if (stats)
size = sprintf(buffer, "%6s:%8lu %7lu %4lu %4lu %4lu %5lu %10lu %9lu %8lu %7lu %4lu %4lu %4lu %5lu %7lu %10lu\n",
dev->name,
- stats->rx_bytes,
- stats->rx_packets, stats->rx_errors,
- stats->rx_dropped + stats->rx_missed_errors,
- stats->rx_fifo_errors,
- stats->rx_length_errors + stats->rx_over_errors
- + stats->rx_crc_errors + stats->rx_frame_errors,
- stats->rx_compressed, stats->multicast,
- stats->tx_bytes,
- stats->tx_packets, stats->tx_errors, stats->tx_dropped,
- stats->tx_fifo_errors, stats->collisions,
- stats->tx_carrier_errors + stats->tx_aborted_errors
- + stats->tx_window_errors + stats->tx_heartbeat_errors,
- stats->tx_compressed);
+ statctr_read(&stats->rx_bytes),
+ statctr_read(&stats->rx_packets), statctr_read(&stats->rx_errors),
+ statctr_read(&stats->rx_dropped) + statctr_read(&stats->rx_missed_errors),
+ statctr_read(&stats->rx_fifo_errors),
+ statctr_read(&stats->rx_length_errors) + statctr_read(&stats->rx_over_errors)
+ + statctr_read(&stats->rx_crc_errors) + statctr_read(&stats->rx_frame_errors),
+ statctr_read(&stats->rx_compressed), statctr_read(&stats->multicast),
+ statctr_read(&stats->tx_bytes),
+ statctr_read(&stats->tx_packets), statctr_read(&stats->tx_errors), statctr_read(&stats->tx_dropped),
+ statctr_read(&stats->tx_fifo_errors), statctr_read(&stats->collisions),
+ statctr_read(&stats->tx_carrier_errors) + statctr_read(&stats->tx_aborted_errors)
+ + statctr_read(&stats->tx_window_errors) + statctr_read(&stats->tx_heartbeat_errors),
+ statctr_read(&stats->tx_compressed));
else
size = sprintf(buffer, "%6s: No statistics available.\n", dev->name);

-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
\
 
 \ /
  Last update: 2005-03-22 13:21    [W:2.475 / U:0.036 seconds]
©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site