Messages in this thread Patch in this message | | | Date | Thu, 09 Sep 2010 18:19:13 +0200 | From | Tejun Heo <> | Subject | [PATCH UPDATED 1/2] x86, percpu: Optimize this_cpu_ptr |
| |
Allow arches to implement __this_cpu_ptr, and provide an x86 version.
Before: movq $foo, %rax movq %gs:this_cpu_off, %rdx addq %rdx, %rax
After: movq $foo, %rax addq %gs:this_cpu_off, %rax
The benefit is doing it in one less instruction and not clobbering a temporary register.
tj: beefed up the comment a bit and renamed in-macro temp variable to match neighboring macros.
Signed-off-by: Brian Gerst <brgerst@gmail.com> Signed-off-by: Tejun Heo <tj@kernel.org> --- Applied both patches to the following branch. I updated the first patch slightly.
git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu.git for-next
Thank you.
arch/x86/include/asm/percpu.h | 14 ++++++++++++++ include/asm-generic/percpu.h | 9 +++++++-- 2 files changed, 21 insertions(+), 2 deletions(-)
Index: percpu/arch/x86/include/asm/percpu.h =================================================================== --- percpu.orig/arch/x86/include/asm/percpu.h +++ percpu/arch/x86/include/asm/percpu.h @@ -47,6 +47,20 @@ #ifdef CONFIG_SMP #define __percpu_arg(x) "%%"__stringify(__percpu_seg)":%P" #x #define __my_cpu_offset percpu_read(this_cpu_off) + +/* + * Compared to the generic __my_cpu_offset version, the following + * saves one instruction and avoids clobbering a temp register. + */ +#define __this_cpu_ptr(ptr) \ +({ \ + typeof(ptr) tcp_ptr__ = (ptr); \ + __verify_pcpu_ptr(ptr); \ + asm volatile("add " __percpu_arg(1) ", %0" \ + : "+r" (tcp_ptr__) \ + : "m" (this_cpu_off)); \ + tcp_ptr__; \ +}) #else #define __percpu_arg(x) "%P" #x #endif Index: percpu/include/asm-generic/percpu.h =================================================================== --- percpu.orig/include/asm-generic/percpu.h +++ percpu/include/asm-generic/percpu.h @@ -60,9 +60,14 @@ extern unsigned long __per_cpu_offset[NR #define __raw_get_cpu_var(var) \ (*SHIFT_PERCPU_PTR(&(var), __my_cpu_offset))
-#define this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, my_cpu_offset) +#ifndef __this_cpu_ptr #define __this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset) - +#endif +#ifdef CONFIG_DEBUG_PREEMPT +#define this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, my_cpu_offset) +#else +#define this_cpu_ptr(ptr) __this_cpu_ptr(ptr) +#endif
#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA extern void setup_per_cpu_areas(void);
| |