Messages in this thread | | | Subject | Re: [PATCH -v3 1/2] irq_work: generic hard-irq context callbacks | From | Peter Zijlstra <> | Date | Mon, 30 Aug 2010 11:41:26 +0200 |
| |
On Tue, 2010-07-13 at 12:59 +0800, Huang Ying wrote: > From: Peter Zijlstra <a.p.zijlstra@chello.nl> > > In order for other NMI context users that want to run things from > hard-IRQ context, extract the perf_event callback mechanism. > > Huang Ying: some fixes > > This patch is only tested on x86 platform.
Right, looks ok, although it would require some acks from relevant architecture maintainers, all of whoem you forgot to CC.
> Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> > Signed-off-by: Huang Ying <ying.huang@intel.com> > --- > arch/alpha/Kconfig | 1 > arch/alpha/include/asm/perf_event.h | 9 - > arch/arm/Kconfig | 1 > arch/arm/include/asm/perf_event.h | 12 -- > arch/arm/kernel/perf_event.c | 4 > arch/frv/Kconfig | 1 > arch/frv/lib/perf_event.c | 19 ---- > arch/parisc/Kconfig | 1 > arch/parisc/include/asm/perf_event.h | 7 - > arch/powerpc/Kconfig | 1 > arch/powerpc/kernel/time.c | 42 ++++---- > arch/s390/Kconfig | 1 > arch/s390/include/asm/perf_event.h | 10 -- > arch/sh/Kconfig | 1 > arch/sh/include/asm/perf_event.h | 7 - > arch/sparc/Kconfig | 2 > arch/sparc/include/asm/perf_event.h | 4 > arch/sparc/kernel/pcr.c | 8 - > arch/x86/Kconfig | 1 > arch/x86/include/asm/entry_arch.h | 4 > arch/x86/include/asm/hardirq.h | 2 > arch/x86/include/asm/hw_irq.h | 2 > arch/x86/include/asm/irq_vectors.h | 4 > arch/x86/kernel/Makefile | 1 > arch/x86/kernel/cpu/perf_event.c | 19 ---- > arch/x86/kernel/entry_64.S | 6 - > arch/x86/kernel/irq.c | 8 - > arch/x86/kernel/irq_work.c | 30 ++++++ > arch/x86/kernel/irqinit.c | 6 - > include/linux/irq_work.h | 20 ++++ > include/linux/perf_event.h | 11 -- > init/Kconfig | 8 + > kernel/Makefile | 2 > kernel/irq_work.c | 164 +++++++++++++++++++++++++++++++++++ > kernel/perf_event.c | 104 +--------------------- > kernel/timer.c | 7 + > 36 files changed, 290 insertions(+), 240 deletions(-) > > --- /dev/null > +++ b/include/linux/irq_work.h > @@ -0,0 +1,20 @@ > +#ifndef _LINUX_IRQ_WORK_H > +#define _LINUX_IRQ_WORK_H > + > +struct irq_work { > + struct irq_work *next; > + void (*func)(struct irq_work *); > +}; > + > +static inline > +void init_irq_work(struct irq_work *entry, void (*func)(struct irq_work *)) > +{ > + entry->next = NULL; > + entry->func = func; > +} > + > +bool irq_work_queue(struct irq_work *entry); > +void irq_work_run(void); > +void irq_work_sync(struct irq_work *entry); > + > +#endif /* _LINUX_IRQ_WORK_H */ > --- /dev/null > +++ b/kernel/irq_work.c > @@ -0,0 +1,164 @@ > +/* > + * Copyright (C) 2010 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> > + * > + * Provides a framework for enqueueing and running callbacks from hardirq > + * context. The enqueueing is NMI-safe. > + */ > + > +#include <linux/kernel.h> > +#include <linux/module.h> > +#include <linux/irq_work.h> > +#include <linux/hardirq.h> > + > +/* > + * An entry can be in one of four states: > + * > + * free NULL, 0 -> {claimed} : free to be used > + * claimed NULL, 3 -> {pending} : claimed to be enqueued > + * pending next, 3 -> {busy} : queued, pending callback > + * busy NULL, 2 -> {free, claimed} : callback in progress, can be claimed > + * > + * We use the lower two bits of the next pointer to keep PENDING and BUSY > + * flags. > + */ > + > +#define IRQ_WORK_PENDING 1UL > +#define IRQ_WORK_BUSY 2UL > +#define IRQ_WORK_FLAGS 3UL > + > +static inline bool irq_work_is_set(struct irq_work *entry, int flags) > +{ > + return (unsigned long)entry->next & flags; > +} > + > +static inline struct irq_work *irq_work_next(struct irq_work *entry) > +{ > + unsigned long next = (unsigned long)entry->next; > + next &= ~IRQ_WORK_FLAGS; > + return (struct irq_work *)next; > +} > + > +static inline struct irq_work *next_flags(struct irq_work *entry, int flags) > +{ > + unsigned long next = (unsigned long)entry; > + next |= flags; > + return (struct irq_work *)next; > +} > + > +static DEFINE_PER_CPU(struct irq_work *, irq_work_list); > + > +/* > + * Claim the entry so that no one else will poke at it. > + */ > +static bool irq_work_claim(struct irq_work *entry) > +{ > + struct irq_work *next, *nflags; > + > + do { > + next = entry->next; > + if ((unsigned long)next & IRQ_WORK_PENDING) > + return false; > + nflags = next_flags(next, IRQ_WORK_FLAGS); > + } while (cmpxchg(&entry->next, next, nflags) != next); > + > + return true; > +} > + > + > +void __weak arch_irq_work_raise(void) > +{ > + /* > + * Lame architectures will get the timer tick callback > + */ > +} > + > +/* > + * Queue the entry and raise the IPI if needed. > + */ > +static void __irq_work_queue(struct irq_work *entry) > +{ > + struct irq_work **head, *next; > + > + head = &get_cpu_var(irq_work_list); > + > + do { > + next = *head; > + /* Can assign non-atomic because we keep the flags set. */ > + entry->next = next_flags(next, IRQ_WORK_FLAGS); > + } while (cmpxchg(head, next, entry) != next); > + > + /* The list was empty, raise self-interrupt to start processing. */ > + if (!irq_work_next(entry)) > + arch_irq_work_raise(); > + > + put_cpu_var(irq_work_list); > +} > + > +/* > + * Enqueue the irq_work @entry, returns true on success, failure when the > + * @entry was already enqueued by someone else. > + * > + * Can be re-enqueued while the callback is still in progress. > + */ > +bool irq_work_queue(struct irq_work *entry) > +{ > + if (!irq_work_claim(entry)) { > + /* > + * Already enqueued, can't do! > + */ > + return false; > + } > + > + __irq_work_queue(entry); > + return true; > +} > +EXPORT_SYMBOL_GPL(irq_work_queue); > + > +/* > + * Run the irq_work entries on this cpu. Requires to be ran from hardirq > + * context with local IRQs disabled. > + */ > +void irq_work_run(void) > +{ > + struct irq_work *list, **head; > + > + head = &__get_cpu_var(irq_work_list); > + if (*head == NULL) > + return; > + > + BUG_ON(!in_irq()); > + BUG_ON(!irqs_disabled()); > + > + list = xchg(head, NULL); > + while (list != NULL) { > + struct irq_work *entry = list; > + > + list = irq_work_next(list); > + > + /* > + * Clear the PENDING bit, after this point the @entry > + * can be re-used. > + */ > + entry->next = next_flags(NULL, IRQ_WORK_BUSY); > + entry->func(entry); > + /* > + * Clear the BUSY bit and return to the free state if > + * no-one else claimed it meanwhile. > + */ > + cmpxchg(&entry->next, next_flags(NULL, IRQ_WORK_BUSY), NULL); > + } > +} > +EXPORT_SYMBOL_GPL(irq_work_run); > + > +/* > + * Synchronize against the irq_work @entry, ensures the entry is not > + * currently in use. > + */ > +void irq_work_sync(struct irq_work *entry) > +{ > + WARN_ON_ONCE(irqs_disabled()); > + > + while (irq_work_is_set(entry, IRQ_WORK_BUSY)) > + cpu_relax(); > +} > +EXPORT_SYMBOL_GPL(irq_work_sync); > --- a/arch/alpha/Kconfig > +++ b/arch/alpha/Kconfig > @@ -9,6 +9,7 @@ config ALPHA > select HAVE_IDE > select HAVE_OPROFILE > select HAVE_SYSCALL_WRAPPERS > + select HAVE_IRQ_WORK > select HAVE_PERF_EVENTS > select HAVE_DMA_ATTRS > help > --- a/arch/alpha/include/asm/perf_event.h > +++ /dev/null > @@ -1,9 +0,0 @@ > -#ifndef __ASM_ALPHA_PERF_EVENT_H > -#define __ASM_ALPHA_PERF_EVENT_H > - > -/* Alpha only supports software events through this interface. */ > -static inline void set_perf_event_pending(void) { } > - > -#define PERF_EVENT_INDEX_OFFSET 0 > - > -#endif /* __ASM_ALPHA_PERF_EVENT_H */ > --- a/arch/arm/Kconfig > +++ b/arch/arm/Kconfig > @@ -22,6 +22,7 @@ config ARM > select HAVE_KERNEL_GZIP > select HAVE_KERNEL_LZO > select HAVE_KERNEL_LZMA > + select HAVE_IRQ_WORK > select HAVE_PERF_EVENTS > select PERF_USE_VMALLOC > help > --- a/arch/arm/include/asm/perf_event.h > +++ b/arch/arm/include/asm/perf_event.h > @@ -12,18 +12,6 @@ > #ifndef __ARM_PERF_EVENT_H__ > #define __ARM_PERF_EVENT_H__ > > -/* > - * NOP: on *most* (read: all supported) ARM platforms, the performance > - * counter interrupts are regular interrupts and not an NMI. This > - * means that when we receive the interrupt we can call > - * perf_event_do_pending() that handles all of the work with > - * interrupts enabled. > - */ > -static inline void > -set_perf_event_pending(void) > -{ > -} > - > /* ARM performance counters start from 1 (in the cp15 accesses) so use the > * same indexes here for consistency. */ > #define PERF_EVENT_INDEX_OFFSET 1 > --- a/arch/frv/Kconfig > +++ b/arch/frv/Kconfig > @@ -7,6 +7,7 @@ config FRV > default y > select HAVE_IDE > select HAVE_ARCH_TRACEHOOK > + select HAVE_IRQ_WORK > select HAVE_PERF_EVENTS > > config ZONE_DMA > --- a/arch/frv/lib/perf_event.c > +++ /dev/null > @@ -1,19 +0,0 @@ > -/* Performance event handling > - * > - * Copyright (C) 2009 Red Hat, Inc. All Rights Reserved. > - * Written by David Howells (dhowells@redhat.com) > - * > - * This program is free software; you can redistribute it and/or > - * modify it under the terms of the GNU General Public Licence > - * as published by the Free Software Foundation; either version > - * 2 of the Licence, or (at your option) any later version. > - */ > - > -#include <linux/perf_event.h> > - > -/* > - * mark the performance event as pending > - */ > -void set_perf_event_pending(void) > -{ > -} > --- a/arch/parisc/Kconfig > +++ b/arch/parisc/Kconfig > @@ -16,6 +16,7 @@ config PARISC > select RTC_DRV_GENERIC > select INIT_ALL_POSSIBLE > select BUG > + select HAVE_IRQ_WORK > select HAVE_PERF_EVENTS > select GENERIC_ATOMIC64 if !64BIT > help > --- a/arch/parisc/include/asm/perf_event.h > +++ /dev/null > @@ -1,7 +0,0 @@ > -#ifndef __ASM_PARISC_PERF_EVENT_H > -#define __ASM_PARISC_PERF_EVENT_H > - > -/* parisc only supports software events through this interface. */ > -static inline void set_perf_event_pending(void) { } > - > -#endif /* __ASM_PARISC_PERF_EVENT_H */ > --- a/arch/powerpc/Kconfig > +++ b/arch/powerpc/Kconfig > @@ -139,6 +139,7 @@ config PPC > select HAVE_OPROFILE > select HAVE_SYSCALL_WRAPPERS if PPC64 > select GENERIC_ATOMIC64 if PPC32 > + select HAVE_IRQ_WORK > select HAVE_PERF_EVENTS > select HAVE_REGS_AND_STACK_ACCESS_API > > --- a/arch/powerpc/kernel/time.c > +++ b/arch/powerpc/kernel/time.c > @@ -53,7 +53,7 @@ > #include <linux/posix-timers.h> > #include <linux/irq.h> > #include <linux/delay.h> > -#include <linux/perf_event.h> > +#include <linux/irq_work.h> > #include <asm/trace.h> > > #include <asm/io.h> > @@ -532,60 +532,60 @@ void __init iSeries_time_init_early(void > } > #endif /* CONFIG_PPC_ISERIES */ > > -#ifdef CONFIG_PERF_EVENTS > +#ifdef CONFIG_IRQ_WORK > > /* > * 64-bit uses a byte in the PACA, 32-bit uses a per-cpu variable... > */ > #ifdef CONFIG_PPC64 > -static inline unsigned long test_perf_event_pending(void) > +static inline unsigned long test_irq_work_pending(void) > { > unsigned long x; > > asm volatile("lbz %0,%1(13)" > : "=r" (x) > - : "i" (offsetof(struct paca_struct, perf_event_pending))); > + : "i" (offsetof(struct paca_struct, irq_work_pending))); > return x; > } > > -static inline void set_perf_event_pending_flag(void) > +static inline void set_irq_work_pending_flag(void) > { > asm volatile("stb %0,%1(13)" : : > "r" (1), > - "i" (offsetof(struct paca_struct, perf_event_pending))); > + "i" (offsetof(struct paca_struct, irq_work_pending))); > } > > -static inline void clear_perf_event_pending(void) > +static inline void clear_irq_work_pending(void) > { > asm volatile("stb %0,%1(13)" : : > "r" (0), > - "i" (offsetof(struct paca_struct, perf_event_pending))); > + "i" (offsetof(struct paca_struct, irq_work_pending))); > } > > #else /* 32-bit */ > > -DEFINE_PER_CPU(u8, perf_event_pending); > +DEFINE_PER_CPU(u8, irq_work_pending); > > -#define set_perf_event_pending_flag() __get_cpu_var(perf_event_pending) = 1 > -#define test_perf_event_pending() __get_cpu_var(perf_event_pending) > -#define clear_perf_event_pending() __get_cpu_var(perf_event_pending) = 0 > +#define set_irq_work_pending_flag() __get_cpu_var(irq_work_pending) = 1 > +#define test_irq_work_pending() __get_cpu_var(irq_work_pending) > +#define clear_irq_work_pending() __get_cpu_var(irq_work_pending) = 0 > > #endif /* 32 vs 64 bit */ > > -void set_perf_event_pending(void) > +void set_irq_work_pending(void) > { > preempt_disable(); > - set_perf_event_pending_flag(); > + set_irq_work_pending_flag(); > set_dec(1); > preempt_enable(); > } > > -#else /* CONFIG_PERF_EVENTS */ > +#else /* CONFIG_IRQ_WORK */ > > -#define test_perf_event_pending() 0 > -#define clear_perf_event_pending() > +#define test_irq_work_pending() 0 > +#define clear_irq_work_pending() > > -#endif /* CONFIG_PERF_EVENTS */ > +#endif /* CONFIG_IRQ_WORK */ > > /* > * For iSeries shared processors, we have to let the hypervisor > @@ -635,9 +635,9 @@ void timer_interrupt(struct pt_regs * re > > calculate_steal_time(); > > - if (test_perf_event_pending()) { > - clear_perf_event_pending(); > - perf_event_do_pending(); > + if (test_irq_work_pending()) { > + clear_irq_work_pending(); > + irq_work_run(); > } > > #ifdef CONFIG_PPC_ISERIES > --- a/arch/s390/Kconfig > +++ b/arch/s390/Kconfig > @@ -98,6 +98,7 @@ config S390 > select HAVE_KVM if 64BIT > select HAVE_ARCH_TRACEHOOK > select INIT_ALL_POSSIBLE > + select HAVE_IRQ_WORK > select HAVE_PERF_EVENTS > select HAVE_KERNEL_GZIP > select HAVE_KERNEL_BZIP2 > --- a/arch/s390/include/asm/perf_event.h > +++ /dev/null > @@ -1,10 +0,0 @@ > -/* > - * Performance event support - s390 specific definitions. > - * > - * Copyright 2009 Martin Schwidefsky, IBM Corporation. > - */ > - > -static inline void set_perf_event_pending(void) {} > -static inline void clear_perf_event_pending(void) {} > - > -#define PERF_EVENT_INDEX_OFFSET 0 > --- a/arch/sh/Kconfig > +++ b/arch/sh/Kconfig > @@ -16,6 +16,7 @@ config SUPERH > select HAVE_ARCH_TRACEHOOK > select HAVE_DMA_API_DEBUG > select HAVE_DMA_ATTRS > + select HAVE_IRQ_WORK > select HAVE_PERF_EVENTS > select PERF_USE_VMALLOC > select HAVE_KERNEL_GZIP > --- a/arch/sh/include/asm/perf_event.h > +++ b/arch/sh/include/asm/perf_event.h > @@ -26,11 +26,4 @@ extern int register_sh_pmu(struct sh_pmu > extern int reserve_pmc_hardware(void); > extern void release_pmc_hardware(void); > > -static inline void set_perf_event_pending(void) > -{ > - /* Nothing to see here, move along. */ > -} > - > -#define PERF_EVENT_INDEX_OFFSET 0 > - > #endif /* __ASM_SH_PERF_EVENT_H */ > --- a/arch/sparc/Kconfig > +++ b/arch/sparc/Kconfig > @@ -25,6 +25,7 @@ config SPARC > select ARCH_WANT_OPTIONAL_GPIOLIB > select RTC_CLASS > select RTC_DRV_M48T59 > + select HAVE_IRQ_WORK > select HAVE_PERF_EVENTS > select PERF_USE_VMALLOC > select HAVE_DMA_ATTRS > @@ -52,6 +53,7 @@ config SPARC64 > select RTC_DRV_BQ4802 > select RTC_DRV_SUN4V > select RTC_DRV_STARFIRE > + select HAVE_IRQ_WORK > select HAVE_PERF_EVENTS > select PERF_USE_VMALLOC > > --- a/arch/sparc/include/asm/perf_event.h > +++ b/arch/sparc/include/asm/perf_event.h > @@ -1,10 +1,6 @@ > #ifndef __ASM_SPARC_PERF_EVENT_H > #define __ASM_SPARC_PERF_EVENT_H > > -extern void set_perf_event_pending(void); > - > -#define PERF_EVENT_INDEX_OFFSET 0 > - > #ifdef CONFIG_PERF_EVENTS > extern void init_hw_perf_events(void); > #else > --- a/arch/sparc/kernel/pcr.c > +++ b/arch/sparc/kernel/pcr.c > @@ -7,7 +7,7 @@ > #include <linux/init.h> > #include <linux/irq.h> > > -#include <linux/perf_event.h> > +#include <linux/irq_work.h> > #include <linux/ftrace.h> > > #include <asm/pil.h> > @@ -43,14 +43,14 @@ void __irq_entry deferred_pcr_work_irq(i > > old_regs = set_irq_regs(regs); > irq_enter(); > -#ifdef CONFIG_PERF_EVENTS > - perf_event_do_pending(); > +#ifdef CONFIG_IRQ_WORK > + irq_work_run(); > #endif > irq_exit(); > set_irq_regs(old_regs); > } > > -void set_perf_event_pending(void) > +void arch_irq_work_raise(void) > { > set_softint(1 << PIL_DEFERRED_PCR_WORK); > } > --- a/arch/x86/Kconfig > +++ b/arch/x86/Kconfig > @@ -25,6 +25,7 @@ config X86 > select HAVE_IDE > select HAVE_OPROFILE > select HAVE_PERF_EVENTS if (!M386 && !M486) > + select HAVE_IRQ_WORK > select HAVE_IOREMAP_PROT > select HAVE_KPROBES > select ARCH_WANT_OPTIONAL_GPIOLIB > --- a/arch/x86/include/asm/entry_arch.h > +++ b/arch/x86/include/asm/entry_arch.h > @@ -49,8 +49,8 @@ BUILD_INTERRUPT(apic_timer_interrupt,LOC > BUILD_INTERRUPT(error_interrupt,ERROR_APIC_VECTOR) > BUILD_INTERRUPT(spurious_interrupt,SPURIOUS_APIC_VECTOR) > > -#ifdef CONFIG_PERF_EVENTS > -BUILD_INTERRUPT(perf_pending_interrupt, LOCAL_PENDING_VECTOR) > +#ifdef CONFIG_IRQ_WORK > +BUILD_INTERRUPT(irq_work_interrupt, IRQ_WORK_VECTOR) > #endif > > #ifdef CONFIG_X86_THERMAL_VECTOR > --- a/arch/x86/include/asm/hw_irq.h > +++ b/arch/x86/include/asm/hw_irq.h > @@ -29,7 +29,7 @@ > extern void apic_timer_interrupt(void); > extern void x86_platform_ipi(void); > extern void error_interrupt(void); > -extern void perf_pending_interrupt(void); > +extern void irq_work_interrupt(void); > > extern void spurious_interrupt(void); > extern void thermal_interrupt(void); > --- a/arch/x86/kernel/Makefile > +++ b/arch/x86/kernel/Makefile > @@ -33,6 +33,7 @@ obj-y := process_$(BITS).o signal.o en > obj-y += traps.o irq.o irq_$(BITS).o dumpstack_$(BITS).o > obj-y += time.o ioport.o ldt.o dumpstack.o > obj-y += setup.o x86_init.o i8259.o irqinit.o > +obj-$(CONFIG_IRQ_WORK) += irq_work.o > obj-$(CONFIG_X86_VISWS) += visws_quirks.o > obj-$(CONFIG_X86_32) += probe_roms_32.o > obj-$(CONFIG_X86_32) += sys_i386_32.o i386_ksyms_32.o > --- a/arch/x86/kernel/cpu/perf_event.c > +++ b/arch/x86/kernel/cpu/perf_event.c > @@ -1160,25 +1160,6 @@ static int x86_pmu_handle_irq(struct pt_ > return handled; > } > > -void smp_perf_pending_interrupt(struct pt_regs *regs) > -{ > - irq_enter(); > - ack_APIC_irq(); > - inc_irq_stat(apic_pending_irqs); > - perf_event_do_pending(); > - irq_exit(); > -} > - > -void set_perf_event_pending(void) > -{ > -#ifdef CONFIG_X86_LOCAL_APIC > - if (!x86_pmu.apic || !x86_pmu_initialized()) > - return; > - > - apic->send_IPI_self(LOCAL_PENDING_VECTOR); > -#endif > -} > - > void perf_events_lapic_init(void) > { > if (!x86_pmu.apic || !x86_pmu_initialized()) > --- a/arch/x86/kernel/entry_64.S > +++ b/arch/x86/kernel/entry_64.S > @@ -1023,9 +1023,9 @@ apicinterrupt ERROR_APIC_VECTOR \ > apicinterrupt SPURIOUS_APIC_VECTOR \ > spurious_interrupt smp_spurious_interrupt > > -#ifdef CONFIG_PERF_EVENTS > -apicinterrupt LOCAL_PENDING_VECTOR \ > - perf_pending_interrupt smp_perf_pending_interrupt > +#ifdef CONFIG_IRQ_WORK > +apicinterrupt IRQ_WORK_VECTOR \ > + irq_work_interrupt smp_irq_work_interrupt > #endif > > /* > --- /dev/null > +++ b/arch/x86/kernel/irq_work.c > @@ -0,0 +1,30 @@ > +/* > + * x86 specific code for irq_work > + * > + * Copyright (C) 2010 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> > + */ > + > +#include <linux/kernel.h> > +#include <linux/irq_work.h> > +#include <linux/hardirq.h> > +#include <asm/apic.h> > + > +void smp_irq_work_interrupt(struct pt_regs *regs) > +{ > + irq_enter(); > + ack_APIC_irq(); > + inc_irq_stat(apic_irq_work_irqs); > + irq_work_run(); > + irq_exit(); > +} > + > +void arch_irq_work_raise(void) > +{ > +#ifdef CONFIG_X86_LOCAL_APIC > + if (!cpu_has_apic) > + return; > + > + apic->send_IPI_self(IRQ_WORK_VECTOR); > + apic_wait_icr_idle(); > +#endif > +} > --- a/arch/x86/kernel/irqinit.c > +++ b/arch/x86/kernel/irqinit.c > @@ -224,9 +224,9 @@ static void __init apic_intr_init(void) > alloc_intr_gate(SPURIOUS_APIC_VECTOR, spurious_interrupt); > alloc_intr_gate(ERROR_APIC_VECTOR, error_interrupt); > > - /* Performance monitoring interrupts: */ > -# ifdef CONFIG_PERF_EVENTS > - alloc_intr_gate(LOCAL_PENDING_VECTOR, perf_pending_interrupt); > + /* IRQ work interrupts: */ > +# ifdef CONFIG_IRQ_WORK > + alloc_intr_gate(IRQ_WORK_VECTOR, irq_work_interrupt); > # endif > > #endif > --- a/include/linux/perf_event.h > +++ b/include/linux/perf_event.h > @@ -484,6 +484,7 @@ struct perf_guest_info_callbacks { > #include <linux/workqueue.h> > #include <linux/ftrace.h> > #include <linux/cpu.h> > +#include <linux/irq_work.h> > #include <asm/atomic.h> > #include <asm/local.h> > > @@ -608,11 +609,6 @@ struct perf_mmap_data { > void *data_pages[0]; > }; > > -struct perf_pending_entry { > - struct perf_pending_entry *next; > - void (*func)(struct perf_pending_entry *); > -}; > - > struct perf_sample_data; > > typedef void (*perf_overflow_handler_t)(struct perf_event *, int, > @@ -719,7 +715,7 @@ struct perf_event { > int pending_wakeup; > int pending_kill; > int pending_disable; > - struct perf_pending_entry pending; > + struct irq_work pending; > > atomic_t event_limit; > > @@ -831,8 +827,6 @@ extern void perf_event_task_tick(struct > extern int perf_event_init_task(struct task_struct *child); > extern void perf_event_exit_task(struct task_struct *child); > extern void perf_event_free_task(struct task_struct *task); > -extern void set_perf_event_pending(void); > -extern void perf_event_do_pending(void); > extern void perf_event_print_debug(void); > extern void __perf_disable(void); > extern bool __perf_enable(void); > @@ -1031,7 +1025,6 @@ perf_event_task_tick(struct task_struct > static inline int perf_event_init_task(struct task_struct *child) { return 0; } > static inline void perf_event_exit_task(struct task_struct *child) { } > static inline void perf_event_free_task(struct task_struct *task) { } > -static inline void perf_event_do_pending(void) { } > static inline void perf_event_print_debug(void) { } > static inline void perf_disable(void) { } > static inline void perf_enable(void) { } > --- a/init/Kconfig > +++ b/init/Kconfig > @@ -21,6 +21,13 @@ config CONSTRUCTORS > depends on !UML > default y > > +config HAVE_IRQ_WORK > + bool > + > +config IRQ_WORK > + bool > + depends on HAVE_IRQ_WORK > + > menu "General setup" > > config EXPERIMENTAL > @@ -983,6 +990,7 @@ config PERF_EVENTS > default y if (PROFILING || PERF_COUNTERS) > depends on HAVE_PERF_EVENTS > select ANON_INODES > + select IRQ_WORK > help > Enable kernel support for various performance events provided > by software and hardware. > --- a/kernel/Makefile > +++ b/kernel/Makefile > @@ -23,6 +23,7 @@ CFLAGS_REMOVE_rtmutex-debug.o = -pg > CFLAGS_REMOVE_cgroup-debug.o = -pg > CFLAGS_REMOVE_sched_clock.o = -pg > CFLAGS_REMOVE_perf_event.o = -pg > +CFLAGS_REMOVE_irq_work.o = -pg > endif > > obj-$(CONFIG_FREEZER) += freezer.o > @@ -101,6 +102,7 @@ obj-$(CONFIG_RING_BUFFER) += trace/ > obj-$(CONFIG_SMP) += sched_cpupri.o > obj-$(CONFIG_SLOW_WORK) += slow-work.o > obj-$(CONFIG_SLOW_WORK_DEBUG) += slow-work-debugfs.o > +obj-$(CONFIG_IRQ_WORK) += irq_work.o > obj-$(CONFIG_PERF_EVENTS) += perf_event.o > obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o > obj-$(CONFIG_USER_RETURN_NOTIFIER) += user-return-notifier.o > --- a/kernel/perf_event.c > +++ b/kernel/perf_event.c > @@ -1882,12 +1882,11 @@ static void free_event_rcu(struct rcu_he > kfree(event); > } > > -static void perf_pending_sync(struct perf_event *event); > static void perf_mmap_data_put(struct perf_mmap_data *data); > > static void free_event(struct perf_event *event) > { > - perf_pending_sync(event); > + irq_work_sync(&event->pending); > > if (!event->parent) { > atomic_dec(&nr_events); > @@ -2824,16 +2823,7 @@ void perf_event_wakeup(struct perf_event > } > } > > -/* > - * Pending wakeups > - * > - * Handle the case where we need to wakeup up from NMI (or rq->lock) context. > - * > - * The NMI bit means we cannot possibly take locks. Therefore, maintain a > - * single linked list and use cmpxchg() to add entries lockless. > - */ > - > -static void perf_pending_event(struct perf_pending_entry *entry) > +static void perf_pending_event(struct irq_work *entry) > { > struct perf_event *event = container_of(entry, > struct perf_event, pending); > @@ -2849,89 +2839,6 @@ static void perf_pending_event(struct pe > } > } > > -#define PENDING_TAIL ((struct perf_pending_entry *)-1UL) > - > -static DEFINE_PER_CPU(struct perf_pending_entry *, perf_pending_head) = { > - PENDING_TAIL, > -}; > - > -static void perf_pending_queue(struct perf_pending_entry *entry, > - void (*func)(struct perf_pending_entry *)) > -{ > - struct perf_pending_entry **head; > - > - if (cmpxchg(&entry->next, NULL, PENDING_TAIL) != NULL) > - return; > - > - entry->func = func; > - > - head = &get_cpu_var(perf_pending_head); > - > - do { > - entry->next = *head; > - } while (cmpxchg(head, entry->next, entry) != entry->next); > - > - set_perf_event_pending(); > - > - put_cpu_var(perf_pending_head); > -} > - > -static int __perf_pending_run(void) > -{ > - struct perf_pending_entry *list; > - int nr = 0; > - > - list = xchg(&__get_cpu_var(perf_pending_head), PENDING_TAIL); > - while (list != PENDING_TAIL) { > - void (*func)(struct perf_pending_entry *); > - struct perf_pending_entry *entry = list; > - > - list = list->next; > - > - func = entry->func; > - entry->next = NULL; > - /* > - * Ensure we observe the unqueue before we issue the wakeup, > - * so that we won't be waiting forever. > - * -- see perf_not_pending(). > - */ > - smp_wmb(); > - > - func(entry); > - nr++; > - } > - > - return nr; > -} > - > -static inline int perf_not_pending(struct perf_event *event) > -{ > - /* > - * If we flush on whatever cpu we run, there is a chance we don't > - * need to wait. > - */ > - get_cpu(); > - __perf_pending_run(); > - put_cpu(); > - > - /* > - * Ensure we see the proper queue state before going to sleep > - * so that we do not miss the wakeup. -- see perf_pending_handle() > - */ > - smp_rmb(); > - return event->pending.next == NULL; > -} > - > -static void perf_pending_sync(struct perf_event *event) > -{ > - wait_event(event->waitq, perf_not_pending(event)); > -} > - > -void perf_event_do_pending(void) > -{ > - __perf_pending_run(); > -} > - > /* > * Callchain support -- arch specific > */ > @@ -2996,8 +2903,7 @@ static void perf_output_wakeup(struct pe > > if (handle->nmi) { > handle->event->pending_wakeup = 1; > - perf_pending_queue(&handle->event->pending, > - perf_pending_event); > + irq_work_queue(&handle->event->pending); > } else > perf_event_wakeup(handle->event); > } > @@ -3976,8 +3882,7 @@ static int __perf_event_overflow(struct > event->pending_kill = POLL_HUP; > if (nmi) { > event->pending_disable = 1; > - perf_pending_queue(&event->pending, > - perf_pending_event); > + irq_work_queue(&event->pending); > } else > perf_event_disable(event); > } > @@ -4831,6 +4736,7 @@ perf_event_alloc(struct perf_event_attr > INIT_LIST_HEAD(&event->event_entry); > INIT_LIST_HEAD(&event->sibling_list); > init_waitqueue_head(&event->waitq); > + init_irq_work(&event->pending, perf_pending_event); > > mutex_init(&event->mmap_mutex); > > --- a/kernel/timer.c > +++ b/kernel/timer.c > @@ -37,7 +37,7 @@ > #include <linux/delay.h> > #include <linux/tick.h> > #include <linux/kallsyms.h> > -#include <linux/perf_event.h> > +#include <linux/irq_work.h> > #include <linux/sched.h> > #include <linux/slab.h> > > @@ -1264,7 +1264,10 @@ void update_process_times(int user_tick) > run_local_timers(); > rcu_check_callbacks(cpu, user_tick); > printk_tick(); > - perf_event_do_pending(); > +#ifdef CONFIG_IRQ_WORK > + if (in_irq()) > + irq_work_run(); > +#endif > scheduler_tick(); > run_posix_cpu_timers(p); > } > --- a/arch/arm/kernel/perf_event.c > +++ b/arch/arm/kernel/perf_event.c > @@ -1045,7 +1045,7 @@ armv6pmu_handle_irq(int irq_num, > * platforms that can have the PMU interrupts raised as a PMI, this > * will not work. > */ > - perf_event_do_pending(); > + irq_work_run(); > > return IRQ_HANDLED; > } > @@ -2021,7 +2021,7 @@ static irqreturn_t armv7pmu_handle_irq(i > * platforms that can have the PMU interrupts raised as a PMI, this > * will not work. > */ > - perf_event_do_pending(); > + irq_work_run(); > > return IRQ_HANDLED; > } > --- a/arch/x86/include/asm/irq_vectors.h > +++ b/arch/x86/include/asm/irq_vectors.h > @@ -114,9 +114,9 @@ > #define X86_PLATFORM_IPI_VECTOR 0xed > > /* > - * Performance monitoring pending work vector: > + * IRQ work vector: > */ > -#define LOCAL_PENDING_VECTOR 0xec > +#define IRQ_WORK_VECTOR 0xec > > #define UV_BAU_MESSAGE 0xea > > --- a/arch/x86/include/asm/hardirq.h > +++ b/arch/x86/include/asm/hardirq.h > @@ -14,7 +14,7 @@ typedef struct { > #endif > unsigned int x86_platform_ipis; /* arch dependent */ > unsigned int apic_perf_irqs; > - unsigned int apic_pending_irqs; > + unsigned int apic_irq_work_irqs; > #ifdef CONFIG_SMP > unsigned int irq_resched_count; > unsigned int irq_call_count; > --- a/arch/x86/kernel/irq.c > +++ b/arch/x86/kernel/irq.c > @@ -67,10 +67,10 @@ static int show_other_interrupts(struct > for_each_online_cpu(j) > seq_printf(p, "%10u ", irq_stats(j)->apic_perf_irqs); > seq_printf(p, " Performance monitoring interrupts\n"); > - seq_printf(p, "%*s: ", prec, "PND"); > + seq_printf(p, "%*s: ", prec, "IWI"); > for_each_online_cpu(j) > - seq_printf(p, "%10u ", irq_stats(j)->apic_pending_irqs); > - seq_printf(p, " Performance pending work\n"); > + seq_printf(p, "%10u ", irq_stats(j)->apic_irq_work_irqs); > + seq_printf(p, " IRQ work interrupts\n"); > #endif > if (x86_platform_ipi_callback) { > seq_printf(p, "%*s: ", prec, "PLT"); > @@ -185,7 +185,7 @@ u64 arch_irq_stat_cpu(unsigned int cpu) > sum += irq_stats(cpu)->apic_timer_irqs; > sum += irq_stats(cpu)->irq_spurious_count; > sum += irq_stats(cpu)->apic_perf_irqs; > - sum += irq_stats(cpu)->apic_pending_irqs; > + sum += irq_stats(cpu)->apic_irq_work_irqs; > #endif > if (x86_platform_ipi_callback) > sum += irq_stats(cpu)->x86_platform_ipis;
| |