Messages in this thread Patch in this message | | | Date | Thu, 31 Aug 2006 07:40:11 -0700 | From | Stephane Eranian <> | Subject | [PATCH] x86_64 fix idle notifier |
| |
Hello,
Here is a patch against Linus GIT tree that fixes the idle notifier for the X86-64 tree. With this notifiier you can get called when entering/exiting the core of the idle loop. I don't think it is used yet but I am planning to leverage it for perfmon.
There was an issue in that it would not capture all re-entries into the idle loop because of missing hooks on the return from interrupt path.
I have also modified the header file (idle.h) to avoid branching in case this is not needed. The enter_idle()/exit_idle() calls are not integrated into irq_enter()/irq_exit() because those are generic and for now the idle notifier is arch specific.
Changelog: - fix the x86-64 idle notifier to cover all re-entries into the core idle loop after an interrupt. - define enter_idle/exit_idle in idle.h to avoid branching when not needed
Signed-off-by: Stephane Eranian <eranain@hpl.hp.com>
diff --git a/arch/i386/kernel/apic.c b/arch/i386/kernel/apic.c --- a/arch/i386/kernel/apic.c +++ b/arch/i386/kernel/apic.c @@ -1230,9 +1230,11 @@ fastcall void smp_apic_timer_interrupt(s * Besides, if we don't timer interrupts ignore the global * interrupt lock, which is the WrongThing (tm) to do. */ + exit_idle(); irq_enter(); smp_local_timer_interrupt(regs); irq_exit(); + enter_idle(); } #ifndef CONFIG_SMP @@ -1279,6 +1281,7 @@ fastcall void smp_spurious_interrupt(str { unsigned long v; + exit_idle(); irq_enter(); /* * Check if this really is a spurious interrupt and ACK it @@ -1293,6 +1296,7 @@ fastcall void smp_spurious_interrupt(str printk(KERN_INFO "spurious APIC interrupt on CPU#%d, should never happen.\n", smp_processor_id()); irq_exit(); + enter_idle(); } /* @@ -1303,6 +1307,7 @@ fastcall void smp_error_interrupt(struct { unsigned long v, v1; + exit_idle(); irq_enter(); /* First tickle the hardware, only then report what went on. -- REW */ v = apic_read(APIC_ESR); @@ -1324,6 +1329,7 @@ fastcall void smp_error_interrupt(struct printk (KERN_DEBUG "APIC error on CPU%d: %02lx(%02lx)\n", smp_processor_id(), v , v1); irq_exit(); + enter_idle(); } /* diff --git a/arch/i386/kernel/cpu/mcheck/p4.c b/arch/i386/kernel/cpu/mcheck/p4.c --- a/arch/i386/kernel/cpu/mcheck/p4.c +++ b/arch/i386/kernel/cpu/mcheck/p4.c @@ -70,9 +70,11 @@ static void (*vendor_thermal_interrupt)( fastcall void smp_thermal_interrupt(struct pt_regs *regs) { + exit_idle(); irq_enter(); vendor_thermal_interrupt(regs); irq_exit(); + enter_idle(); } /* P4/Xeon Thermal regulation detect and init */ diff --git a/arch/i386/kernel/process.c b/arch/i386/kernel/process.c --- a/arch/i386/kernel/process.c +++ b/arch/i386/kernel/process.c @@ -78,6 +78,35 @@ void (*pm_idle)(void); EXPORT_SYMBOL(pm_idle); static DEFINE_PER_CPU(unsigned int, cpu_idle_state); +static ATOMIC_NOTIFIER_HEAD(idle_notifier); + +void idle_notifier_register(struct notifier_block *n) +{ + atomic_notifier_chain_register(&idle_notifier, n); +} +EXPORT_SYMBOL_GPL(idle_notifier_register); + +void idle_notifier_unregister(struct notifier_block *n) +{ + atomic_notifier_chain_unregister(&idle_notifier, n); +} +EXPORT_SYMBOL(idle_notifier_unregister); + +enum idle_state { CPU_IDLE, CPU_NOT_IDLE }; +static DEFINE_PER_CPU(enum idle_state, idle_state) = CPU_NOT_IDLE; + +void __enter_idle(void) +{ + __get_cpu_var(idle_state) = CPU_IDLE; + atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL); +} + +void __exit_idle(void) +{ + __get_cpu_var(idle_state) = CPU_NOT_IDLE; + atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL); +} + void disable_hlt(void) { hlt_counter++; @@ -193,7 +222,9 @@ void cpu_idle(void) play_dead(); __get_cpu_var(irq_stat).idle_timestamp = jiffies; + __enter_idle(); idle(); + __exit_idle(); } preempt_enable_no_resched(); schedule(); diff --git a/arch/i386/kernel/smp.c b/arch/i386/kernel/smp.c --- a/arch/i386/kernel/smp.c +++ b/arch/i386/kernel/smp.c @@ -624,6 +624,7 @@ fastcall void smp_call_function_interrup /* * At this point the info structure may be out of scope unless wait==1 */ + exit_idle(); irq_enter(); (*func)(info); irq_exit(); @@ -632,5 +633,6 @@ fastcall void smp_call_function_interrup mb(); atomic_inc(&call_data->finished); } + enter_idle(); } diff --git a/arch/x86_64/kernel/apic.c b/arch/x86_64/kernel/apic.c --- a/arch/x86_64/kernel/apic.c +++ b/arch/x86_64/kernel/apic.c @@ -988,6 +988,7 @@ void smp_apic_timer_interrupt(struct pt_ irq_enter(); smp_local_timer_interrupt(regs); irq_exit(); + enter_idle(); } /* @@ -1069,6 +1070,7 @@ asmlinkage void smp_spurious_interrupt(v } #endif irq_exit(); + enter_idle(); } /* @@ -1101,6 +1103,7 @@ asmlinkage void smp_error_interrupt(void printk (KERN_DEBUG "APIC error on CPU%d: %02x(%02x)\n", smp_processor_id(), v , v1); irq_exit(); + enter_idle(); } int disable_apic; diff --git a/arch/x86_64/kernel/irq.c b/arch/x86_64/kernel/irq.c --- a/arch/x86_64/kernel/irq.c +++ b/arch/x86_64/kernel/irq.c @@ -131,6 +131,7 @@ asmlinkage unsigned int do_IRQ(struct pt #endif __do_IRQ(irq, regs); irq_exit(); + enter_idle(); return 1; } diff --git a/arch/x86_64/kernel/mce_amd.c b/arch/x86_64/kernel/mce_amd.c --- a/arch/x86_64/kernel/mce_amd.c +++ b/arch/x86_64/kernel/mce_amd.c @@ -224,6 +224,7 @@ asmlinkage void mce_threshold_interrupt( } out: irq_exit(); + enter_idle(); } /* diff --git a/arch/x86_64/kernel/mce_intel.c b/arch/x86_64/kernel/mce_intel.c --- a/arch/x86_64/kernel/mce_intel.c +++ b/arch/x86_64/kernel/mce_intel.c @@ -42,6 +42,7 @@ asmlinkage void smp_thermal_interrupt(vo mce_log(&m); done: irq_exit(); + enter_idle(); } static void __cpuinit intel_init_thermal(struct cpuinfo_x86 *c) diff --git a/arch/x86_64/kernel/process.c b/arch/x86_64/kernel/process.c --- a/arch/x86_64/kernel/process.c +++ b/arch/x86_64/kernel/process.c @@ -83,26 +83,18 @@ EXPORT_SYMBOL(idle_notifier_unregister); enum idle_state { CPU_IDLE, CPU_NOT_IDLE }; static DEFINE_PER_CPU(enum idle_state, idle_state) = CPU_NOT_IDLE; -void enter_idle(void) +void __enter_idle(void) { __get_cpu_var(idle_state) = CPU_IDLE; atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL); } -static void __exit_idle(void) +void __exit_idle(void) { __get_cpu_var(idle_state) = CPU_NOT_IDLE; atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL); } -/* Called from interrupts to signify idle end */ -void exit_idle(void) -{ - if (current->pid | read_pda(irqcount)) - return; - __exit_idle(); -} - /* * We use this if we don't have any better * idle routine.. @@ -218,7 +210,7 @@ void cpu_idle (void) idle = default_idle; if (cpu_is_offline(smp_processor_id())) play_dead(); - enter_idle(); + __enter_idle(); idle(); __exit_idle(); } diff --git a/arch/x86_64/kernel/smp.c b/arch/x86_64/kernel/smp.c --- a/arch/x86_64/kernel/smp.c +++ b/arch/x86_64/kernel/smp.c @@ -520,6 +520,7 @@ asmlinkage void smp_call_function_interr mb(); atomic_inc(&call_data->finished); } + enter_idle(); } int safe_smp_processor_id(void) diff --git a/include/asm-x86_64/idle.h b/include/asm-x86_64/idle.h --- a/include/asm-x86_64/idle.h +++ b/include/asm-x86_64/idle.h @@ -8,7 +8,25 @@ struct notifier_block; void idle_notifier_register(struct notifier_block *n); void idle_notifier_unregister(struct notifier_block *n); -void enter_idle(void); -void exit_idle(void); +void __exit_idle(void); +void __enter_idle(void); + +/* Called from interrupts to signify back to idle */ +static inline void enter_idle(void) +{ + /* bitwise or is intentional */ + if (current->pid | read_pda(irqcount)) + return; + __enter_idle(); +} + +/* Called from interrupts to signify idle end */ +static inline void exit_idle(void) +{ + /* bitwise or is intentional */ + if (current->pid | read_pda(irqcount)) + return; + __exit_idle(); +} #endif - To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/
| |