lkml.org 
[lkml]   [2008]   [Jun]   [22]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    Date
    Subject[PATCH] softirq softlockup debugging
    From
    Hi,

    I'm debugging a problem with a softirq that gets stuck for a long time,
    so I wrote this patch to help find out what's going wrong.

    I actually think it can be useful in general as well, see for example
    http://www.kerneloops.org/search.php?search=__do_softirq&btnG=Function+Search

    ..and these cases are virtually impossible to debug since we don't know
    anything about *what* it was that got stuck. (The NMI watchdog could
    help, though.)

    The patch is #ifdef-ugly, I know... Suggestions are welcome.


    Vegard


    From: Vegard Nossum <vegard.nossum@gmail.com>
    Date: Sun, 22 Jun 2008 14:12:31 +0200
    Subject: [PATCH] softirq softlockup debugging

    From the Kconfig: If a softlockup happens in a softirq, the softlockup
    stack trace is utterly unhelpful; it will only show the stack up to
    __do_softirq(), since this is where interrupts are reenabled.

    This patch adds a line to the output of the softlockup report which
    contains the address of the function that was last scheduled to run in
    a softirq.

    Signed-off-by: Vegard Nossum <vegard.nossum@gmail.com>
    ---
    include/linux/interrupt.h | 3 +++
    kernel/softirq.c | 13 +++++++++++++
    kernel/softlockup.c | 6 ++++++
    lib/Kconfig.debug | 10 ++++++++++
    4 files changed, 32 insertions(+), 0 deletions(-)

    diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
    index f1fc747..97d47cf 100644
    --- a/include/linux/interrupt.h
    +++ b/include/linux/interrupt.h
    @@ -296,6 +296,9 @@ extern void softirq_init(void);
    extern void raise_softirq_irqoff(unsigned int nr);
    extern void raise_softirq(unsigned int nr);

    +#ifdef CONFIG_SOFTLOCKUP_SOFTIRQ_DEBUG
    +extern void *get_last_softirq_action(int cpu);
    +#endif

    /* Tasklets --- multithreaded analogue of BHs.

    diff --git a/kernel/softirq.c b/kernel/softirq.c
    index 36e0617..b49899a 100644
    --- a/kernel/softirq.c
    +++ b/kernel/softirq.c
    @@ -196,6 +196,15 @@ void local_bh_enable_ip(unsigned long ip)
    }
    EXPORT_SYMBOL(local_bh_enable_ip);

    +#ifdef CONFIG_SOFTLOCKUP_SOFTIRQ_DEBUG
    +static DEFINE_PER_CPU(void *, last_softirq_action);
    +
    +void *get_last_softirq_action(int cpu)
    +{
    + return per_cpu(last_softirq_action, cpu);
    +}
    +#endif
    +
    /*
    * We restart softirq processing MAX_SOFTIRQ_RESTART times,
    * and we fall back to softirqd after that.
    @@ -231,6 +240,10 @@ restart:

    do {
    if (pending & 1) {
    +#ifdef CONFIG_SOFTLOCKUP_SOFTIRQ_DEBUG
    + per_cpu(last_softirq_action, cpu) = h->action;
    +#endif
    +
    h->action(h);
    rcu_bh_qsctr_inc(cpu);
    }
    diff --git a/kernel/softlockup.c b/kernel/softlockup.c
    index c828c23..2bf4fa1 100644
    --- a/kernel/softlockup.c
    +++ b/kernel/softlockup.c
    @@ -10,8 +10,10 @@
    #include <linux/cpu.h>
    #include <linux/nmi.h>
    #include <linux/init.h>
    +#include <linux/interrupt.h>
    #include <linux/delay.h>
    #include <linux/freezer.h>
    +#include <linux/kallsyms.h>
    #include <linux/kthread.h>
    #include <linux/notifier.h>
    #include <linux/module.h>
    @@ -120,6 +122,10 @@ void softlockup_tick(void)
    printk(KERN_ERR "BUG: soft lockup - CPU#%d stuck for %lus! [%s:%d]\n",
    this_cpu, now - touch_timestamp,
    current->comm, task_pid_nr(current));
    +#ifdef CONFIG_SOFTLOCKUP_SOFTIRQ_DEBUG
    + print_symbol(KERN_ERR "Last softirq was %s\n",
    + (unsigned long) get_last_softirq_action(this_cpu));
    +#endif
    if (regs)
    show_regs(regs);
    else
    diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
    index d2099f4..19a7dfc 100644
    --- a/lib/Kconfig.debug
    +++ b/lib/Kconfig.debug
    @@ -159,6 +159,16 @@ config DETECT_SOFTLOCKUP
    can be detected via the NMI-watchdog, on platforms that
    support it.)

    +config SOFTLOCKUP_SOFTIRQ_DEBUG
    + bool "Debug softirq lockups"
    + depends on DETECT_SOFTLOCKUP
    + default n
    + help
    + If a softlockup happens in a softirq, the softlockup
    + stack trace is utterly unhelpful; it will only show the
    + stack up to __do_softirq(), since this is where interrupts
    + are reenabled.
    +
    config SCHED_DEBUG
    bool "Collect scheduler debugging info"
    depends on DEBUG_KERNEL && PROC_FS
    --
    1.5.4.1


    \
     
     \ /
      Last update: 2008-06-22 14:33    [W:0.026 / U:30.048 seconds]
    ©2003-2016 Jasper Spaans. hosted at Digital OceanAdvertise on this site