lkml.org 
[lkml]   [2016]   [Mar]   [3]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
/
Date
SubjectBUG: NMI Watchdog detected LOCKUP on CPU19, ip ffffffff814c5aee, registers:
From
hi,
my server has panic many times, how to fix it?
more info: https://bugzilla.kernel.org/show_bug.cgi?id=111721

sys:
KERNEL: /usr/lib/debug/lib/modules/2.6.32-220.7.1.el6.3.0.1.1.x86_64/vmlinux
DUMPFILE: vmcore [PARTIAL DUMP]
CPUS: 24
DATE: Tue Feb 23 17:51:42 2016
UPTIME: 133 days, 03:29:28
LOAD AVERAGE: 16.42, 6.35, 4.31
TASKS: 3074
NODENAME: openstack583
RELEASE: 2.6.32-220.7.1.el6.3.0.1.1.x86_64
VERSION: #1 SMP Mon Feb 2 13:39:59 CST 2015
MACHINE: x86_64 (2599 Mhz)
MEMORY: 96 GB
PANIC: ""
PID: 19466
COMMAND: "vhost-19464"
TASK: ffff880c5e229850 [THREAD_INFO: ffff880c5f8e8000]
CPU: 19
STATE: TASK_RUNNING (NMI)

log:
HTB: quantum of class 18943 is small. Consider r2q change.
HTB: quantum of class 10001 is small. Consider r2q change.
kvm: 17981: cpu0 unhandled wrmsr: 0x198 data 0
kvm: 17981: cpu1 unhandled wrmsr: 0x198 data 0
BUG: NMI Watchdog detected LOCKUP on CPU19, ip ffffffff814c5aee, registers:
CPU 19
Modules linked in: act_police cls_u32 sch_ingress sch_htb ip6_tables
iptable_filter ip_tables ebtable_nat ebtables stp llc openvswitch(U)
libcrc32c ttm vhost_net macvtap macvlan tun kvm_intel kvm wmi acpi_pad
microcode i2c_i801 iTCO_wdt iTCO_vendor_support sg igb hwmon dca
i2c_algo_bit i2c_core shpchp ext4 mbcache jbd2 isci libsas
scsi_transport_sas sd_mod crc_t10dif ahci libata megaraid_sas scsi_mod
dm_mirror dm_region_hash dm_log dm_mod [last unloaded: scsi_wait_scan]

Pid: 19466, comm: vhost-19464 Not tainted
2.6.32-220.7.1.el6.3.0.1.1.x86_64 #1 Sugon I620-G10/X9DR3-F
RIP: 0010:[<ffffffff814c5aee>] [<ffffffff814c5aee>] _spin_lock+0x1e/0x30
RSP: 0018:ffff880caa6e3eb8 EFLAGS: 00000016
RAX: 0000000000001c38 RBX: ffff880c5e897718 RCX: 0000000000000000
RDX: 0000000000003eb0 RSI: ffff88002834fda8 RDI: ffff88002834fd60
RBP: ffff880caa6e3eb8 R08: ffff880caa6edb60 R09: 0000000000000001
R10: 0000000000000018 R11: ffff880c5f610b80 R12: ffff88002834fda8
R13: ffff88002834fd60 R14: 0000000000000000 R15: 0000000000000000
FS: 0000000000000000(0000) GS:ffff880caa6e0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b
CR2: 0000000000640cf0 CR3: 000000185e462000 CR4: 00000000001426e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Process vhost-19464 (pid: 19466, threadinfo ffff880c5f8e8000, task
ffff880c5e229850)
Stack:
ffff880caa6e3f08 ffffffff8108d487 ffff880caa6e3ed8 ffffffff810129f9
<0> ffff880caa6e3f08 ffff880caa6efd68 ffff880caa6e3f48 ffff880caa6efd60
<0> 0000000000000001 7fffffffffffffff ffff880caa6e3f88 ffffffff8108d7f6
Call Trace:
<IRQ>
[<ffffffff8108d487>] __run_hrtimer+0x97/0x170
[<ffffffff810129f9>] ? read_tsc+0x9/0x20
[<ffffffff8108d7f6>] hrtimer_interrupt+0xe6/0x250
[<ffffffff814cb76b>] smp_apic_timer_interrupt+0x6b/0x9b
[<ffffffff8100bbd3>] apic_timer_interrupt+0x13/0x20
<EOI>
[<ffffffff81265f8d>] ? copy_user_generic_string+0x2d/0x40
[<ffffffff813fa9db>] ? memcpy_fromiovecend+0x7b/0xa0
[<ffffffff813fbf0d>] skb_copy_datagram_from_iovec+0x5d/0x2b0
[<ffffffffa028764d>] tun_sendmsg+0x10d/0x4ec [tun]
[<ffffffffa02a1f25>] handle_tx+0x275/0x5e0 [vhost_net]
[<ffffffffa02a22c5>] handle_tx_kick+0x15/0x20 [vhost_net]
[<ffffffffa029f4fc>] vhost_worker+0xbc/0x140 [vhost_net]
[<ffffffffa029f440>] ? vhost_worker+0x0/0x140 [vhost_net]
[<ffffffff81088b16>] kthread+0x96/0xa0
[<ffffffff8100c10a>] child_rip+0xa/0x20
[<ffffffff81088a80>] ? kthread+0x0/0xa0
[<ffffffff8100c100>] ? child_rip+0x0/0x20
Code: 00 00 00 01 74 05 e8 82 18 da ff c9 c3 55 48 89 e5 0f 1f 44 00
00 b8 00 00 01 00 f0 0f c1 07 0f b7 d0 c1 e8 10 39 c2 74 0e f3 90 <0f>
b7 17 eb f5 83 3f 00 75 f4 eb df c9 c3 0f 1f 40 00 55 48 89

crash> bt
PID: 19466 TASK: ffff880c5e229850 CPU: 19 COMMAND: "vhost-19464"
#0 [ffff880caa6e7d50] machine_kexec at ffffffff8103229b
#1 [ffff880caa6e7db0] crash_kexec at ffffffff810b0392
#2 [ffff880caa6e7e80] oops_end at ffffffff814c6ca0
#3 [ffff880caa6e7eb0] die_nmi at ffffffff814c6e49
#4 [ffff880caa6e7ee0] nmi_watchdog_tick at ffffffff814c752a
#5 [ffff880caa6e7f10] do_nmi at ffffffff814c69c1
#6 [ffff880caa6e7f50] nmi at ffffffff814c6280
[exception RIP: _spin_lock+30]
RIP: ffffffff814c5aee RSP: ffff880caa6e3eb8 RFLAGS: 00000016
RAX: 0000000000001c38 RBX: ffff880c5e897718 RCX: 0000000000000000
RDX: 0000000000003eb0 RSI: ffff88002834fda8 RDI: ffff88002834fd60
RBP: ffff880caa6e3eb8 R8: ffff880caa6edb60 R9: 0000000000000001
R10: 0000000000000018 R11: ffff880c5f610b80 R12: ffff88002834fda8
R13: ffff88002834fd60 R14: 0000000000000000 R15: 0000000000000000
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#7 [ffff880caa6e3eb8] _spin_lock at ffffffff814c5aee
#8 [ffff880caa6e3ec0] __run_hrtimer at ffffffff8108d487
#9 [ffff880caa6e3f10] hrtimer_interrupt at ffffffff8108d7f6
#10 [ffff880caa6e3f90] smp_apic_timer_interrupt at ffffffff814cb76b
#11 [ffff880caa6e3fb0] apic_timer_interrupt at ffffffff8100bbd3
--- <IRQ stack> ---
#12 [ffff880c5f8e9bc8] apic_timer_interrupt at ffffffff8100bbd3
[exception RIP: copy_user_generic_string+45]
RIP: ffffffff81265f8d RSP: ffff880c5f8e9c78 RFLAGS: 00010206
RAX: ffff880c5f8e8000 RBX: ffff880c5f8e9ca0 RCX: 0000000000000003
RDX: 0000000000000006 RSI: 00007fdd33b2332f RDI: ffff88185edd7c40
RBP: ffffffff8100bbce R8: 000000000000005c R9: ffff88118dc513c0
R10: 0000000000000018 R11: ffff880c5f610b80 R12: ffff880c5f8e9c30
R13: ffff880c7fc162c0 R14: 00000000ffffffff R15: 00000000000004d0
ORIG_RAX: ffffffffffffff10 CS: 0010 SS: 0018
#13 [ffff880c5f8e9c78] memcpy_fromiovecend at ffffffff813fa9db
#14 [ffff880c5f8e9ca8] skb_copy_datagram_from_iovec at ffffffff813fbf0d
#15 [ffff880c5f8e9d28] tun_sendmsg at ffffffffa028764d [tun]
#16 [ffff880c5f8e9da8] handle_tx at ffffffffa02a1f25 [vhost_net]
#17 [ffff880c5f8e9e88] handle_tx_kick at ffffffffa02a22c5 [vhost_net]
#18 [ffff880c5f8e9e98] vhost_worker at ffffffffa029f4fc [vhost_net]
#19 [ffff880c5f8e9ee8] kthread at ffffffff81088b16
#20 [ffff880c5f8e9f48] kernel_thread at ffffffff8100c10a
[exception RIP: child_rip]
RIP: ffffffff8100c100 RSP: ffff880c5f8e9f58 RFLAGS: 00000200
RAX: 0000000000000000 RBX: ffff88185d0bfbf8 RCX: 0000000000000000
RDX: 0000000000000611 RSI: ffff88185d0bfbf8 RDI: ffffffff81088a80
RBP: ffff880c61e11f40 R8: ffff880c61e10000 R9: 00000000ffffffff
R10: 0000000000000000 R11: 0000000000000000 R12: ffff88185d0bfc30
R13: ffffffff81a86768 R14: ffff880c61df4790 R15: ffff880c61df4790
ORIG_RAX: 0000000000000000 CS: 0010 SS: 0018
bt: WARNING: possibly bogus exception frame
crash> l *_spin_lock+30
0xffffffff814c5aee is in _spin_lock
(/usr/src/debug/kernel-kernel_32_hadoop/linux-2.6.32-220.7.1.el6.3.0.1.1.x86_64/arch/x86/include/asm/spinlock.h:127).
122 static __always_inline void __ticket_spin_lock(raw_spinlock_t *lock)
123 {
124 int inc;
125 int tmp;
126
127 asm volatile("1:\t\n"
128 "mov $0x10000, %0\n\t"
129 LOCK_PREFIX "xaddl %0, %1\n"
130 "movzwl %w0, %2\n\t"
131 "shrl $16, %0\n\t"
crash> l *copy_user_generic_string+45
0xffffffff81265f8d is at arch/x86/lib/copy_user_64.S:248.
243 jb 2f /* less than 8 bytes, go to byte copy loop */
244 ALIGN_DESTINATION
245 movl %edx,%ecx
246 shrl $3,%ecx
247 andl $7,%edx
248 1: rep
249 movsq
250 2: movl %edx,%ecx
251 3: rep
252 movsb

\
 
 \ /
  Last update: 2016-03-03 07:02    [W:0.037 / U:0.224 seconds]
©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site