bisecting fixing commit since 4fa640dc52302b5e62b01b05c755b055549633ae
building syzkaller on d88894e6773ab63ac8b3f4b2edbae88290aaf0d6
testing commit 4fa640dc52302b5e62b01b05c755b055549633ae with gcc (GCC) 8.1.0
kernel signature: 3ac696cb96fc1016221e4e9e68021dd277298e1feecfe67bd21a12a3b4a4db05
run #0: crashed: BUG: workqueue lockup
run #1: crashed: BUG: workqueue lockup
run #2: crashed: INFO: rcu detected stall in do_idle
run #3: crashed: INFO: rcu detected stall in do_idle
run #4: crashed: INFO: rcu detected stall in do_idle
run #5: crashed: INFO: rcu detected stall in corrupted
run #6: crashed: INFO: rcu detected stall in do_idle
run #7: crashed: BUG: workqueue lockup
run #8: crashed: INFO: rcu detected stall in do_idle
run #9: crashed: INFO: rcu detected stall in do_idle
testing current HEAD 1e2a199f6ccdc15cf111d68d212e2fd4ce65682e
testing commit 1e2a199f6ccdc15cf111d68d212e2fd4ce65682e with gcc (GCC) 8.1.0
kernel signature: 7b9007e1a38a7ef31f3b4eea8c146ae6b57627105aa42475feabb35f36499743
run #0: crashed: BUG: workqueue lockup
run #1: crashed: BUG: workqueue lockup
run #2: crashed: BUG: workqueue lockup
run #3: crashed: BUG: workqueue lockup
run #4: crashed: BUG: workqueue lockup
run #5: crashed: BUG: workqueue lockup
run #6: crashed: BUG: workqueue lockup
run #7: crashed: BUG: workqueue lockup
run #8: crashed: INFO: rcu detected stall in smp_call_function
run #9: crashed: BUG: workqueue lockup
revisions tested: 2, total time: 25m1.05420545s (build: 10m44.946878038s, test: 13m30.420320476s)
the crash still happens on HEAD
commit msg: Merge tag 'spi-fix-v5.11-rc4' of git://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi
crash: BUG: workqueue lockup
BUG: workqueue lockup - pool cpus=0 node=0 flags=0x0 nice=0 stuck for 80s!
Showing busy workqueues and worker pools:
workqueue events: flags=0x0
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
in-flight: 7387:free_ipc free_ipc, 29:linkwatch_event
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=12/256 refcnt=13
pending: kfree_rcu_monitor, nsim_dev_trap_report_work, nsim_dev_trap_report_work, nsim_dev_trap_report_work, nsim_dev_trap_report_work, psi_avgs_work, ovs_dp_masks_rebalance, ovs_dp_masks_rebalance, ovs_dp_masks_rebalance, ovs_dp_masks_rebalance, vmstat_shepherd, cache_reap
workqueue events_long: flags=0x0
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=4/256 refcnt=5
pending: defense_work_handler, defense_work_handler, defense_work_handler, defense_work_handler
rcu: INFO: rcu_preempt self-detected stall on CPU
rcu: 0-....: (1 GPs behind) idle=6f2/0/0x3 softirq=9448/9450 fqs=3
(t=17576 jiffies g=3905 q=520)
NMI backtrace for cpu 0
CPU: 0 PID: 0 Comm: swapper/0 Not tainted 5.11.0-rc4-syzkaller #0
Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011
Call Trace:
__dump_stack lib/dump_stack.c:79 [inline]
dump_stack+0xa3/0xc8 lib/dump_stack.c:120
nmi_cpu_backtrace.cold.8+0x53/0x6d lib/nmi_backtrace.c:105
nmi_trigger_cpumask_backtrace+0xd5/0xf0 lib/nmi_backtrace.c:62
trigger_single_cpu_backtrace include/linux/nmi.h:164 [inline]
rcu_dump_cpu_stacks+0xa2/0xce kernel/rcu/tree_stall.h:337
print_cpu_stall kernel/rcu/tree_stall.h:569 [inline]
check_cpu_stall kernel/rcu/tree_stall.h:643 [inline]
rcu_pending kernel/rcu/tree.c:3751 [inline]
rcu_sched_clock_irq.cold.97+0x61/0x5eb kernel/rcu/tree.c:2580
update_process_times+0x8e/0xc0 kernel/time/timer.c:1782
tick_sched_handle.isra.21+0x1a/0x50 kernel/time/tick-sched.c:226
tick_sched_timer+0x6a/0x80 kernel/time/tick-sched.c:1369
__run_hrtimer kernel/time/hrtimer.c:1519 [inline]
__hrtimer_run_queues+0x1e3/0x4f0 kernel/time/hrtimer.c:1583
hrtimer_interrupt+0xf9/0x210 kernel/time/hrtimer.c:1645
local_apic_timer_interrupt arch/x86/kernel/apic/apic.c:1085 [inline]
__sysvec_apic_timer_interrupt+0x8e/0x290 arch/x86/kernel/apic/apic.c:1102
run_sysvec_on_irqstack_cond arch/x86/include/asm/irq_stack.h:91 [inline]
sysvec_apic_timer_interrupt+0x52/0xf0 arch/x86/kernel/apic/apic.c:1096
asm_sysvec_apic_timer_interrupt+0x12/0x20 arch/x86/include/asm/idtentry.h:628
RIP: 0010:arch_local_irq_restore arch/x86/include/asm/paravirt.h:653 [inline]
RIP: 0010:__raw_spin_unlock_irqrestore include/linux/spinlock_api_smp.h:160 [inline]
RIP: 0010:_raw_spin_unlock_irqrestore+0x32/0x70 kernel/locking/spinlock.c:191
Code: 53 48 89 f3 48 8b 74 24 10 e8 5a 73 11 fe 48 89 ef e8 f2 b6 11 fe f6 c7 02 75 2c 48 83 3d cd 95 3a 01 00 74 31 48 89 df 57 9d <0f> 1f 44 00 00 bf 01 00 00 00 e8 2f f9 0d fe 65 8b 05 28 39 ef 7c
RSP: 0018:ffffc90000003e08 EFLAGS: 00000282
RAX: 000000000003b63c RBX: 0000000000000282 RCX: 0000000000000002
RDX: 0000000000000000 RSI: ffffffff83f3e2eb RDI: 0000000000000282
RBP: ffff888237c2d640 R08: 0000000000000001 R09: 0000000000000001
R10: ffffc90000003c10 R11: ffffc90000003c08 R12: 0000000000000282
R13: ffffffff83eb8261 R14: 00000000ffffb7b9 R15: ffff888237c32200
show_workqueue_state.cold.58+0xa4/0x2da kernel/workqueue.c:4798
wq_watchdog_timer_fn+0x224/0x230 kernel/workqueue.c:5815
call_timer_fn+0xa5/0x300 kernel/time/timer.c:1417
expire_timers kernel/time/timer.c:1462 [inline]
__run_timers kernel/time/timer.c:1731 [inline]
run_timer_softirq+0x4d2/0x570 kernel/time/timer.c:1746
__do_softirq+0xeb/0x52d kernel/softirq.c:343
asm_call_irq_on_stack+0xf/0x20
__run_on_irqstack arch/x86/include/asm/irq_stack.h:26 [inline]
run_on_irqstack_cond arch/x86/include/asm/irq_stack.h:77 [inline]
do_softirq_own_stack+0x7c/0xa0 arch/x86/kernel/irq_64.c:77
invoke_softirq kernel/softirq.c:226 [inline]
__irq_exit_rcu kernel/softirq.c:420 [inline]
irq_exit_rcu+0xec/0x110 kernel/softirq.c:432
sysvec_apic_timer_interrupt+0x57/0xf0 arch/x86/kernel/apic/apic.c:1096
asm_sysvec_apic_timer_interrupt+0x12/0x20 arch/x86/include/asm/idtentry.h:628
RIP: 0010:native_safe_halt+0xe/0x10 arch/x86/include/asm/irqflags.h:61
Code: 5b c3 65 48 8b 04 25 c0 7e 01 00 f0 80 48 02 20 48 8b 00 a8 08 75 c3 e9 7c ff ff ff e9 07 00 00 00 0f 00 2d 6c da 4f 00 fb f4 90 e9 07 00 00 00 0f 00 2d 5c da 4f 00 f4 c3 cc cc e8 db 07 ff
RSP: 0018:ffffffff84403e38 EFLAGS: 00000286
RAX: 000000000003a7b9 RBX: ffff888100c7c400 RCX: 00000000ffffffff
RDX: 0000000000000000 RSI: ffffffff83f3e2eb RDI: ffffffff83f7b9ae
RBP: ffffffff8472cdc0 R08: 0000000000000001 R09: 0000000000000001
R10: ffff888237c2c684 R11: ffff888237c2c664 R12: 0000000000000001
R13: ffff888100ef2464 R14: 0000000000000001 R15: 0000000000000000
arch_safe_halt arch/x86/include/asm/paravirt.h:150 [inline]
acpi_safe_halt drivers/acpi/processor_idle.c:110 [inline]
acpi_idle_do_entry+0x50/0x90 drivers/acpi/processor_idle.c:516
acpi_idle_enter+0xa0/0xf0 drivers/acpi/processor_idle.c:647
cpuidle_enter_state+0x94/0x520 drivers/cpuidle/cpuidle.c:237
cpuidle_enter+0x24/0x40 drivers/cpuidle/cpuidle.c:351
call_cpuidle kernel/sched/idle.c:158 [inline]
cpuidle_idle_call kernel/sched/idle.c:239 [inline]
do_idle+0x2dc/0x350 kernel/sched/idle.c:299
cpu_startup_entry+0x14/0x20 kernel/sched/idle.c:396
start_kernel+0x4f2/0x511 init/main.c:1061
secondary_startup_64_no_verify+0xb0/0xbb