bisecting fixing commit since 7b6ae471e5415bc2bf4384a83ccb4c21de7824c0 building syzkaller on 29c3f20f99b6befe0395fe8e474b1e24240cac95 testing commit 7b6ae471e5415bc2bf4384a83ccb4c21de7824c0 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 5988f4234514afa725ad165bcc1a226ff36ef6524eb814d30e9c7c52ee589afd run #0: crashed: INFO: rcu detected stall in addrconf_rs_timer run #1: crashed: INFO: rcu detected stall in mac80211_hwsim_beacon run #2: crashed: INFO: rcu detected stall in ieee80211_iface_work run #3: crashed: BUG: soft lockup in mac80211_hwsim_beacon run #4: crashed: BUG: soft lockup in mac80211_hwsim_beacon run #5: crashed: INFO: rcu detected stall in devkmsg_write run #6: crashed: INFO: rcu detected stall in process_one_work run #7: crashed: INFO: rcu detected stall in corrupted run #8: crashed: INFO: rcu detected stall in addrconf_rs_timer run #9: crashed: INFO: rcu detected stall in smp_call_function run #10: crashed: INFO: rcu detected stall in mac80211_hwsim_beacon run #11: crashed: BUG: soft lockup in addrconf_rs_timer run #12: crashed: INFO: rcu detected stall in mac80211_hwsim_beacon run #13: crashed: BUG: soft lockup in smp_call_function run #14: crashed: INFO: rcu detected stall in mac80211_hwsim_beacon run #15: crashed: BUG: soft lockup in addrconf_rs_timer run #16: crashed: INFO: task hung in corrupted run #17: crashed: INFO: rcu detected stall in addrconf_rs_timer run #18: crashed: INFO: rcu detected stall in mac80211_hwsim_beacon run #19: crashed: INFO: rcu detected stall in smp_call_function testing current HEAD 0280e3c58f92b2fe0e8fbbdf8d386449168de4a8 testing commit 0280e3c58f92b2fe0e8fbbdf8d386449168de4a8 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 61dfadc71a3d3a78f9c710b35673a31d50affacc8154e4e4685252be1861e96f run #0: crashed: BUG: soft lockup in smp_call_function run #1: crashed: INFO: rcu detected stall in process_one_work run #2: crashed: INFO: rcu detected stall in gc_worker run #3: crashed: INFO: rcu detected stall in batadv_nc_worker run #4: crashed: INFO: task hung in corrupted run #5: crashed: INFO: rcu detected stall in macvlan_process_broadcast run #6: crashed: INFO: rcu detected stall in neigh_periodic_work run #7: crashed: INFO: rcu detected stall in batadv_tt_purge run #8: crashed: no output from test machine run #9: crashed: INFO: rcu detected stall in wg_ratelimiter_gc_entries revisions tested: 2, total time: 30m46.328977727s (build: 14m43.167837531s, test: 15m12.219022849s) the crash still happens on HEAD commit msg: Merge tag 'nfs-for-5.17-1' of git://git.linux-nfs.org/projects/anna/linux-nfs crash: INFO: rcu detected stall in wg_ratelimiter_gc_entries rcu: INFO: rcu_preempt detected stalls on CPUs/tasks: rcu: 1-...!: (1 GPs behind) idle=3e3/1/0x4000000000000000 softirq=7849/7854 fqs=1 (detected by 0, t=11906 jiffies, g=7213, q=779) Sending NMI from CPU 0 to CPUs 1: NMI backtrace for cpu 1 CPU: 1 PID: 25 Comm: kworker/1:1 Not tainted 5.17.0-rc1-syzkaller #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Workqueue: events_power_efficient wg_ratelimiter_gc_entries RIP: 0010:arch_local_irq_restore arch/x86/include/asm/irqflags.h:137 [inline] RIP: 0010:lock_is_held_type+0xf0/0x140 kernel/locking/lockdep.c:5684 Code: f0 41 0f 94 c5 48 c7 c7 a0 6a eb 88 e8 09 0c 00 00 b8 ff ff ff ff 65 0f c1 05 9c 80 83 77 83 f8 01 75 29 9c 58 f6 c4 02 75 3d <48> f7 04 24 00 02 00 00 74 01 fb 48 83 c4 08 44 89 e8 5b 5d 41 5c RSP: 0018:ffffc90000fd8ca8 EFLAGS: 00000046 RAX: 0000000000000046 RBX: 0000000000000004 RCX: 0000000000000001 RDX: 0000000000000000 RSI: ffffffff88eb6aa0 RDI: ffffffff89419360 RBP: ffffffff8ad75aa0 R08: 0000000000000000 R09: ffffffff8ca47917 R10: fffffbfff1948f22 R11: 0000000000000001 R12: ffff888010e3c000 R13: 0000000000000000 R14: 00000000ffffffff R15: ffff888010e3ca98 FS: 0000000000000000(0000) GS:ffff8880b9f00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 00007f521de04310 CR3: 000000001b49f000 CR4: 00000000003506e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 Call Trace: lock_is_held include/linux/lockdep.h:283 [inline] rcu_read_lock_sched_held+0x3a/0x70 kernel/rcu/update.c:125 trace_lock_release include/trace/events/lock.h:58 [inline] lock_release+0x522/0x720 kernel/locking/lockdep.c:5650 __raw_spin_unlock include/linux/spinlock_api_smp.h:141 [inline] _raw_spin_unlock+0x12/0x40 kernel/locking/spinlock.c:186 spin_unlock include/linux/spinlock.h:389 [inline] advance_sched+0x37a/0x920 net/sched/sch_taprio.c:763 __run_hrtimer kernel/time/hrtimer.c:1685 [inline] __hrtimer_run_queues+0x4d7/0xb00 kernel/time/hrtimer.c:1749 hrtimer_interrupt+0x2f5/0x780 kernel/time/hrtimer.c:1811 local_apic_timer_interrupt arch/x86/kernel/apic/apic.c:1086 [inline] __sysvec_apic_timer_interrupt+0x146/0x530 arch/x86/kernel/apic/apic.c:1103 sysvec_apic_timer_interrupt+0x8e/0xc0 arch/x86/kernel/apic/apic.c:1097 asm_sysvec_apic_timer_interrupt+0x12/0x20 arch/x86/include/asm/idtentry.h:638 RIP: 0010:lock_acquire+0x1ef/0x510 kernel/locking/lockdep.c:5607 Code: 1d af 7e 83 f8 01 0f 85 b4 02 00 00 9c 58 f6 c4 02 0f 85 9f 02 00 00 48 83 7c 24 08 00 74 01 fb 48 b8 00 00 00 00 00 fc ff df <48> 01 c3 48 c7 03 00 00 00 00 48 c7 43 08 00 00 00 00 48 8b 84 24 RSP: 0018:ffffc90001017be8 EFLAGS: 00000206 RAX: dffffc0000000000 RBX: 1ffff92000202f7f RCX: 000000000000c67f RDX: 1ffff110021c7943 RSI: ffffffff88eb6aa0 RDI: ffffffff89419360 RBP: 0000000000000001 R08: 0000000000000000 R09: ffffffff8f0c696f R10: fffffbfff1e18d2d R11: 0000000000000001 R12: 0000000000000000 R13: 0000000000000000 R14: ffffffff8b9ffd38 R15: 0000000000000000 __raw_spin_lock include/linux/spinlock_api_smp.h:133 [inline] _raw_spin_lock+0x2a/0x40 kernel/locking/spinlock.c:154 spin_lock include/linux/spinlock.h:349 [inline] wg_ratelimiter_gc_entries+0x4a/0x370 drivers/net/wireguard/ratelimiter.c:63 process_one_work+0x879/0x1410 kernel/workqueue.c:2307 worker_thread+0x5a0/0xf60 kernel/workqueue.c:2454 kthread+0x299/0x340 kernel/kthread.c:377 ret_from_fork+0x1f/0x30 arch/x86/entry/entry_64.S:295 rcu: rcu_preempt kthread timer wakeup didn't happen for 3510 jiffies! g7213 f0x0 RCU_GP_WAIT_FQS(5) ->state=0x402 rcu: Possible timer handling issue on cpu=0 timer-softirq=3583 rcu: rcu_preempt kthread starved for 3511 jiffies! g7213 f0x0 RCU_GP_WAIT_FQS(5) ->state=0x402 ->cpu=0 rcu: Unless rcu_preempt kthread gets sufficient CPU time, OOM is now expected behavior. rcu: RCU grace-period kthread stack dump: task:rcu_preempt state:I stack:28816 pid: 14 ppid: 2 flags:0x00004000 Call Trace: context_switch kernel/sched/core.c:4986 [inline] __schedule+0x90d/0x2680 kernel/sched/core.c:6295 schedule+0xd2/0x260 kernel/sched/core.c:6368 schedule_timeout+0x11d/0x250 kernel/time/timer.c:1881 rcu_gp_fqs_loop+0x186/0x810 kernel/rcu/tree.c:1963 rcu_gp_kthread+0x1de/0x320 kernel/rcu/tree.c:2136 kthread+0x299/0x340 kernel/kthread.c:377 ret_from_fork+0x1f/0x30 arch/x86/entry/entry_64.S:295 rcu: Stack dump where RCU GP kthread last ran: NMI backtrace for cpu 0 CPU: 0 PID: 44 Comm: kworker/u4:2 Not tainted 5.17.0-rc1-syzkaller #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Workqueue: events_unbound toggle_allocation_gate Call Trace: __dump_stack lib/dump_stack.c:88 [inline] dump_stack_lvl+0x57/0x7d lib/dump_stack.c:106 nmi_cpu_backtrace.cold+0x30/0xc0 lib/nmi_backtrace.c:111 nmi_trigger_cpumask_backtrace+0x11f/0x170 lib/nmi_backtrace.c:62 trigger_single_cpu_backtrace include/linux/nmi.h:164 [inline] rcu_check_gp_kthread_starvation.cold+0x1fb/0x200 kernel/rcu/tree_stall.h:458 print_other_cpu_stall kernel/rcu/tree_stall.h:563 [inline] check_cpu_stall kernel/rcu/tree_stall.h:706 [inline] rcu_pending kernel/rcu/tree.c:3919 [inline] rcu_sched_clock_irq+0x1f7c/0x2150 kernel/rcu/tree.c:2617 update_process_times+0x13b/0x1c0 kernel/time/timer.c:1785 tick_sched_handle+0x6f/0x130 kernel/time/tick-sched.c:226 tick_sched_timer+0x132/0x210 kernel/time/tick-sched.c:1428 __run_hrtimer kernel/time/hrtimer.c:1685 [inline] __hrtimer_run_queues+0x18a/0xb00 kernel/time/hrtimer.c:1749 hrtimer_interrupt+0x2f5/0x780 kernel/time/hrtimer.c:1811 local_apic_timer_interrupt arch/x86/kernel/apic/apic.c:1086 [inline] __sysvec_apic_timer_interrupt+0x146/0x530 arch/x86/kernel/apic/apic.c:1103 sysvec_apic_timer_interrupt+0x8e/0xc0 arch/x86/kernel/apic/apic.c:1097 asm_sysvec_apic_timer_interrupt+0x12/0x20 arch/x86/include/asm/idtentry.h:638 RIP: 0010:csd_lock_wait kernel/smp.c:440 [inline] RIP: 0010:smp_call_function_many_cond+0x271/0xa50 kernel/smp.c:969 Code: 38 d0 7c 08 84 d2 0f 85 47 05 00 00 8b 43 08 a8 01 74 2e 48 89 ca 49 89 cf 48 c1 ea 03 41 83 e7 07 4c 01 e2 41 83 c7 03 f3 90 <0f> b6 02 41 38 c7 7c 08 84 c0 0f 85 d4 04 00 00 8b 43 08 a8 01 75 RSP: 0018:ffffc90001157a60 EFLAGS: 00000202 RAX: 0000000000000011 RBX: ffff8880b9f3f360 RCX: ffff8880b9f3f368 RDX: ffffed10173e7e6d RSI: ffff8880b9e3ad88 RDI: ffffffff8a8618a8 RBP: ffff8880b9e3ad80 R08: 0000000000000001 R09: ffffffff8f0c6887 R10: 0000000000000001 R11: 0000000000000001 R12: dffffc0000000000 R13: ffff8880b9e3ad88 R14: ffffed10173c75b0 R15: 0000000000000003 on_each_cpu_cond_mask+0x3f/0x70 kernel/smp.c:1135 on_each_cpu include/linux/smp.h:71 [inline] text_poke_sync arch/x86/kernel/alternative.c:1112 [inline] text_poke_bp_batch+0x319/0x520 arch/x86/kernel/alternative.c:1353 text_poke_flush arch/x86/kernel/alternative.c:1470 [inline] text_poke_flush arch/x86/kernel/alternative.c:1467 [inline] text_poke_finish+0x16/0x30 arch/x86/kernel/alternative.c:1477 arch_jump_label_transform_apply+0x13/0x20 arch/x86/kernel/jump_label.c:146 static_key_disable_cpuslocked+0x100/0x160 kernel/jump_label.c:207 static_key_disable+0x11/0x20 kernel/jump_label.c:215 toggle_allocation_gate mm/kfence/core.c:748 [inline] toggle_allocation_gate+0x154/0x310 mm/kfence/core.c:726 process_one_work+0x879/0x1410 kernel/workqueue.c:2307 worker_thread+0x5a0/0xf60 kernel/workqueue.c:2454 kthread+0x299/0x340 kernel/kthread.c:377 ret_from_fork+0x1f/0x30 arch/x86/entry/entry_64.S:295 ---------------- Code disassembly (best guess): 0: f0 41 0f 94 c5 lock sete %r13b 5: 48 c7 c7 a0 6a eb 88 mov $0xffffffff88eb6aa0,%rdi c: e8 09 0c 00 00 callq 0xc1a 11: b8 ff ff ff ff mov $0xffffffff,%eax 16: 65 0f c1 05 9c 80 83 xadd %eax,%gs:0x7783809c(%rip) # 0x778380ba 1d: 77 1e: 83 f8 01 cmp $0x1,%eax 21: 75 29 jne 0x4c 23: 9c pushfq 24: 58 pop %rax 25: f6 c4 02 test $0x2,%ah 28: 75 3d jne 0x67 * 2a: 48 f7 04 24 00 02 00 testq $0x200,(%rsp) <-- trapping instruction 31: 00 32: 74 01 je 0x35 34: fb sti 35: 48 83 c4 08 add $0x8,%rsp 39: 44 89 e8 mov %r13d,%eax 3c: 5b pop %rbx 3d: 5d pop %rbp 3e: 41 5c pop %r12