BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=0 stuck for 186s!
Showing busy workqueues and worker pools:
workqueue events: flags=0x100
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=16 refcnt=17
in-flight: 6120:switchdev_deferred_process_work switchdev_deferred_process_work
pending: 5*ovs_dp_masks_rebalance, 5*nsim_dev_hwstats_traffic_work, psi_avgs_work, vmstat_shepherd, psi_avgs_work, rht_deferred_worker
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=15 refcnt=16
pending: 5*nsim_dev_hwstats_traffic_work, 2*psi_avgs_work, perf_sched_delayed, psi_avgs_work, 3*ovs_dp_masks_rebalance, xfrm_state_gc_task, debugfs_reap_work, rht_deferred_worker
workqueue events_long: flags=0x100
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=7 refcnt=8
pending: 6*defense_work_handler, br_multicast_gc_work
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=5 refcnt=6
pending: 2*defense_work_handler, 2*br_multicast_gc_work, br_fdb_cleanup
workqueue events_unbound: flags=0x2
pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
pending: toggle_allocation_gate, flush_memcg_stats_dwork
workqueue events_unbound: flags=0x2
pwq 8: cpus=0-1 flags=0x4 nice=0 active=17 refcnt=18
in-flight: 8633:nsim_dev_trap_report_work ,8632:cfg80211_wiphy_work cfg80211_wiphy_work ,74:linkwatch_event
pending: 4*nsim_dev_trap_report_work, 3*cfg80211_wiphy_work, crng_reseed, 5*nsim_dev_trap_report_work
pwq 8: cpus=0-1 flags=0x4 nice=0 active=3 refcnt=4
pending: 3*cfg80211_wiphy_work
workqueue events_freezable: flags=0x104
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: update_balloon_stats_func
workqueue events_power_efficient: flags=0x180
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=5 refcnt=6
in-flight: 5938:gc_worker
pending: nf_flow_offload_work_gc, wg_ratelimiter_gc_entries, neigh_managed_work, neigh_periodic_work
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=8 refcnt=9
pending: hash_ipportnet6_gc, neigh_managed_work, neigh_periodic_work, do_cache_clean, check_lifetime, reg_check_chans_work, 2*check_lifetime
workqueue kvfree_rcu_reclaim: flags=0xa
pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
pending: 2*kfree_rcu_monitor
workqueue netns: flags=0x6000a
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=4
in-flight: 8634:cleanup_net
workqueue mm_percpu_wq: flags=0x8
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: vmstat_update
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: vmstat_update
workqueue kblockd: flags=0x18
pwq 7: cpus=1 node=0 flags=0x0 nice=-20 active=1 refcnt=2
pending: blk_mq_requeue_work
workqueue ipv6_addrconf: flags=0x6000a
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=11
in-flight: 6489:addrconf_verify_work
inactive: 7*addrconf_verify_work
workqueue krxrpcd: flags=0x2001a
pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
pending: rxrpc_peer_keepalive_worker
workqueue bat_events: flags=0x6000a
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=61
in-flight: 1168:batadv_iv_send_outstanding_bat_ogm_packet
inactive: 2*batadv_tt_purge, 7*batadv_mcast_mla_update, 5*batadv_purge_orig, 3*batadv_tt_purge, batadv_dat_purge, 2*batadv_tt_purge, 15*batadv_iv_send_outstanding_bat_ogm_packet, 3*batadv_bla_periodic_work, batadv_dat_purge, 4*batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_purge_orig, 3*batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, 3*batadv_dat_purge
workqueue wg-kex-wg0: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
pending: 2*wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
in-flight: 3531:wg_packet_handshake_send_worker
pending: wg_packet_handshake_send_worker
workqueue wg-kex-wg0: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
pending: 2*wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg2: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x128
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg0: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
in-flight: 3841:wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
pending: 2*wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg2: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
pending: 2*wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x128
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg0: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg2: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x128
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue hci3: flags=0x20012
pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
in-flight: 5834:hci_cmd_sync_work
workqueue wg-kex-wg0: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue hci0: flags=0x20012
pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
in-flight: 5845:hci_cmd_sync_work
workqueue wg-kex-wg0: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg2: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x128
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue hci1: flags=0x20012
pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
in-flight: 5844:hci_cmd_sync_work
workqueue wg-kex-wg1: flags=0x124
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg1: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=3 refcnt=4
pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg2: flags=0x124
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg2: flags=0x128
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=3 refcnt=4
pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker, wg_packet_tx_worker
pool 2: cpus=0 node=0 flags=0x0 nice=0 hung=20s workers=13 idle: 5971 12958 10 846 5974 9 5902 5955 5882 11985 5976
pool 8: cpus=0-1 flags=0x4 nice=0 hung=6s workers=32 idle: 8626 8637 1083 8629 6413 7527 3729 6476 12968 8631 8625 4416 6434 1153 8630 50 12 8627 134 13 8628 1014 8635 4587
pool 9: cpus=0-1 node=0 flags=0x4 nice=-20 hung=1s workers=6 idle: 5846 12961 12960
Showing backtraces of running workers in stalled CPU-bound worker pools:
NMI backtrace for cpu 0
CPU: 0 UID: 0 PID: 12949 Comm: syz.2.1723 Not tainted syzkaller #0 PREEMPT(full)
Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 10/25/2025
RIP: 0010:task_irq_context kernel/locking/lockdep.c:4683 [inline]
RIP: 0010:__lock_acquire+0x2ec/0x2cf0 kernel/locking/lockdep.c:5174
Code: 89 54 c7 20 4d 89 44 c7 08 4d 89 4c c7 10 48 8b 94 24 20 01 00 00 49 89 54 c7 18 65 8b 15 b8 85 e0 10 31 f6 85 d2 40 0f 95 c6 <31> d2 41 83 be 14 0b 00 00 00 0f 95 c2 c1 e2 0d c1 e6 0e 09 d6 8b
RSP: 0018:ffffc900000063d8 EFLAGS: 00000002
RAX: 0000000000000019 RBX: 0000000000000000 RCX: 0000000000000000
RDX: 0000000000000001 RSI: 0000000000000001 RDI: 0000000000000000
RBP: 0000000000000005 R08: ffffffff84993637 R09: ffffffff99a893f8
R10: ffffc90000006188 R11: fffff52000000c33 R12: 0000000000000000
R13: 0000000000000000 R14: ffff888051105b80 R15: ffff8880511066b0
FS: 00007f416303d6c0(0000) GS:ffff888125e34000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f95e99e5f40 CR3: 000000002a170000 CR4: 00000000003526f0
DR0: 0000200000000300 DR1: 0000000000000000 DR2: 0000200000000240
DR3: 0000200000000240 DR6: 00000000ffff0ff0 DR7: 0000000000000600
Call Trace:
lock_acquire+0x117/0x340 kernel/locking/lockdep.c:5868
__raw_spin_lock_irqsave include/linux/spinlock_api_smp.h:110 [inline]
_raw_spin_lock_irqsave+0xa7/0xf0 kernel/locking/spinlock.c:162
debug_object_activate+0x87/0x540 lib/debugobjects.c:818
debug_hrtimer_activate kernel/time/hrtimer.c:438 [inline]
debug_activate kernel/time/hrtimer.c:477 [inline]
enqueue_hrtimer+0x30/0x380 kernel/time/hrtimer.c:1081
__run_hrtimer kernel/time/hrtimer.c:1794 [inline]
__hrtimer_run_queues+0x636/0xc30 kernel/time/hrtimer.c:1841
hrtimer_interrupt+0x45b/0xaa0 kernel/time/hrtimer.c:1903
local_apic_timer_interrupt arch/x86/kernel/apic/apic.c:1045 [inline]
__sysvec_apic_timer_interrupt+0x102/0x3e0 arch/x86/kernel/apic/apic.c:1062
instr_sysvec_apic_timer_interrupt arch/x86/kernel/apic/apic.c:1056 [inline]
sysvec_apic_timer_interrupt+0x52/0xc0 arch/x86/kernel/apic/apic.c:1056
asm_sysvec_apic_timer_interrupt+0x1a/0x20 arch/x86/include/asm/idtentry.h:697
RIP: 0010:lock_acquire+0x16c/0x340 kernel/locking/lockdep.c:5872
Code: 00 00 00 00 9c 8f 44 24 30 f7 44 24 30 00 02 00 00 0f 85 cd 00 00 00 f7 44 24 08 00 02 00 00 74 01 fb 65 48 8b 05 f4 48 e0 10 <48> 3b 44 24 58 0f 85 e5 00 00 00 48 83 c4 60 5b 41 5c 41 5d 41 5e
RSP: 0018:ffffc90000006a78 EFLAGS: 00000206
RAX: a200fa409b272700 RBX: 0000000000000000 RCX: a200fa409b272700
RDX: 000000007a7315a3 RSI: ffffffff8d979ceb RDI: ffffffff8bc086e0
RBP: ffffffff81743f85 R08: ffffffff81743f85 R09: ffffffff8df41a20
R10: ffffc90000006708 R11: fffff52000000ce3 R12: 0000000000000002
R13: ffffffff8df41a20 R14: 0000000000000000 R15: 0000000000000246
rcu_lock_acquire include/linux/rcupdate.h:331 [inline]
rcu_read_lock include/linux/rcupdate.h:867 [inline]
class_rcu_constructor include/linux/rcupdate.h:1195 [inline]
unwind_next_frame+0xc2/0x23d0 arch/x86/kernel/unwind_orc.c:495
arch_stack_walk+0x11c/0x150 arch/x86/kernel/stacktrace.c:25
stack_trace_save+0x9c/0xe0 kernel/stacktrace.c:122
kasan_save_stack+0x3e/0x60 mm/kasan/common.c:56
kasan_record_aux_stack+0xbd/0xd0 mm/kasan/generic.c:556
kvfree_call_rcu+0x11b/0x480 mm/slab_common.c:1994
cfg80211_update_known_bss+0x830/0x1590 net/wireless/scan.c:1944
__cfg80211_bss_update+0x147/0x2110 net/wireless/scan.c:1989
cfg80211_inform_single_bss_data+0xbc8/0x1ad0 net/wireless/scan.c:2369
cfg80211_inform_bss_data+0x203/0x3b40 net/wireless/scan.c:3228
cfg80211_inform_bss_frame_data+0x3c7/0x710 net/wireless/scan.c:3319
ieee80211_bss_info_update+0x749/0x9e0 net/mac80211/scan.c:230
ieee80211_scan_rx+0x593/0xa20 net/mac80211/scan.c:359
__ieee80211_rx_handle_packet net/mac80211/rx.c:5282 [inline]
ieee80211_rx_list+0x2485/0x2f90 net/mac80211/rx.c:5539
ieee80211_rx_napi+0x1a8/0x3d0 net/mac80211/rx.c:5562
ieee80211_rx include/net/mac80211.h:5216 [inline]
ieee80211_handle_queued_frames+0xe8/0x1e0 net/mac80211/main.c:452
tasklet_action_common+0x358/0x550 kernel/softirq.c:925
handle_softirqs+0x27d/0x850 kernel/softirq.c:622
__do_softirq kernel/softirq.c:656 [inline]
invoke_softirq kernel/softirq.c:496 [inline]
__irq_exit_rcu+0xca/0x1f0 kernel/softirq.c:723
irq_exit_rcu+0x9/0x30 kernel/softirq.c:739
instr_sysvec_apic_timer_interrupt arch/x86/kernel/apic/apic.c:1056 [inline]
sysvec_apic_timer_interrupt+0xa6/0xc0 arch/x86/kernel/apic/apic.c:1056
asm_sysvec_apic_timer_interrupt+0x1a/0x20 arch/x86/include/asm/idtentry.h:697
RIP: 0010:native_save_fl arch/x86/include/asm/irqflags.h:26 [inline]
RIP: 0010:arch_local_save_flags arch/x86/include/asm/irqflags.h:109 [inline]
RIP: 0010:arch_irqs_disabled arch/x86/include/asm/irqflags.h:151 [inline]
RIP: 0010:preempt_schedule_irq+0xbe/0x150 kernel/sched/core.c:7191
Code: 03 10 00 00 00 74 64 bf 01 00 00 00 e8 8b d6 33 f6 e8 26 54 6c f6 fb bf 01 00 00 00 e8 1b a8 ff ff 48 c7 44 24 40 00 00 00 00 <9c> 8f 44 24 40 8b 44 24 40 fa a9 00 02 00 00 74 05 e8 9c 55 6c f6
RSP: 0018:ffffc900037571e0 EFLAGS: 00000246
RAX: a200fa409b272700 RBX: 0000000000000000 RCX: a200fa409b272700
RDX: 0000000000000000 RSI: ffffffff8bc086c0 RDI: ffffffff8bc08680
RBP: ffffc90003757280 R08: ffffffff8f822377 R09: 1ffffffff1f0446e
R10: dffffc0000000000 R11: fffffbfff1f0446f R12: 0000000000000000
R13: 0000000000000000 R14: dffffc0000000000 R15: 1ffff920006eae3c
----------------
Code disassembly (best guess):
0: 89 54 c7 20 mov %edx,0x20(%rdi,%rax,8)
4: 4d 89 44 c7 08 mov %r8,0x8(%r15,%rax,8)
9: 4d 89 4c c7 10 mov %r9,0x10(%r15,%rax,8)
e: 48 8b 94 24 20 01 00 mov 0x120(%rsp),%rdx
15: 00
16: 49 89 54 c7 18 mov %rdx,0x18(%r15,%rax,8)
1b: 65 8b 15 b8 85 e0 10 mov %gs:0x10e085b8(%rip),%edx # 0x10e085da
22: 31 f6 xor %esi,%esi
24: 85 d2 test %edx,%edx
26: 40 0f 95 c6 setne %sil
* 2a: 31 d2 xor %edx,%edx <-- trapping instruction
2c: 41 83 be 14 0b 00 00 cmpl $0x0,0xb14(%r14)
33: 00
34: 0f 95 c2 setne %dl
37: c1 e2 0d shl $0xd,%edx
3a: c1 e6 0e shl $0xe,%esi
3d: 09 d6 or %edx,%esi
3f: 8b .byte 0x8b