BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=0 stuck for 186s!
Showing busy workqueues and worker pools:
workqueue events: flags=0x0
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=4 refcnt=5
in-flight: 5922:nsim_fib_event_work ,5912:free_ipc
pending: nsim_dev_hwstats_traffic_work, vmstat_shepherd
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=16 refcnt=17
pending: 4*nsim_dev_hwstats_traffic_work, 3*psi_avgs_work, 2*ovs_dp_masks_rebalance, psi_avgs_work, 2*bpf_prog_free_deferred, free_obj_work, psi_avgs_work, kernfs_notify_workfn, psi_avgs_work
workqueue events_long: flags=0x0
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: defense_work_handler
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=5 refcnt=6
pending: 5*defense_work_handler
workqueue events_unbound: flags=0x2
pwq 8: cpus=0-1 flags=0x4 nice=0 active=13 refcnt=14
pending: 2*nsim_dev_trap_report_work, toggle_allocation_gate, 3*nsim_dev_trap_report_work, 6*cfg80211_wiphy_work, flush_memcg_stats_dwork
pwq 8: cpus=0-1 flags=0x4 nice=0 active=4 refcnt=5
pending: 4*cfg80211_wiphy_work
workqueue events_freezable: flags=0x4
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: update_balloon_stats_func
workqueue events_power_efficient: flags=0x80
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: crda_timeout_work
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=10 refcnt=11
pending: wg_ratelimiter_gc_entries, 2*neigh_managed_work, 2*neigh_periodic_work, gc_worker, 4*check_lifetime
workqueue mm_percpu_wq: flags=0x8
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: vmstat_update
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: vmstat_update
workqueue writeback: flags=0x4a
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
pending: wb_update_bandwidth_workfn
workqueue mld: flags=0x40008
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: mld_dad_work
workqueue ipv6_addrconf: flags=0x6000a
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=215
in-flight: 5983:addrconf_dad_work
inactive: 128*addrconf_dad_work, addrconf_verify_work, 7*addrconf_dad_work, 5*addrconf_verify_work, 70*addrconf_dad_work
workqueue bat_events: flags=0x6000a
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=44
in-flight: 77:batadv_tt_purge
inactive: batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, 5*batadv_nc_worker, 5*batadv_mcast_mla_update, 5*batadv_purge_orig, 15*batadv_iv_send_outstanding_bat_ogm_packet
workqueue hci0: flags=0x20012
pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
in-flight: 5837:hci_cmd_sync_work
workqueue hci3: flags=0x20012
pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
in-flight: 51:hci_cmd_sync_work
workqueue wg-kex-wg0: flags=0x24
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_tx_worker, wg_packet_decrypt_worker
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_decrypt_worker
workqueue wg-kex-wg1: flags=0x24
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg0: flags=0x24
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_tx_worker, wg_packet_decrypt_worker
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_decrypt_worker
workqueue wg-kex-wg1: flags=0x24
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_decrypt_worker
workqueue wg-kex-wg0: flags=0x24
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_tx_worker, wg_packet_decrypt_worker
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_decrypt_worker
workqueue wg-kex-wg1: flags=0x24
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg2: flags=0x24
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg2: flags=0x28
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-kex-wg0: flags=0x24
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_tx_worker, wg_packet_decrypt_worker
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_decrypt_worker
workqueue wg-kex-wg0: flags=0x24
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_decrypt_worker
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_decrypt_worker
workqueue wg-kex-wg1: flags=0x24
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg1: flags=0x24
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_decrypt_worker
workqueue wg-kex-wg2: flags=0x24
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg2: flags=0x24
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
pool 2: cpus=0 node=0 flags=0x0 nice=0 hung=4s workers=8 idle: 5846 5987 5898 10 978 9
pool 6: cpus=1 node=0 flags=0x0 nice=0 hung=0s workers=7 idle: 24 5953 5920 5954 5839
pool 8: cpus=0-1 flags=0x4 nice=0 hung=0s workers=11 idle: 1004 12 5985 3000 64 49 36 3518
pool 9: cpus=0-1 node=0 flags=0x4 nice=-20 hung=0s workers=9 idle: 5845 5840 5848 5155 5842 5847 5849
Showing backtraces of running workers in stalled CPU-bound worker pools:
pool 6:
task:kworker/1:1 state:R running task stack:24912 pid:48 tgid:48 ppid:2 task_flags:0x4208060 flags:0x00004000
Workqueue: events_power_efficient neigh_periodic_work
Call Trace:
context_switch kernel/sched/core.c:5396 [inline]
__schedule+0x16f5/0x4d00 kernel/sched/core.c:6785
preempt_schedule_common+0x83/0xd0 kernel/sched/core.c:6965
preempt_schedule+0xae/0xc0 kernel/sched/core.c:6989
preempt_schedule_thunk+0x16/0x30 arch/x86/entry/thunk.S:12
__local_bh_enable_ip+0x13e/0x1c0 kernel/softirq.c:414
neigh_periodic_work+0xb41/0xd60 net/core/neighbour.c:966
process_one_work kernel/workqueue.c:3238 [inline]
process_scheduled_works+0xae1/0x17b0 kernel/workqueue.c:3321
worker_thread+0x8a0/0xda0 kernel/workqueue.c:3402
kthread+0x70e/0x8a0 kernel/kthread.c:464
ret_from_fork+0x3f9/0x770 arch/x86/kernel/process.c:148
ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245
pool 6:
task:kworker/1:2 state:R running task stack:24472 pid:980 tgid:980 ppid:2 task_flags:0x4208060 flags:0x00004000
Workqueue: events_power_efficient gc_worker
Call Trace:
context_switch kernel/sched/core.c:5396 [inline]
__schedule+0x16f5/0x4d00 kernel/sched/core.c:6785
preempt_schedule_irq+0xb5/0x150 kernel/sched/core.c:7108
irqentry_exit+0x6f/0x90 kernel/entry/common.c:307
asm_sysvec_apic_timer_interrupt+0x1a/0x20 arch/x86/include/asm/idtentry.h:702
RIP: 0010:__sanitizer_cov_trace_pc+0x5d/0x70 kernel/kcov.c:235
Code: 18 16 00 00 83 fa 02 75 21 48 8b 91 20 16 00 00 48 8b 32 48 8d 7e 01 8b 89 1c 16 00 00 48 39 cf 73 08 48 89 3a 48 89 44 f2 08 7e bc a8 09 cc 66 66 66 66 2e 0f 1f 84 00 00 00 00 00 90 90 90
RSP: 0018:ffffc9000390f918 EFLAGS: 00000293
RAX: ffffffff899781f9 RBX: 0000000000000001 RCX: ffff888025d31e00
RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000000
RBP: ffffc9000390fa70 R08: ffffffff8fa10df7 R09: 1ffffffff1f421be
R10: dffffc0000000000 R11: fffffbfff1f421bf R12: ffff8880b873b080
R13: ffff888024c3d018 R14: 0000000000000000 R15: 0000000000040000
nf_conntrack_get_ht include/net/netfilter/nf_conntrack.h:333 [inline]
gc_worker+0x329/0x1380 net/netfilter/nf_conntrack_core.c:1537
process_one_work kernel/workqueue.c:3238 [inline]
process_scheduled_works+0xae1/0x17b0 kernel/workqueue.c:3321
worker_thread+0x8a0/0xda0 kernel/workqueue.c:3402
kthread+0x70e/0x8a0 kernel/kthread.c:464
ret_from_fork+0x3f9/0x770 arch/x86/kernel/process.c:148
ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245
----------------
Code disassembly (best guess):
0: 18 16 sbb %dl,(%rsi)
2: 00 00 add %al,(%rax)
4: 83 fa 02 cmp $0x2,%edx
7: 75 21 jne 0x2a
9: 48 8b 91 20 16 00 00 mov 0x1620(%rcx),%rdx
10: 48 8b 32 mov (%rdx),%rsi
13: 48 8d 7e 01 lea 0x1(%rsi),%rdi
17: 8b 89 1c 16 00 00 mov 0x161c(%rcx),%ecx
1d: 48 39 cf cmp %rcx,%rdi
20: 73 08 jae 0x2a
22: 48 89 3a mov %rdi,(%rdx)
25: 48 89 44 f2 08 mov %rax,0x8(%rdx,%rsi,8)
* 2a: e9 7e bc a8 09 jmp 0x9a8bcad <-- trapping instruction
2f: cc int3
30: 66 66 66 66 2e 0f 1f data16 data16 data16 cs nopw 0x0(%rax,%rax,1)
37: 84 00 00 00 00 00
3d: 90 nop
3e: 90 nop
3f: 90 nop