BUG: workqueue lockup - pool cpus=0 node=0 flags=0x0 nice=0 stuck for 168s!
Showing busy workqueues and worker pools:
workqueue events: flags=0x0
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=10 refcnt=11
in-flight: 9:fqdir_free_fn
pending: 3*nsim_dev_hwstats_traffic_work, psi_avgs_work, 2*ovs_dp_masks_rebalance, vmstat_shepherd, bpf_prog_free_deferred, delayed_vfree_work
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=12 refcnt=13
in-flight: 24:reg_todo
pending: free_obj_work, 2*nsim_dev_hwstats_traffic_work, 4*psi_avgs_work, 4*ovs_dp_masks_rebalance
workqueue events_long: flags=0x0
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=6 refcnt=7
pending: 6*defense_work_handler
workqueue events_unbound: flags=0x2
pwq 8: cpus=0-1 flags=0x4 nice=0 active=12 refcnt=13
in-flight: 59:fsnotify_mark_destroy_workfn ,6075:cfg80211_wiphy_work cfg80211_wiphy_work ,1103:fsnotify_connector_destroy_workfn ,6084:cfg80211_wiphy_work cfg80211_wiphy_work
pending: 6*cfg80211_wiphy_work
pwq 8: cpus=0-1 flags=0x4 nice=0 active=12 refcnt=13
in-flight: 3562:nsim_dev_trap_report_work ,6083:nsim_dev_trap_report_work ,1329:cfg80211_wiphy_work cfg80211_wiphy_work ,6077:toggle_allocation_gate ,13:cfg80211_wiphy_work cfg80211_wiphy_work
pending: 3*nsim_dev_trap_report_work, flush_memcg_stats_dwork, crng_reseed
workqueue events_freezable: flags=0x4
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: update_balloon_stats_func
workqueue events_power_efficient: flags=0x80
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=6 refcnt=7
pending: do_cache_clean, 2*neigh_managed_work, 2*neigh_periodic_work, check_lifetime
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=4 refcnt=5
in-flight: 5888:gc_worker ,5838:reg_check_chans_work ,43:crda_timeout_work
pending: wg_ratelimiter_gc_entries
workqueue rcu_gp: flags=0x8
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
in-flight: 10:process_srcu
pending: process_srcu
workqueue kvfree_rcu_reclaim: flags=0xa
pwq 8: cpus=0-1 flags=0x4 nice=0 active=4 refcnt=5
in-flight: 6081:kfree_rcu_work ,12:kfree_rcu_work
pending: 2*kfree_rcu_monitor
workqueue mm_percpu_wq: flags=0x8
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: vmstat_update
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: vmstat_update
workqueue writeback: flags=0x4a
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
in-flight: 3533:wb_workfn
workqueue kblockd: flags=0x18
pwq 7: cpus=1 node=0 flags=0x0 nice=-20 active=1 refcnt=2
pending: blk_mq_timeout_work
workqueue ipv6_addrconf: flags=0x6000a
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=5
in-flight: 3172(RESCUER):addrconf_verify_work
workqueue bat_events: flags=0x6000a
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=46
in-flight: 5972:batadv_nc_worker
inactive: 5*batadv_tt_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, 2*batadv_bla_periodic_work, 2*batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, 2*batadv_nc_worker, 5*batadv_mcast_mla_update, 6*batadv_iv_send_outstanding_bat_ogm_packet, 5*batadv_purge_orig, 5*batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_nc_worker, 2*batadv_iv_send_outstanding_bat_ogm_packet
workqueue hci0: flags=0x20012
pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
in-flight: 5851:hci_cmd_sync_work
workqueue hci1: flags=0x20012
pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
in-flight: 5843:hci_cmd_sync_work
workqueue hci2: flags=0x20012
pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
in-flight: 51:hci_cmd_sync_work
workqueue hci3: flags=0x20012
pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
in-flight: 5848:hci_cmd_sync_work
workqueue wg-kex-wg0: flags=0x24
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg0: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
pending: wg_packet_tx_worker, wg_packet_decrypt_worker, wg_packet_encrypt_worker
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_decrypt_worker
workqueue wg-kex-wg1: flags=0x24
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg1: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
pending: 2*wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_decrypt_worker
workqueue wg-kex-wg0: flags=0x24
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg0: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_decrypt_worker
workqueue wg-kex-wg2: flags=0x24
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg2: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x28
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x24
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg1: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
pending: 2*wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg2: flags=0x24
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg2: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x28
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg0: flags=0x24
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
in-flight: 5888:wg_packet_handshake_receive_worker
workqueue wg-kex-wg0: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
pending: wg_packet_tx_worker, wg_packet_decrypt_worker, wg_packet_encrypt_worker
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_decrypt_worker
workqueue wg-kex-wg1: flags=0x24
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
in-flight: 5886:wg_packet_handshake_receive_worker
workqueue wg-kex-wg1: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
pending: 2*wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_decrypt_worker
workqueue wg-kex-wg2: flags=0x24
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg2: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x28
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg0: flags=0x24
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg0: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_decrypt_worker
workqueue wg-kex-wg1: flags=0x24
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg1: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
pending: 2*wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg2: flags=0x24
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg2: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x28
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg0: flags=0x24
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg1: flags=0x24
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg1: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x24
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg2: flags=0x28
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_encrypt_worker
pool 2: cpus=0 node=0 flags=0x0 nice=0 hung=170s workers=7 idle: 5885 5910 5927 980 5926
pool 6: cpus=1 node=0 flags=0x0 nice=0 hung=0s workers=9 idle: 5887 917 5889
pool 8: cpus=0-1 flags=0x4 nice=0 hung=0s workers=17 idle: 3562 6075 6081 12 6078 44
pool 9: cpus=0-1 node=0 flags=0x4 nice=-20 hung=0s workers=9 idle: 5155 5849 5841 5839 5842
Showing backtraces of running workers in stalled CPU-bound worker pools:
pool 2:
task:kworker/0:0 state:R running task stack:24136 pid:9 tgid:9 ppid:2 task_flags:0x4208060 flags:0x00004000
Workqueue: events fqdir_free_fn
Call Trace:
context_switch kernel/sched/core.c:5396 [inline]
__schedule+0x16f5/0x4d00 kernel/sched/core.c:6785
__schedule_loop kernel/sched/core.c:6863 [inline]
schedule+0x165/0x360 kernel/sched/core.c:6878
schedule_timeout+0x9a/0x270 kernel/time/sleep_timeout.c:75
do_wait_for_common kernel/sched/completion.c:95 [inline]
__wait_for_common kernel/sched/completion.c:116 [inline]
wait_for_common kernel/sched/completion.c:127 [inline]
wait_for_completion+0x2bf/0x5d0 kernel/sched/completion.c:148
rcu_barrier+0x463/0x570 kernel/rcu/tree.c:3850
fqdir_free_fn+0x33/0x140 net/ipv4/inet_fragment.c:166
process_one_work kernel/workqueue.c:3238 [inline]
process_scheduled_works+0xae1/0x17b0 kernel/workqueue.c:3321
worker_thread+0x8a0/0xda0 kernel/workqueue.c:3402
kthread+0x70e/0x8a0 kernel/kthread.c:464
ret_from_fork+0x3f9/0x770 arch/x86/kernel/process.c:148
ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245
pool 2:
task:kworker/0:1 state:R running task stack:22856 pid:10 tgid:10 ppid:2 task_flags:0x4208060 flags:0x00004000
Workqueue: rcu_gp process_srcu
Call Trace:
context_switch kernel/sched/core.c:5396 [inline]
__schedule+0x16f5/0x4d00 kernel/sched/core.c:6785
preempt_schedule_irq+0xb5/0x150 kernel/sched/core.c:7108
irqentry_exit+0x6f/0x90 kernel/entry/common.c:307
asm_sysvec_apic_timer_interrupt+0x1a/0x20 arch/x86/include/asm/idtentry.h:702
RIP: 0010:queue_delayed_work_on+0x201/0x280 kernel/workqueue.c:2566
Code: 4d 85 e4 75 16 e8 ef 81 35 00 eb 15 e8 e8 81 35 00 e8 a3 29 db 09 4d 85 e4 74 ea e8 d9 81 35 00 fb 48 c7 44 24 40 0e 36 e0 45 <4b> c7 04 2f 00 00 00 00 66 43 c7 44 2f 09 00 00 43 c6 44 2f 0b 00
RSP: 0018:ffffc900000f7980 EFLAGS: 00000293
RAX: ffffffff818adf77 RBX: ffff88801de9d201 RCX: ffff88801d2b1e00
RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000000
RBP: ffffc900000f7a70 R08: ffffffff8fa101f7 R09: 1ffffffff1f4203e
R10: dffffc0000000000 R11: fffffbfff1f4203f R12: 0000000000000200
R13: dffffc0000000000 R14: 0000000000000000 R15: 1ffff9200001ef38
process_one_work kernel/workqueue.c:3238 [inline]
process_scheduled_works+0xae1/0x17b0 kernel/workqueue.c:3321
worker_thread+0x8a0/0xda0 kernel/workqueue.c:3402
kthread+0x70e/0x8a0 kernel/kthread.c:464
ret_from_fork+0x3f9/0x770 arch/x86/kernel/process.c:148
ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245
----------------
Code disassembly (best guess):
0: 4d 85 e4 test %r12,%r12
3: 75 16 jne 0x1b
5: e8 ef 81 35 00 call 0x3581f9
a: eb 15 jmp 0x21
c: e8 e8 81 35 00 call 0x3581f9
11: e8 a3 29 db 09 call 0x9db29b9
16: 4d 85 e4 test %r12,%r12
19: 74 ea je 0x5
1b: e8 d9 81 35 00 call 0x3581f9
20: fb sti
21: 48 c7 44 24 40 0e 36 movq $0x45e0360e,0x40(%rsp)
28: e0 45
* 2a: 4b c7 04 2f 00 00 00 movq $0x0,(%r15,%r13,1) <-- trapping instruction
31: 00
32: 66 43 c7 44 2f 09 00 movw $0x0,0x9(%r15,%r13,1)
39: 00
3a: 43 c6 44 2f 0b 00 movb $0x0,0xb(%r15,%r13,1)