BUG: workqueue lockup - pool cpus=0 node=0 flags=0x0 nice=0 stuck for 192s! Showing busy workqueues and worker pools: workqueue events: flags=0x0 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=12 refcnt=13 pending: 3*nsim_dev_hwstats_traffic_work, psi_avgs_work, vmstat_shepherd, psi_avgs_work, 3*ovs_dp_masks_rebalance, 3*psi_avgs_work pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=8 refcnt=9 in-flight: 5880:free_ipc ,47:reg_todo pending: 2*nsim_dev_hwstats_traffic_work, psi_avgs_work, 3*ovs_dp_masks_rebalance workqueue events_long: flags=0x0 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: 2*defense_work_handler pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=4 refcnt=5 pending: 4*defense_work_handler workqueue events_unbound: flags=0x2 pwq 8: cpus=0-1 flags=0x4 nice=0 active=10 refcnt=11 in-flight: 6176:cfg80211_wiphy_work cfg80211_wiphy_work ,6165:cfg80211_wiphy_work cfg80211_wiphy_work ,6170:cfg80211_wiphy_work cfg80211_wiphy_work ,6172:cfg80211_wiphy_work cfg80211_wiphy_work pending: 2*cfg80211_wiphy_work pwq 8: cpus=0-1 flags=0x4 nice=0 active=19 refcnt=20 in-flight: 6168:fsnotify_connector_destroy_workfn fsnotify_connector_destroy_workfn ,6164:cfg80211_wiphy_work cfg80211_wiphy_work ,6157:cfg80211_wiphy_work cfg80211_wiphy_work ,6167:cfg80211_wiphy_work cfg80211_wiphy_work ,6171:cfg80211_wiphy_work cfg80211_wiphy_work ,6149:toggle_allocation_gate ,6169:fsnotify_mark_destroy_workfn fsnotify_mark_destroy_workfn pending: 5*nsim_dev_trap_report_work, flush_memcg_stats_dwork workqueue events_freezable: flags=0x4 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: update_balloon_stats_func workqueue events_power_efficient: flags=0x80 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=8 refcnt=9 in-flight: 5924:wg_ratelimiter_gc_entries pending: neigh_managed_work, neigh_periodic_work, gc_worker, 4*check_lifetime pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=5 refcnt=6 in-flight: 5882:reg_check_chans_work ,5846:crda_timeout_work pending: do_cache_clean, neigh_managed_work, neigh_periodic_work workqueue rcu_gp: flags=0x8 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: srcu_invoke_callbacks workqueue kvfree_rcu_reclaim: flags=0xa pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: kfree_rcu_monitor pwq 8: cpus=0-1 flags=0x4 nice=0 active=3 refcnt=4 in-flight: 6152:kfree_rcu_work ,6158:kfree_rcu_work pending: kfree_rcu_monitor workqueue mm_percpu_wq: flags=0x8 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: vmstat_update pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: vmstat_update workqueue writeback: flags=0x4a pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3 pending: wb_update_bandwidth_workfn, wb_workfn workqueue ipv6_addrconf: flags=0x6000a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=9 in-flight: 6163:addrconf_verify_work inactive: 5*addrconf_verify_work workqueue krxrpcd: flags=0x2001a pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=9 pending: rxrpc_peer_keepalive_worker inactive: 5*rxrpc_peer_keepalive_worker workqueue bat_events: flags=0x6000a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=47 in-flight: 6162:batadv_iv_send_outstanding_bat_ogm_packet inactive: 3*batadv_iv_send_outstanding_bat_ogm_packet, 3*batadv_nc_worker, batadv_iv_send_outstanding_bat_ogm_packet, batadv_purge_orig, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, 3*batadv_tt_purge, 2*batadv_nc_worker, 5*batadv_mcast_mla_update, 6*batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_purge_orig, 2*batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_purge_orig, batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_tt_purge workqueue ext4-rsv-conversion: flags=0xa pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=4 pending: ext4_end_io_rsv_work inactive: 2*ext4_end_io_rsv_work workqueue hci1: flags=0x20012 pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4 in-flight: 5831:hci_cmd_sync_work workqueue hci2: flags=0x20012 pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4 in-flight: 5840:hci_cmd_sync_work workqueue hci3: flags=0x20012 pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4 in-flight: 5140:hci_cmd_sync_work workqueue hci4: flags=0x20012 pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4 in-flight: 5836:hci_cmd_sync_work workqueue wg-kex-wg0: flags=0x24 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-crypt-wg0: flags=0x28 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4 pending: wg_packet_tx_worker, wg_packet_encrypt_worker, wg_packet_decrypt_worker workqueue wg-kex-wg0: flags=0x24 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-crypt-wg0: flags=0x28 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4 pending: wg_packet_tx_worker, wg_packet_encrypt_worker, wg_packet_decrypt_worker workqueue wg-kex-wg1: flags=0x24 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-crypt-wg1: flags=0x28 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4 pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg1: flags=0x24 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-crypt-wg1: flags=0x28 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4 pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg2: flags=0x24 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-kex-wg2: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg2: flags=0x28 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_encrypt_worker, wg_packet_tx_worker workqueue wg-kex-wg2: flags=0x24 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-kex-wg2: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg2: flags=0x28 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_encrypt_worker, wg_packet_tx_worker workqueue wg-kex-wg0: flags=0x24 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 in-flight: 24:wg_packet_handshake_receive_worker workqueue wg-crypt-wg0: flags=0x28 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4 pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker workqueue wg-kex-wg1: flags=0x24 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-kex-wg1: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg1: flags=0x28 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4 pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker workqueue wg-kex-wg2: flags=0x24 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-crypt-wg2: flags=0x28 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_encrypt_worker, wg_packet_tx_worker workqueue wg-kex-wg0: flags=0x24 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-crypt-wg0: flags=0x28 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4 pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker workqueue wg-kex-wg1: flags=0x24 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-crypt-wg1: flags=0x28 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4 pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker, wg_packet_tx_worker workqueue wg-kex-wg2: flags=0x24 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-crypt-wg2: flags=0x28 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_encrypt_worker, wg_packet_tx_worker workqueue wg-kex-wg0: flags=0x24 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-crypt-wg0: flags=0x28 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4 pending: wg_packet_decrypt_worker, wg_packet_tx_worker, wg_packet_encrypt_worker workqueue wg-kex-wg1: flags=0x24 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-kex-wg1: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3 pending: 2*wg_packet_handshake_send_worker workqueue wg-crypt-wg1: flags=0x28 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4 pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker workqueue wg-kex-wg2: flags=0x24 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-crypt-wg2: flags=0x28 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_encrypt_worker, wg_packet_tx_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pool 2: cpus=0 node=0 flags=0x0 nice=0 hung=193s workers=8 idle: 10 5910 5930 5879 838 9 5883 pool 6: cpus=1 node=0 flags=0x0 nice=0 hung=12s workers=8 idle: 3079 5911 5881 pool 8: cpus=0-1 flags=0x4 nice=0 hung=21s workers=71 idle: 6173 6160 6150 6174 6154 6156 6166 6151 6144 6153 6155 6159 6142 6134 6148 6147 6128 6140 6146 6139 6141 6135 6143 6117 6138 6118 6120 6113 6137 6136 3028 1155 6112 6119 5961 5958 6115 6145 74 6133 6126 13 6132 53 12 6125 6124 6130 6122 5960 1039 2974 6114 36 6121 6129 pool 9: cpus=0-1 node=0 flags=0x4 nice=-20 hung=18s workers=10 idle: 55 5843 5844 5835 5839 5848 Showing backtraces of running workers in stalled CPU-bound worker pools: pool 2: task:kworker/0:6 state:R running task stack:23600 pid:5924 tgid:5924 ppid:2 task_flags:0x4208060 flags:0x00004000 Workqueue: events_power_efficient wg_ratelimiter_gc_entries Call Trace: context_switch kernel/sched/core.c:5382 [inline] __schedule+0x16e2/0x4cd0 kernel/sched/core.c:6767 preempt_schedule_irq+0xb5/0x150 kernel/sched/core.c:7090 irqentry_exit+0x6f/0x90 kernel/entry/common.c:354 asm_sysvec_apic_timer_interrupt+0x1a/0x20 arch/x86/include/asm/idtentry.h:702 RIP: 0010:preempt_schedule+0x0/0xc0 kernel/sched/core.c:6964 Code: 00 00 e8 83 c7 39 f6 c3 cc cc cc cc cc 66 66 66 66 2e 0f 1f 84 00 00 00 00 00 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 0f 1e fa 55 48 89 e5 41 56 53 48 83 e4 e0 48 83 ec 60 65 48 8b RSP: 0018:ffffc9000469fa58 EFLAGS: 00000246 RAX: 0000000000000000 RBX: ffffffff8ea67be0 RCX: 0000000000000000 RDX: 0000000000000000 RSI: 0000000000000004 RDI: 00000000ffffffff RBP: ffffc9000469faa8 R08: ffffffff8ea67be3 R09: 1ffffffff1d4cf7c R10: dffffc0000000000 R11: fffffbfff1d4cf7d R12: ffff888060123a98 R13: ffff888026104618 R14: ffff888060133a98 R15: ffffffffffffffa0 preempt_schedule_thunk+0x16/0x30 arch/x86/entry/thunk.S:12 __raw_spin_unlock include/linux/spinlock_api_smp.h:143 [inline] _raw_spin_unlock+0x3f/0x50 kernel/locking/spinlock.c:186 spin_unlock include/linux/spinlock.h:391 [inline] wg_ratelimiter_gc_entries+0x384/0x450 drivers/net/wireguard/ratelimiter.c:76 process_one_work kernel/workqueue.c:3238 [inline] process_scheduled_works+0xadb/0x17a0 kernel/workqueue.c:3319 worker_thread+0x8a0/0xda0 kernel/workqueue.c:3400 kthread+0x70e/0x8a0 kernel/kthread.c:464 ret_from_fork+0x4b/0x80 arch/x86/kernel/process.c:153 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245 ---------------- Code disassembly (best guess): 0: 00 00 add %al,(%rax) 2: e8 83 c7 39 f6 call 0xf639c78a 7: c3 ret 8: cc int3 9: cc int3 a: cc int3 b: cc int3 c: cc int3 d: 66 66 66 66 2e 0f 1f data16 data16 data16 cs nopw 0x0(%rax,%rax,1) 14: 84 00 00 00 00 00 1a: 90 nop 1b: 90 nop 1c: 90 nop 1d: 90 nop 1e: 90 nop 1f: 90 nop 20: 90 nop 21: 90 nop 22: 90 nop 23: 90 nop 24: 90 nop 25: 90 nop 26: 90 nop 27: 90 nop 28: 90 nop 29: 90 nop * 2a: f3 0f 1e fa endbr64 <-- trapping instruction 2e: 55 push %rbp 2f: 48 89 e5 mov %rsp,%rbp 32: 41 56 push %r14 34: 53 push %rbx 35: 48 83 e4 e0 and $0xffffffffffffffe0,%rsp 39: 48 83 ec 60 sub $0x60,%rsp 3d: 65 gs 3e: 48 rex.W 3f: 8b .byte 0x8b