BUG: workqueue lockup - pool cpus=1 node=0 flags=0x2 nice=0 stuck for 170s! Showing busy workqueues and worker pools: workqueue events: flags=0x100 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=12 refcnt=13 in-flight: 10:free_ipc pending: psi_avgs_work, 2*nsim_dev_hwstats_traffic_work, psi_avgs_work, vmstat_shepherd, 2*psi_avgs_work, 4*ovs_dp_masks_rebalance pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=20 refcnt=21 pending: 8*nsim_dev_hwstats_traffic_work, psi_avgs_work, 4*ovs_dp_masks_rebalance, psi_avgs_work, xfrm_state_gc_task, debugfs_reap_work, delayed_vfree_work, switchdev_deferred_process_work, rht_deferred_worker, free_obj_work workqueue events_long: flags=0x100 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=5 refcnt=6 pending: 5*defense_work_handler pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=4 refcnt=5 pending: 3*defense_work_handler, br_multicast_gc_work workqueue events_unbound: flags=0x2 pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3 pending: toggle_allocation_gate, flush_memcg_stats_dwork workqueue events_unbound: flags=0x2 pwq 8: cpus=0-1 flags=0x4 nice=0 active=23 refcnt=24 in-flight: 3427:cfg80211_wiphy_work cfg80211_wiphy_work ,1149:cfg80211_wiphy_work cfg80211_wiphy_work ,57:cfg80211_wiphy_work cfg80211_wiphy_work ,6231:cfg80211_wiphy_work cfg80211_wiphy_work ,4113:cfg80211_wiphy_work cfg80211_wiphy_work ,143:cfg80211_wiphy_work cfg80211_wiphy_work pending: 10*nsim_dev_trap_report_work pwq 8: cpus=0-1 flags=0x4 nice=0 active=8 refcnt=9 in-flight: 36:cfg80211_wiphy_work cfg80211_wiphy_work ,6240:linkwatch_event linkwatch_event pending: 5*cfg80211_wiphy_work workqueue events_freezable: flags=0x104 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: update_balloon_stats_func workqueue events_power_efficient: flags=0x180 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_ratelimiter_gc_entries, neigh_managed_work pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=9 refcnt=10 pending: neigh_periodic_work, neigh_managed_work, gc_worker, check_lifetime, reg_check_chans_work, 4*check_lifetime workqueue kvfree_rcu_reclaim: flags=0xa pwq 8: cpus=0-1 flags=0x4 nice=0 active=4 refcnt=5 in-flight: 1059:kfree_rcu_monitor kfree_rcu_monitor ,3002:kfree_rcu_work ,13734:kfree_rcu_work pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: kfree_rcu_monitor workqueue netns: flags=0x6000a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=5 in-flight: 7158:cleanup_net inactive: cleanup_net workqueue mm_percpu_wq: flags=0x8 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: vmstat_update pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=2 pending: vmstat_update workqueue kblockd: flags=0x18 pwq 7: cpus=1 node=0 flags=0x0 nice=-20 active=1 refcnt=2 pending: blk_mq_requeue_work workqueue ipv6_addrconf: flags=0x6000a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=72 in-flight: 3550:addrconf_dad_work inactive: 60*addrconf_dad_work, 8*addrconf_verify_work workqueue bat_events: flags=0x6000a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=59 in-flight: 7171:batadv_tt_purge inactive: batadv_tt_purge, 7*batadv_mcast_mla_update, 3*batadv_iv_send_outstanding_bat_ogm_packet, batadv_purge_orig, 2*batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_purge_orig, 4*batadv_iv_send_outstanding_bat_ogm_packet, batadv_purge_orig, 3*batadv_iv_send_outstanding_bat_ogm_packet, batadv_purge_orig, 5*batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_purge_orig, batadv_iv_send_outstanding_bat_ogm_packet, batadv_bla_periodic_work, 3*batadv_iv_send_outstanding_bat_ogm_packet, batadv_tt_purge, 2*batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, 2*batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, 3*batadv_dat_purge, 2*batadv_bla_periodic_work, 4*batadv_tt_purge workqueue wg-kex-wg0: flags=0x124 pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-kex-wg0: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg0: flags=0x128 pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=2 refcnt=3 pending: wg_packet_encrypt_worker, wg_packet_tx_worker workqueue wg-kex-wg1: flags=0x124 pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-kex-wg1: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg1: flags=0x128 pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=2 refcnt=3 pending: wg_packet_encrypt_worker, wg_packet_tx_worker workqueue wg-kex-wg2: flags=0x124 pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-kex-wg2: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 in-flight: 1211:wg_packet_handshake_send_worker workqueue wg-crypt-wg2: flags=0x128 pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=3 refcnt=4 pending: wg_packet_encrypt_worker, wg_packet_decrypt_worker, wg_packet_tx_worker workqueue wg-crypt-wg0: flags=0x128 pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=2 refcnt=3 pending: wg_packet_tx_worker, wg_packet_encrypt_worker workqueue wg-kex-wg1: flags=0x124 pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-kex-wg1: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg1: flags=0x128 pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=3 refcnt=4 pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker workqueue wg-kex-wg2: flags=0x124 pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-kex-wg2: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg2: flags=0x128 pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=3 refcnt=4 pending: wg_packet_encrypt_worker, wg_packet_decrypt_worker, wg_packet_tx_worker workqueue hci2: flags=0x20012 pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4 in-flight: 5848:hci_cmd_sync_work workqueue hci3: flags=0x20012 pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4 in-flight: 5853:hci_cmd_sync_work workqueue hci7: flags=0x20012 pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4 in-flight: 12372:hci_cmd_sync_work workqueue hci8: flags=0x20012 pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4 in-flight: 13740:hci_cmd_sync_work workqueue wg-kex-wg0: flags=0x124 pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-kex-wg0: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg0: flags=0x128 pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=2 refcnt=3 pending: wg_packet_encrypt_worker, wg_packet_tx_worker workqueue wg-kex-wg1: flags=0x124 pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-crypt-wg1: flags=0x128 pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=3 refcnt=4 pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker workqueue wg-kex-wg2: flags=0x124 pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-kex-wg2: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg2: flags=0x128 pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=3 refcnt=4 pending: wg_packet_encrypt_worker, wg_packet_decrypt_worker, wg_packet_tx_worker workqueue wg-kex-wg0: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg0: flags=0x128 pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=2 refcnt=3 pending: wg_packet_encrypt_worker, wg_packet_tx_worker workqueue wg-kex-wg1: flags=0x124 pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-crypt-wg1: flags=0x128 pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=3 refcnt=4 pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker workqueue wg-kex-wg2: flags=0x124 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 in-flight: 8385:wg_packet_handshake_receive_worker pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-crypt-wg2: flags=0x128 pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=3 refcnt=4 pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker workqueue wg-kex-wg0: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg0: flags=0x128 pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=2 refcnt=3 pending: wg_packet_encrypt_worker, wg_packet_tx_worker workqueue wg-crypt-wg1: flags=0x128 pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=2 refcnt=3 pending: wg_packet_encrypt_worker, wg_packet_tx_worker workqueue wg-kex-wg2: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg2: flags=0x128 pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=2 refcnt=3 pending: wg_packet_encrypt_worker, wg_packet_tx_worker workqueue wg-kex-wg0: flags=0x124 pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-crypt-wg0: flags=0x128 pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=2 refcnt=3 pending: wg_packet_encrypt_worker, wg_packet_tx_worker workqueue wg-kex-wg1: flags=0x124 pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-crypt-wg1: flags=0x128 pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=3 refcnt=4 pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker workqueue wg-kex-wg2: flags=0x124 pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-kex-wg2: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg2: flags=0x128 pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=3 refcnt=4 pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker workqueue wg-kex-wg0: flags=0x124 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-kex-wg0: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg0: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_tx_worker, wg_packet_decrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_decrypt_worker workqueue wg-kex-wg1: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg1: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_decrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_decrypt_worker workqueue wg-kex-wg2: flags=0x124 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue hci4: flags=0x20012 pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=6 in-flight: 5852:hci_rx_work inactive: hci_rx_work, hci_cmd_work pool 2: cpus=0 node=0 flags=0x0 nice=0 hung=7s workers=26 idle: 7626 9 11402 6225 5924 6195 5935 13752 7625 5965 5901 6193 6232 11773 12359 13726 978 1217 6229 12526 9082 11795 6090 13727 pool 6: cpus=1 node=0 flags=0x2 nice=0 hung=0s workers=23 manager: 39 idle: 8383 6241 5997 11793 6196 5926 8371 12356 7782 24 pool 8: cpus=0-1 flags=0x4 nice=0 hung=0s workers=29 idle: 2983 4563 6239 13741 13 13750 11794 12 1092 3018 6233 11256 pool 9: cpus=0-1 node=0 flags=0x4 nice=-20 hung=0s workers=12 idle: 13747 5153 13749 13746 13745 5847 13744 Showing backtraces of running workers in stalled CPU-bound worker pools: pool 6: task:kworker/1:19 state:R running task stack:24088 pid:12360 tgid:12360 ppid:2 task_flags:0x4208060 flags:0x00080000 Workqueue: events_power_efficient gc_worker Call Trace: context_switch kernel/sched/core.c:5256 [inline] __schedule+0x14bc/0x5000 kernel/sched/core.c:6863 preempt_schedule_irq+0xb5/0x150 kernel/sched/core.c:7190 irqentry_exit+0x5d8/0x660 kernel/entry/common.c:216 asm_sysvec_apic_timer_interrupt+0x1a/0x20 arch/x86/include/asm/idtentry.h:697 RIP: 0010:seqcount_lockdep_reader_access+0x17e/0x1c0 include/linux/seqlock.h:75 Code: f8 4d 85 e4 75 16 e8 61 45 3d f8 eb 15 e8 5a 45 3d f8 e8 35 a5 d7 01 4d 85 e4 74 ea e8 4b 45 3d f8 fb 48 c7 04 24 0e 36 e0 45 <4b> c7 04 3e 00 00 00 00 66 43 c7 44 3e 09 00 00 43 c6 44 3e 0b 00 RSP: 0018:ffffc90003e6f820 EFLAGS: 00000293 RAX: ffffffff89846ce5 RBX: 0000000000000000 RCX: ffff88802a2e3d00 RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000000 RBP: ffffc90003e6f8d0 R08: ffffffff8f822477 R09: 1ffffffff1f0448e R10: dffffc0000000000 R11: fffffbfff1f0448f R12: 0000000000000200 R13: ffff888028a19418 R14: 1ffff920007cdf04 R15: dffffc0000000000 nf_conntrack_get_ht include/net/netfilter/nf_conntrack.h:342 [inline] gc_worker+0x308/0x1380 net/netfilter/nf_conntrack_core.c:1548 process_one_work kernel/workqueue.c:3257 [inline] process_scheduled_works+0xad1/0x1770 kernel/workqueue.c:3340 worker_thread+0x8a0/0xda0 kernel/workqueue.c:3421 kthread+0x711/0x8a0 kernel/kthread.c:463 ret_from_fork+0x599/0xb30 arch/x86/kernel/process.c:158 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:246 ---------------- Code disassembly (best guess): 0: f8 clc 1: 4d 85 e4 test %r12,%r12 4: 75 16 jne 0x1c 6: e8 61 45 3d f8 call 0xf83d456c b: eb 15 jmp 0x22 d: e8 5a 45 3d f8 call 0xf83d456c 12: e8 35 a5 d7 01 call 0x1d7a54c 17: 4d 85 e4 test %r12,%r12 1a: 74 ea je 0x6 1c: e8 4b 45 3d f8 call 0xf83d456c 21: fb sti 22: 48 c7 04 24 0e 36 e0 movq $0x45e0360e,(%rsp) 29: 45 * 2a: 4b c7 04 3e 00 00 00 movq $0x0,(%r14,%r15,1) <-- trapping instruction 31: 00 32: 66 43 c7 44 3e 09 00 movw $0x0,0x9(%r14,%r15,1) 39: 00 3a: 43 c6 44 3e 0b 00 movb $0x0,0xb(%r14,%r15,1)