BUG: workqueue lockup - pool cpus=0 node=0 flags=0x0 nice=0 stuck for 34s! Showing busy workqueues and worker pools: workqueue events: flags=0x100 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=17 refcnt=18 pending: 3*l2cap_info_timeout, ovs_dp_masks_rebalance, rht_deferred_worker, ovs_dp_masks_rebalance, vmstat_shepherd, rht_deferred_worker, psi_avgs_work, ovs_dp_masks_rebalance, rht_deferred_worker, delayed_vfree_work, 2*nsim_dev_hwstats_traffic_work, ovs_dp_masks_rebalance, free_obj_work, psi_avgs_work pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=6 refcnt=7 pending: 2*nsim_dev_hwstats_traffic_work, 2*psi_avgs_work, 2*ovs_dp_masks_rebalance workqueue events_long: flags=0x100 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4 in-flight: 6688:defense_work_handler pending: 2*defense_work_handler pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=6 refcnt=7 pending: 6*defense_work_handler workqueue events_unbound: flags=0x2 pwq 8: cpus=0-1 flags=0x6 nice=0 active=14 refcnt=15 in-flight: 12:cfg80211_wiphy_work ,15:cfg80211_wiphy_work cfg80211_wiphy_work ,746:cfg80211_wiphy_work cfg80211_wiphy_work ,42:cfg80211_wiphy_work cfg80211_wiphy_work ,4828:cfg80211_wiphy_work cfg80211_wiphy_work ,925:cfg80211_wiphy_work cfg80211_wiphy_work pending: 2*cfg80211_wiphy_work, crng_reseed pwq 8: cpus=0-1 flags=0x6 nice=0 active=4 refcnt=5 in-flight: 1070:cfg80211_wiphy_work pending: cfg80211_wiphy_work, toggle_allocation_gate, flush_memcg_stats_dwork workqueue events_unbound: flags=0x2 pwq 8: cpus=0-1 flags=0x6 nice=0 active=3 refcnt=4 in-flight: 41:fsnotify_mark_destroy_workfn ,1029:fsnotify_connector_destroy_workfn pending: macvlan_process_broadcast pwq 8: cpus=0-1 flags=0x6 nice=0 active=6 refcnt=7 pending: 4*nsim_dev_trap_report_work, 2*macvlan_process_broadcast workqueue events_power_efficient: flags=0x182 pwq 8: cpus=0-1 flags=0x6 nice=0 active=7 refcnt=8 in-flight: 6820:neigh_periodic_work pending: do_cache_clean, neigh_managed_work, gc_worker, fb_flashcursor, neigh_managed_work, wg_ratelimiter_gc_entries workqueue rcu_gp: flags=0x108 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: srcu_invoke_callbacks, process_srcu workqueue netns: flags=0x6000a pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=4 in-flight: 264:cleanup_net workqueue mm_percpu_wq: flags=0x8 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: vmstat_update pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: vmstat_update workqueue writeback: flags=0x4a pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=4 MAYDAY in-flight: 34(RESCUER):wb_workfn workqueue mld: flags=0x40108 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=60 pending: mld_ifc_work inactive: 2*mld_ifc_work, 2*mld_dad_work, 3*mld_ifc_work, mld_dad_work, 3*mld_ifc_work, 2*mld_dad_work, mld_ifc_work, mld_dad_work, mld_ifc_work, mld_dad_work, mld_ifc_work, mld_dad_work, 3*mld_ifc_work, mld_dad_work, 2*mld_ifc_work, mld_dad_work, mld_ifc_work, mld_dad_work, 2*mld_ifc_work, 2*mld_dad_work, 3*mld_ifc_work, 4*mld_dad_work, 3*mld_ifc_work, 3*mld_dad_work, 9*mld_ifc_work, 2*mld_dad_work, mld_ifc_work, mld_dad_work pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=10 in-flight: 6701:mld_ifc_work inactive: 2*mld_ifc_work, 2*mld_dad_work, 3*mld_ifc_work, mld_dad_work workqueue ipv6_addrconf: flags=0x6000a pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=260 in-flight: 4236(RESCUER):addrconf_dad_work inactive: 254*addrconf_dad_work, addrconf_verify_work workqueue krxrpcd: flags=0x2001a pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=5 pending: rxrpc_peer_keepalive_worker inactive: rxrpc_peer_keepalive_worker workqueue bat_events: flags=0x6000a pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=50 in-flight: 4312(RESCUER):batadv_tt_purge inactive: 2*batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_purge_orig, 2*batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_tt_purge, 4*batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_tt_purge, 2*batadv_iv_send_outstanding_bat_ogm_packet, 4*batadv_purge_orig, 6*batadv_iv_send_outstanding_bat_ogm_packet, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, 6*batadv_mcast_mla_update, batadv_tt_purge workqueue wg-kex-wg0: flags=0x6 pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg0: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3 in-flight: 6752:wg_packet_encrypt_worker pending: wg_packet_tx_worker workqueue wg-crypt-wg1: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_encrypt_worker, wg_packet_tx_worker workqueue wg-crypt-wg2: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_encrypt_worker, wg_packet_tx_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg0: flags=0x124 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-crypt-wg0: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4 pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3 in-flight: 6706:wg_packet_tx_worker pending: wg_packet_encrypt_worker workqueue wg-kex-wg1: flags=0x6 pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg1: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=4 refcnt=5 in-flight: 1883:wg_packet_decrypt_worker pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_decrypt_worker, wg_packet_tx_worker workqueue wg-crypt-wg2: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_encrypt_worker, wg_packet_tx_worker workqueue wg-kex-wg0: flags=0x124 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-kex-wg0: flags=0x6 pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg0: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4 pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=3 refcnt=4 pending: wg_packet_decrypt_worker, wg_packet_tx_worker, wg_packet_encrypt_worker workqueue wg-kex-wg0: flags=0x6 pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg0: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4 pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-crypt-wg1: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4 pending: wg_packet_decrypt_worker, wg_packet_tx_worker, wg_packet_encrypt_worker workqueue wg-kex-wg2: flags=0x6 pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg2: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_tx_worker, wg_packet_encrypt_worker workqueue wg-kex-wg1: flags=0x6 pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg1: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4 pending: wg_packet_encrypt_worker, wg_packet_decrypt_worker, wg_packet_tx_worker workqueue wg-crypt-wg2: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_tx_worker, wg_packet_encrypt_worker pool 2: cpus=0 node=0 flags=0x0 nice=0 hung=34s workers=8 idle: 9 6760 6779 6753 11 pool 6: cpus=1 node=0 flags=0x0 nice=0 hung=5s workers=6 idle: 6703 24 3975 26 pool 8: cpus=0-1 flags=0x6 nice=0 hung=3s workers=12 manager: 6821 Showing backtraces of running workers in stalled CPU-bound worker pools: pool 2: task:kworker/0:3 state:R running task stack:0 pid:6688 tgid:6688 ppid:2 task_flags:0x4208060 flags:0x00000010 Workqueue: events_long defense_work_handler Call trace: __switch_to+0x418/0x87c arch/arm64/kernel/process.c:741 (T) context_switch kernel/sched/core.c:5325 [inline] __schedule+0x13b0/0x2864 kernel/sched/core.c:6929 preempt_schedule_common+0xe0/0x194 kernel/sched/core.c:7113 preempt_schedule+0x60/0x80 kernel/sched/core.c:7137 __local_bh_enable_ip+0x20c/0x35c kernel/softirq.c:457 local_bh_enable+0x28/0x34 include/linux/bottom_half.h:33 update_defense_level+0x5ec/0x984 net/netfilter/ipvs/ip_vs_ctl.c:210 defense_work_handler+0x30/0xdc net/netfilter/ipvs/ip_vs_ctl.c:235 process_one_work+0x7e8/0x155c kernel/workqueue.c:3263 process_scheduled_works kernel/workqueue.c:3346 [inline] worker_thread+0x958/0xed8 kernel/workqueue.c:3427 kthread+0x5fc/0x75c kernel/kthread.c:463 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:844 pool 2: task:kworker/0:2 state:R running task stack:0 pid:1883 tgid:1883 ppid:2 task_flags:0x4208060 flags:0x00000010 Workqueue: wg-crypt-wg1 wg_packet_decrypt_worker Call trace: __switch_to+0x418/0x87c arch/arm64/kernel/process.c:741 (T) context_switch kernel/sched/core.c:5325 [inline] __schedule+0x13b0/0x2864 kernel/sched/core.c:6929 preempt_schedule_common+0xe0/0x194 kernel/sched/core.c:7113 preempt_schedule+0x60/0x80 kernel/sched/core.c:7137 __raw_spin_unlock_irq include/linux/spinlock_api_smp.h:160 [inline] _raw_spin_unlock_irq+0x7c/0x80 kernel/locking/spinlock.c:202 process_one_work+0x5cc/0x155c kernel/workqueue.c:3232 process_scheduled_works kernel/workqueue.c:3346 [inline] worker_thread+0x958/0xed8 kernel/workqueue.c:3427 kthread+0x5fc/0x75c kernel/kthread.c:463 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:844 pool 2: task:kworker/0:4 state:R running task stack:0 pid:6752 tgid:6752 ppid:2 task_flags:0x4208060 flags:0x20000010 Workqueue: wg-crypt-wg0 wg_packet_encrypt_worker Call trace: __switch_to+0x418/0x87c arch/arm64/kernel/process.c:741 (T) context_switch kernel/sched/core.c:5325 [inline] __schedule+0x13b0/0x2864 kernel/sched/core.c:6929 preempt_schedule_common+0xe0/0x194 kernel/sched/core.c:7113 preempt_schedule+0x60/0x80 kernel/sched/core.c:7137 __local_bh_enable_ip+0x20c/0x35c kernel/softirq.c:457 local_bh_enable+0x28/0x34 include/linux/bottom_half.h:33 put_cpu_fpsimd_context arch/arm64/kernel/fpsimd.c:256 [inline] kernel_neon_begin+0x2c0/0x4a8 arch/arm64/kernel/fpsimd.c:1890 poly1305_blocks lib/crypto/arm64/poly1305.h:34 [inline] poly1305_update+0x224/0x5ac lib/crypto/poly1305.c:59 chacha20poly1305_crypt_sg_inplace+0x6b8/0xbc0 lib/crypto/chacha20poly1305.c:304 chacha20poly1305_encrypt_sg_inplace+0x54/0x70 lib/crypto/chacha20poly1305.c:340 encrypt_packet drivers/net/wireguard/send.c:216 [inline] wg_packet_encrypt_worker+0x710/0x11e0 drivers/net/wireguard/send.c:297 process_one_work+0x7e8/0x155c kernel/workqueue.c:3263 process_scheduled_works kernel/workqueue.c:3346 [inline] worker_thread+0x958/0xed8 kernel/workqueue.c:3427 kthread+0x5fc/0x75c kernel/kthread.c:463 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:844