BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=0 stuck for 34s! Showing busy workqueues and worker pools: workqueue events: flags=0x0 pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=1 refcnt=2 in-flight: 11:reg_todo pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=17 refcnt=18 pending: 3*destroy_super_work, 4*nsim_dev_hwstats_traffic_work, 3*psi_avgs_work, destroy_super_work, psi_avgs_work, 4*ovs_dp_masks_rebalance, free_obj_work workqueue events_long: flags=0x0 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=3 refcnt=4 in-flight: 6591:defense_work_handler pending: 2*defense_work_handler workqueue events_unbound: flags=0x2 pwq 8: cpus=0-1 flags=0x6 nice=0 active=20 refcnt=21 in-flight: 41:cfg80211_wiphy_work cfg80211_wiphy_work ,9487:nsim_dev_trap_report_work ,121:cfg80211_wiphy_work cfg80211_wiphy_work ,348:cfg80211_wiphy_work cfg80211_wiphy_work ,2164:cfg80211_wiphy_work cfg80211_wiphy_work ,97:cfg80211_wiphy_work cfg80211_wiphy_work ,9486:cfg80211_wiphy_work pending: nsim_dev_trap_report_work, crng_reseed, macvlan_process_broadcast, 2*nsim_dev_trap_report_work, cfg80211_wiphy_work, nsim_dev_trap_report_work, toggle_allocation_gate pwq 8: cpus=0-1 flags=0x6 nice=0 active=14 refcnt=15 in-flight: 1649:cfg80211_wiphy_work cfg80211_wiphy_work ,670:cfg80211_wiphy_work cfg80211_wiphy_work ,12:cfg80211_wiphy_work cfg80211_wiphy_work ,14:cfg80211_wiphy_work cfg80211_wiphy_work pending: 4*macvlan_process_broadcast, nsim_dev_trap_report_work, flush_memcg_stats_dwork workqueue events_power_efficient: flags=0x82 pwq 8: cpus=0-1 flags=0x6 nice=0 active=10 refcnt=11 in-flight: 42:crda_timeout_work pending: 2*neigh_managed_work, 4*check_lifetime, gc_worker, neigh_periodic_work, do_cache_clean pwq 8: cpus=0-1 flags=0x6 nice=0 active=6 refcnt=7 pending: reg_check_chans_work, check_lifetime, fb_flashcursor, wg_ratelimiter_gc_entries, neigh_periodic_work, check_lifetime workqueue mm_percpu_wq: flags=0x8 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: vmstat_update workqueue iou_exit: flags=0x2 pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2 pending: io_ring_exit_work workqueue ipv6_addrconf: flags=0x6000a pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=10 in-flight: 4165(RESCUER):addrconf_verify_work inactive: 5*addrconf_verify_work workqueue krxrpcd: flags=0x2001a pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=7 pending: rxrpc_peer_keepalive_worker inactive: 3*rxrpc_peer_keepalive_worker workqueue bat_events: flags=0x6000a pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=48 in-flight: 4246(RESCUER):batadv_dat_purge inactive: 2*batadv_bla_periodic_work, 2*batadv_dat_purge, batadv_tt_purge, batadv_dat_purge, batadv_bla_periodic_work, 5*batadv_nc_worker, 5*batadv_mcast_mla_update, 2*batadv_iv_send_outstanding_bat_ogm_packet, batadv_purge_orig, 6*batadv_iv_send_outstanding_bat_ogm_packet, batadv_purge_orig, batadv_iv_send_outstanding_bat_ogm_packet, 3*batadv_purge_orig, 6*batadv_iv_send_outstanding_bat_ogm_packet, 4*batadv_tt_purge, 2*batadv_bla_periodic_work workqueue wg-crypt-wg0: flags=0x28 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_encrypt_worker, wg_packet_tx_worker workqueue wg-kex-wg1: flags=0x6 pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg1: flags=0x28 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_tx_worker, wg_packet_encrypt_worker workqueue wg-kex-wg2: flags=0x6 pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg2: flags=0x28 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=4 refcnt=5 in-flight: 1785:wg_packet_encrypt_worker pending: wg_packet_decrypt_worker, wg_packet_tx_worker, wg_packet_encrypt_worker workqueue wg-kex-wg0: flags=0x24 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-kex-wg0: flags=0x6 pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg0: flags=0x28 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_tx_worker, wg_packet_encrypt_worker workqueue wg-kex-wg1: flags=0x6 pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg1: flags=0x28 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_tx_worker, wg_packet_encrypt_worker workqueue wg-kex-wg2: flags=0x6 pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg2: flags=0x28 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_decrypt_worker workqueue wg-kex-wg0: flags=0x6 pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg0: flags=0x28 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_encrypt_worker, wg_packet_tx_worker workqueue wg-kex-wg1: flags=0x6 pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg1: flags=0x28 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_decrypt_worker, wg_packet_tx_worker workqueue wg-kex-wg2: flags=0x24 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-kex-wg2: flags=0x6 pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg2: flags=0x28 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_decrypt_worker workqueue wg-crypt-wg0: flags=0x28 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_encrypt_worker, wg_packet_tx_worker workqueue wg-kex-wg1: flags=0x6 pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg1: flags=0x28 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_tx_worker, wg_packet_encrypt_worker workqueue wg-kex-wg2: flags=0x6 pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg2: flags=0x28 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker workqueue wg-kex-wg0: flags=0x6 pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg0: flags=0x28 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_encrypt_worker, wg_packet_tx_worker workqueue wg-kex-wg1: flags=0x6 pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg1: flags=0x28 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_tx_worker, wg_packet_encrypt_worker workqueue wg-kex-wg2: flags=0x6 pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg2: flags=0x28 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=3 refcnt=4 pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker, wg_packet_tx_worker pool 2: cpus=0 node=0 flags=0x2 nice=0 hung=0s workers=7 manager: 2324 idle: 6592 9 6599 6527 6631 pool 6: cpus=1 node=0 flags=0x0 nice=0 hung=34s workers=6 idle: 24 26 6577 6584 pool 8: cpus=0-1 flags=0x6 nice=0 hung=0s workers=13 manager: 9488 Showing backtraces of running workers in stalled CPU-bound worker pools: pool 6: task:kworker/1:5 state:R running task stack:0 pid:6591 tgid:6591 ppid:2 task_flags:0x4208060 flags:0x00000010 Workqueue: events_long defense_work_handler Call trace: __switch_to+0x414/0x834 arch/arm64/kernel/process.c:737 (T) context_switch kernel/sched/core.c:5396 [inline] __schedule+0x1414/0x2a28 kernel/sched/core.c:6785 preempt_schedule_common+0xe0/0x194 kernel/sched/core.c:6965 preempt_schedule+0x60/0x80 kernel/sched/core.c:6989 __local_bh_enable_ip+0x20c/0x35c kernel/softirq.c:414 local_bh_enable+0x28/0x34 include/linux/bottom_half.h:33 update_defense_level+0x5ec/0x984 net/netfilter/ipvs/ip_vs_ctl.c:210 defense_work_handler+0x30/0xdc net/netfilter/ipvs/ip_vs_ctl.c:235 process_one_work+0x7e8/0x155c kernel/workqueue.c:3238 process_scheduled_works kernel/workqueue.c:3321 [inline] worker_thread+0x958/0xed8 kernel/workqueue.c:3402 kthread+0x5fc/0x75c kernel/kthread.c:464 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:847 pool 6: task:kworker/1:2 state:R running task stack:0 pid:1785 tgid:1785 ppid:2 task_flags:0x4208060 flags:0x20000010 Workqueue: wg-crypt-wg2 wg_packet_encrypt_worker Call trace: __switch_to+0x414/0x834 arch/arm64/kernel/process.c:737 (T) context_switch kernel/sched/core.c:5396 [inline] __schedule+0x1414/0x2a28 kernel/sched/core.c:6785 preempt_schedule_common+0xe0/0x194 kernel/sched/core.c:6965 preempt_schedule+0x60/0x80 kernel/sched/core.c:6989 __local_bh_enable_ip+0x20c/0x35c kernel/softirq.c:414 local_bh_enable+0x28/0x34 include/linux/bottom_half.h:33 put_cpu_fpsimd_context arch/arm64/kernel/fpsimd.c:244 [inline] kernel_neon_begin+0x29c/0x484 arch/arm64/kernel/fpsimd.c:1876 poly1305_blocks_arch+0x40/0xa8 arch/arm64/lib/crypto/poly1305-glue.c:40 poly1305_blocks lib/crypto/poly1305.c:36 [inline] poly1305_update+0xd0/0x188 lib/crypto/poly1305.c:44 chacha20poly1305_crypt_sg_inplace+0x6b8/0xbc0 lib/crypto/chacha20poly1305.c:304 chacha20poly1305_encrypt_sg_inplace+0x54/0x70 lib/crypto/chacha20poly1305.c:340 encrypt_packet drivers/net/wireguard/send.c:216 [inline] wg_packet_encrypt_worker+0x718/0x11e0 drivers/net/wireguard/send.c:297 process_one_work+0x7e8/0x155c kernel/workqueue.c:3238 process_scheduled_works kernel/workqueue.c:3321 [inline] worker_thread+0x958/0xed8 kernel/workqueue.c:3402 kthread+0x5fc/0x75c kernel/kthread.c:464 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:847 net_ratelimit: 29595 callbacks suppressed bridge0: received packet on bridge_slave_0 with own address as source address (addr:aa:aa:aa:aa:aa:1b, vlan:0) bridge0: received packet on bridge_slave_0 with own address as source address (addr:aa:aa:aa:aa:aa:1b, vlan:0) bridge0: received packet on bridge_slave_0 with own address as source address (addr:aa:aa:aa:aa:aa:1b, vlan:0) bridge0: received packet on bridge_slave_0 with own address as source address (addr:aa:aa:aa:aa:aa:1b, vlan:0) bridge0: received packet on bridge_slave_0 with own address as source address (addr:aa:aa:aa:aa:aa:1b, vlan:0) bridge0: received packet on bridge_slave_0 with own address as source address (addr:aa:aa:aa:aa:aa:1b, vlan:0) bridge0: received packet on bridge_slave_0 with own address as source address (addr:aa:aa:aa:aa:aa:1b, vlan:0) bridge0: received packet on bridge_slave_0 with own address as source address (addr:aa:aa:aa:aa:aa:1b, vlan:0) bridge0: received packet on bridge_slave_0 with own address as source address (addr:aa:aa:aa:aa:aa:1b, vlan:0) bridge0: received packet on bridge_slave_0 with own address as source address (addr:aa:aa:aa:aa:aa:1b, vlan:0)