net_ratelimit: 62 callbacks suppressed
net_ratelimit: 1 callbacks suppressed
BUG: workqueue lockup - pool cpus=0 node=0 flags=0x0 nice=0 stuck for 30s!
Showing busy workqueues and worker pools:
workqueue events: flags=0x100
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=25 refcnt=26
in-flight: 4798:xfrm_state_gc_task for 31s
pending: psi_avgs_work, vmstat_shepherd, 10*nsim_dev_hwstats_traffic_work, psi_avgs_work, free_obj_work, 8*ovs_dp_masks_rebalance, key_garbage_collector, psi_avgs_work
workqueue events_unbound: flags=0x2
pwq 8: cpus=0-1 flags=0x6 nice=0 active=43 refcnt=44
in-flight: 12:cfg80211_wiphy_work for 65s ,3772:nsim_dev_trap_report_work for 49s ,3729:linkwatch_event for 53s ,64:cfg80211_wiphy_work for 70s cfg80211_wiphy_work ,14:call_usermodehelper_exec_work for 74s ,602:cfg80211_wiphy_work for 54s ,54:cfg80211_wiphy_work for 69s cfg80211_wiphy_work ,40:cfg80211_wiphy_work for 70s cfg80211_wiphy_work ,3658:cfg80211_wiphy_work for 58s ,3438:cfg80211_wiphy_work for 69s ,3108:cfg80211_wiphy_work for 55s ,623:cfg80211_wiphy_work for 70s cfg80211_wiphy_work ,94:cfg80211_wiphy_work for 44s
pending: nsim_dev_trap_report_work, 2*cfg80211_wiphy_work, 3*macvlan_process_broadcast, 3*cfg80211_wiphy_work, macvlan_process_broadcast, 7*nsim_dev_trap_report_work, macvlan_process_broadcast, flush_memcg_stats_dwork, cfg80211_wiphy_work, nsim_dev_trap_report_work, macvlan_process_broadcast, crng_reseed, macvlan_process_broadcast, toggle_allocation_gate, macvlan_process_broadcast
workqueue events_power_efficient: flags=0x82
pwq 8: cpus=0-1 flags=0x6 nice=0 active=13 refcnt=14
pending: check_lifetime, neigh_periodic_work, gc_worker, wg_ratelimiter_gc_entries, fb_flashcursor, do_cache_clean, neigh_managed_work, neigh_periodic_work, reg_check_chans_work, neigh_managed_work, 3*check_lifetime
workqueue events_dfl_long: flags=0x2
pwq 8: cpus=0-1 flags=0x6 nice=0 active=8 refcnt=9
pending: 8*defense_work_handler
workqueue rcu_gp: flags=0x108
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: process_srcu
workqueue netns: flags=0x6000a
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=6 MAYDAY
in-flight: 8(RESCUER):cleanup_net for 30s
pending: mayday_cursor_func
workqueue mm_percpu_wq: flags=0x108
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: vmstat_update
workqueue writeback: flags=0x4a
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
in-flight: 6176:wb_workfn for 33s
workqueue kblockd: flags=0x118
pwq 3: cpus=0 node=0 flags=0x0 nice=-20 active=2 refcnt=3
in-flight: 4389:blk_mq_timeout_work for 10s
pending: blk_mq_timeout_work
workqueue mld: flags=0x40108
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=14
in-flight: 4314:mld_ifc_work for 30s
inactive: 5*mld_ifc_work, 2*mld_dad_work, 5*mld_ifc_work
workqueue ipv6_addrconf: flags=0x6000a
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=46
in-flight: 3318:addrconf_dad_work for 73s
inactive: 39*addrconf_dad_work, 3*addrconf_verify_work
workqueue bat_events: flags=0x6000a
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=58 MAYDAY
in-flight: 2854(RESCUER):batadv_tt_purge for 3s
pending: mayday_cursor_func
inactive: batadv_dat_purge, batadv_bla_periodic_work, 3*batadv_tt_purge, batadv_dat_purge, batadv_bla_periodic_work, 2*batadv_tt_purge, batadv_bla_periodic_work, 2*batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, 2*batadv_bla_periodic_work, batadv_dat_purge, 7*batadv_mcast_mla_update, 6*batadv_iv_send_outstanding_bat_ogm_packet, 5*batadv_purge_orig, 10*batadv_iv_send_outstanding_bat_ogm_packet, batadv_purge_orig, batadv_iv_send_outstanding_bat_ogm_packet, batadv_purge_orig, 2*batadv_iv_send_outstanding_bat_ogm_packet
workqueue loop3: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
pending: loop_rootcg_workfn
workqueue wg-kex-wg0: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
pending: wg_packet_decrypt_worker, wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
pending: 2*wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-crypt-wg0: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-crypt-wg2: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
in-flight: 870:wg_packet_encrypt_worker for 43s
pending: wg_packet_encrypt_worker
workqueue wg-kex-wg0: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
in-flight: 143:wg_packet_handshake_send_worker for 10s
workqueue wg-crypt-wg0: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
pending: 2*wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
pending: 2*wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
pending: 2*wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_encrypt_worker
workqueue bond1: flags=0x2000a
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=5
pending: bond_mii_monitor
inactive: bond_3ad_state_machine_handler
workqueue hci3: flags=0x20012
pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
in-flight: 4713:hci_cmd_sync_work for 23s
workqueue bond0: flags=0x2000a
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=5 MAYDAY
pending: bond_arp_monitor
workqueue wg-kex-wg0: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
pending: wg_packet_decrypt_worker, wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-crypt-wg1: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker
workqueue wg-crypt-wg2: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_encrypt_worker
workqueue hci0: flags=0x20012
pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
in-flight: 4722:hci_cmd_sync_work for 7s
workqueue wg-kex-wg0: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
in-flight: 4889:wg_packet_decrypt_worker for 41s ,4805:wg_packet_encrypt_worker for 30s
pending: wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg2: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker
workqueue bond1: flags=0x2000a
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=4
pending: bond_mii_monitor
workqueue bond6: flags=0x2000a
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=5
pending: bond_mii_monitor
inactive: bond_3ad_state_machine_handler
workqueue bond7: flags=0x2000a
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=6 MAYDAY
pending: bond_mii_monitor
inactive: bond_3ad_state_machine_handler
workqueue bond2: flags=0x2000a
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=5
pending: bond_mii_monitor
inactive: bond_3ad_state_machine_handler
workqueue bond1: flags=0x2000a
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=5
pending: bond_alb_monitor
inactive: bond_mii_monitor
pool 2: cpus=0 node=0 flags=0x0 nice=0 hung=30s workers=9 idle: 10 9 4813 4890
pool 3: cpus=0 node=0 flags=0x0 nice=-20 hung=0s workers=2 idle: 11
pool 8: cpus=0-1 flags=0x6 nice=0 hung=0s workers=17 manager: 533
pool 9: cpus=0-1 node=0 flags=0x4 nice=-20 hung=0s workers=6 idle: 4719 50 4721 4720
Showing backtraces of busy workers in stalled worker pools:
pool 2:
task:kworker/0:7 state:R running task stack:0 pid:4889 tgid:4889 ppid:2 task_flags:0x4208060 flags:0x00000010
Workqueue: wg-crypt-wg0 wg_packet_decrypt_worker
Call trace:
__switch_to+0x2b4/0x6e4 arch/arm64/kernel/process.c:775 (T)
context_switch kernel/sched/core.c:5520 [inline]
__schedule+0x1370/0x2d80 kernel/sched/core.c:7270
preempt_schedule_common+0xd4/0x190 kernel/sched/core.c:7449
preempt_schedule+0x60/0x78 kernel/sched/core.c:7473
__local_bh_enable_ip+0x20c/0x35c kernel/softirq.c:480
__raw_spin_unlock_bh include/linux/spinlock_api_smp.h:237 [inline]
_raw_spin_unlock_bh+0x3c/0x4c kernel/locking/spinlock.c:245
spin_unlock_bh include/linux/spinlock.h:407 [inline]
ptr_ring_consume_bh include/linux/ptr_ring.h:399 [inline]
wg_packet_decrypt_worker+0x980/0xa00 drivers/net/wireguard/receive.c:499
process_one_work kernel/workqueue.c:3387 [inline]
process_scheduled_works+0x91c/0x1250 kernel/workqueue.c:3470
worker_thread+0x798/0xbd0 kernel/workqueue.c:3551
kthread+0x304/0x3d4 kernel/kthread.c:436
ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:838
pool 2:
task:kworker/0:3 state:R running task stack:0 pid:4314 tgid:4314 ppid:2 task_flags:0x4208060 flags:0x00000010
Workqueue: mld mld_ifc_work
Call trace:
__switch_to+0x2b4/0x6e4 arch/arm64/kernel/process.c:775 (T)
context_switch kernel/sched/core.c:5520 [inline]
__schedule+0x1370/0x2d80 kernel/sched/core.c:7270
preempt_schedule_common+0xd4/0x190 kernel/sched/core.c:7449
preempt_schedule+0x60/0x78 kernel/sched/core.c:7473
__local_bh_enable_ip+0x20c/0x35c kernel/softirq.c:480
__raw_read_unlock_bh include/linux/rwlock_api_smp.h:283 [inline]
_raw_read_unlock_bh+0x3c/0x4c kernel/locking/spinlock.c:319
ipv6_get_lladdr+0x284/0x3c0 net/ipv6/addrconf.c:1951
mld_newpack+0x36c/0x7b4 net/ipv6/mcast.c:1804
add_grhead+0x6c/0x260 net/ipv6/mcast.c:1901
add_grec+0xf9c/0x1214 net/ipv6/mcast.c:2040
mld_send_cr net/ipv6/mcast.c:2165 [inline]
mld_ifc_work+0x620/0x9e4 net/ipv6/mcast.c:2711
process_one_work kernel/workqueue.c:3387 [inline]
process_scheduled_works+0x91c/0x1250 kernel/workqueue.c:3470
worker_thread+0x798/0xbd0 kernel/workqueue.c:3551
kthread+0x304/0x3d4 kernel/kthread.c:436
ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:838
pool 2:
task:kworker/0:5 state:R running task stack:0 pid:4805 tgid:4805 ppid:2 task_flags:0x4208060 flags:0x00000010
Workqueue: wg-crypt-wg0 wg_packet_encrypt_worker
Call trace:
__switch_to+0x2b4/0x6e4 arch/arm64/kernel/process.c:775 (T)
context_switch kernel/sched/core.c:5520 [inline]
__schedule+0x1370/0x2d80 kernel/sched/core.c:7270
preempt_schedule_common+0xd4/0x190 kernel/sched/core.c:7449
preempt_schedule kernel/sched/core.c:7473 [inline]
class_preempt_destructor include/linux/preempt.h:500 [inline]
try_to_wake_up+0x5f8/0xdc4 kernel/sched/core.c:4432
wake_up_process+0x18/0x24 kernel/sched/core.c:4557
process_one_work kernel/workqueue.c:3355 [inline]
process_scheduled_works+0x744/0x1250 kernel/workqueue.c:3470
worker_thread+0x798/0xbd0 kernel/workqueue.c:3551
kthread+0x304/0x3d4 kernel/kthread.c:436
ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:838
pool 2:
task:kworker/0:2 state:R running task stack:0 pid:870 tgid:870 ppid:2 task_flags:0x4208060 flags:0x00000010
Workqueue: wg-crypt-wg2 wg_packet_encrypt_worker
Call trace:
__switch_to+0x2b4/0x6e4 arch/arm64/kernel/process.c:775 (T)
context_switch kernel/sched/core.c:5520 [inline]
__schedule+0x1370/0x2d80 kernel/sched/core.c:7270
preempt_schedule_common+0xd4/0x190 kernel/sched/core.c:7449
preempt_schedule+0x60/0x78 kernel/sched/core.c:7473
__local_bh_enable_ip+0x20c/0x35c kernel/softirq.c:480
__raw_spin_unlock_bh include/linux/spinlock_api_smp.h:237 [inline]
_raw_spin_unlock_bh+0x3c/0x4c kernel/locking/spinlock.c:245
spin_unlock_bh include/linux/spinlock.h:407 [inline]
ptr_ring_consume_bh include/linux/ptr_ring.h:399 [inline]
wg_packet_encrypt_worker+0x25c/0x1290 drivers/net/wireguard/send.c:293
process_one_work kernel/workqueue.c:3387 [inline]
process_scheduled_works+0x91c/0x1250 kernel/workqueue.c:3470
worker_thread+0x798/0xbd0 kernel/workqueue.c:3551
kthread+0x304/0x3d4 kernel/kthread.c:436
ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:838
net_ratelimit: 10 callbacks suppressed
BUG: workqueue lockup - pool cpus=0 node=0 flags=0x0 nice=0 stuck for 60s!
Showing busy workqueues and worker pools:
workqueue events: flags=0x100
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=25 refcnt=26
in-flight: 4798:xfrm_state_gc_task for 61s
pending: psi_avgs_work, vmstat_shepherd, 10*nsim_dev_hwstats_traffic_work, psi_avgs_work, free_obj_work, 8*ovs_dp_masks_rebalance, key_garbage_collector, psi_avgs_work
workqueue events_unbound: flags=0x2
pwq 8: cpus=0-1 flags=0x6 nice=0 active=43 refcnt=44
in-flight: 12:cfg80211_wiphy_work for 95s ,3772:nsim_dev_trap_report_work for 79s ,3729:linkwatch_event for 83s ,64:cfg80211_wiphy_work for 100s cfg80211_wiphy_work ,14:call_usermodehelper_exec_work for 104s ,602:cfg80211_wiphy_work for 84s ,54:cfg80211_wiphy_work for 99s cfg80211_wiphy_work ,40:cfg80211_wiphy_work for 100s cfg80211_wiphy_work ,3658:cfg80211_wiphy_work for 88s ,3438:cfg80211_wiphy_work for 99s ,3108:cfg80211_wiphy_work for 85s ,623:cfg80211_wiphy_work for 101s cfg80211_wiphy_work ,94:cfg80211_wiphy_work for 74s
pending: nsim_dev_trap_report_work, 2*cfg80211_wiphy_work, 3*macvlan_process_broadcast, 3*cfg80211_wiphy_work, macvlan_process_broadcast, 7*nsim_dev_trap_report_work, macvlan_process_broadcast, flush_memcg_stats_dwork, cfg80211_wiphy_work, nsim_dev_trap_report_work, macvlan_process_broadcast, crng_reseed, macvlan_process_broadcast, toggle_allocation_gate, macvlan_process_broadcast
workqueue events_power_efficient: flags=0x82
pwq 8: cpus=0-1 flags=0x6 nice=0 active=14 refcnt=15
pending: check_lifetime, neigh_periodic_work, gc_worker, wg_ratelimiter_gc_entries, fb_flashcursor, do_cache_clean, neigh_managed_work, neigh_periodic_work, reg_check_chans_work, neigh_managed_work, 4*check_lifetime
workqueue events_dfl_long: flags=0x2
pwq 8: cpus=0-1 flags=0x6 nice=0 active=8 refcnt=9
pending: 8*defense_work_handler
workqueue rcu_gp: flags=0x108
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: process_srcu
workqueue netns: flags=0x6000a
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=6 MAYDAY
in-flight: 8(RESCUER):cleanup_net for 60s
pending: mayday_cursor_func
workqueue mm_percpu_wq: flags=0x108
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: vmstat_update
workqueue writeback: flags=0x4a
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
in-flight: 6176:wb_workfn for 63s
workqueue kblockd: flags=0x118
pwq 3: cpus=0 node=0 flags=0x2 nice=-20 active=2 refcnt=5 MAYDAY
in-flight: 38(RESCUER):blk_mq_timeout_work for 17s ,4389:blk_mq_timeout_work for 40s
pending: mayday_cursor_func
workqueue mld: flags=0x40108
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=14
in-flight: 4314:mld_ifc_work for 60s
inactive: 5*mld_ifc_work, 2*mld_dad_work, 5*mld_ifc_work
workqueue ipv6_addrconf: flags=0x6000a
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=47
in-flight: 3318:addrconf_dad_work for 103s
inactive: 39*addrconf_dad_work, 4*addrconf_verify_work
workqueue bat_events: flags=0x6000a
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=59 MAYDAY
in-flight: 2854(RESCUER):batadv_dat_purge for 8s
pending: mayday_cursor_func
inactive: batadv_bla_periodic_work, 3*batadv_tt_purge, batadv_dat_purge, batadv_bla_periodic_work, 2*batadv_tt_purge, batadv_bla_periodic_work, 2*batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, 2*batadv_bla_periodic_work, batadv_dat_purge, 7*batadv_mcast_mla_update, 6*batadv_iv_send_outstanding_bat_ogm_packet, 5*batadv_purge_orig, 10*batadv_iv_send_outstanding_bat_ogm_packet, batadv_purge_orig, batadv_iv_send_outstanding_bat_ogm_packet, batadv_purge_orig, 2*batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_tt_purge
workqueue loop3: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
pending: loop_rootcg_workfn
workqueue wg-kex-wg0: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
pending: wg_packet_decrypt_worker, wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
pending: 2*wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-crypt-wg0: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-crypt-wg2: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
in-flight: 870:wg_packet_encrypt_worker for 73s
pending: wg_packet_encrypt_worker
workqueue wg-kex-wg0: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
in-flight: 143:wg_packet_handshake_send_worker for 40s
workqueue wg-crypt-wg0: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
pending: 2*wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
pending: 2*wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
pending: 2*wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_encrypt_worker
workqueue hci3: flags=0x20012
pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
in-flight: 4713:hci_cmd_sync_work for 53s
workqueue bond0: flags=0x2000a
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=4
pending: bond_arp_monitor
workqueue wg-kex-wg0: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
pending: wg_packet_decrypt_worker, wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-crypt-wg1: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker
workqueue wg-crypt-wg2: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_encrypt_worker
workqueue hci0: flags=0x20012
pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
in-flight: 4722:hci_cmd_sync_work for 37s
workqueue wg-kex-wg0: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
in-flight: 4889:wg_packet_decrypt_worker for 71s ,4805:wg_packet_encrypt_worker for 60s
pending: wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg2: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker
pool 2: cpus=0 node=0 flags=0x0 nice=0 hung=60s workers=9 idle: 10 9 4813 4890
pool 3: cpus=0 node=0 flags=0x2 nice=-20 hung=17s workers=2 manager: 11
pool 8: cpus=0-1 flags=0x6 nice=0 hung=0s workers=17 manager: 533
pool 9: cpus=0-1 node=0 flags=0x4 nice=-20 hung=0s workers=6 idle: 4719 50 4721 4720
Showing backtraces of busy workers in stalled worker pools:
pool 2:
task:kworker/0:7 state:R running task stack:0 pid:4889 tgid:4889 ppid:2 task_flags:0x4208060 flags:0x00000010
Workqueue: wg-crypt-wg0 wg_packet_decrypt_worker
Call trace:
__switch_to+0x2b4/0x6e4 arch/arm64/kernel/process.c:775 (T)
context_switch kernel/sched/core.c:5520 [inline]
__schedule+0x1370/0x2d80 kernel/sched/core.c:7270
preempt_schedule_common+0xd4/0x190 kernel/sched/core.c:7449
preempt_schedule+0x60/0x78 kernel/sched/core.c:7473
__local_bh_enable_ip+0x20c/0x35c kernel/softirq.c:480
__raw_spin_unlock_bh include/linux/spinlock_api_smp.h:237 [inline]
_raw_spin_unlock_bh+0x3c/0x4c kernel/locking/spinlock.c:245
spin_unlock_bh include/linux/spinlock.h:407 [inline]
ptr_ring_consume_bh include/linux/ptr_ring.h:399 [inline]
wg_packet_decrypt_worker+0x980/0xa00 drivers/net/wireguard/receive.c:499
process_one_work kernel/workqueue.c:3387 [inline]
process_scheduled_works+0x91c/0x1250 kernel/workqueue.c:3470
worker_thread+0x798/0xbd0 kernel/workqueue.c:3551
kthread+0x304/0x3d4 kernel/kthread.c:436
ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:838
pool 2:
task:kworker/0:3 state:R running task stack:0 pid:4314 tgid:4314 ppid:2 task_flags:0x4208060 flags:0x00000010
Workqueue: mld mld_ifc_work
Call trace:
__switch_to+0x2b4/0x6e4 arch/arm64/kernel/process.c:775 (T)
context_switch kernel/sched/core.c:5520 [inline]
__schedule+0x1370/0x2d80 kernel/sched/core.c:7270
preempt_schedule_common+0xd4/0x190 kernel/sched/core.c:7449
preempt_schedule+0x60/0x78 kernel/sched/core.c:7473
__local_bh_enable_ip+0x20c/0x35c kernel/softirq.c:480
__raw_read_unlock_bh include/linux/rwlock_api_smp.h:283 [inline]
_raw_read_unlock_bh+0x3c/0x4c kernel/locking/spinlock.c:319
ipv6_get_lladdr+0x284/0x3c0 net/ipv6/addrconf.c:1951
mld_newpack+0x36c/0x7b4 net/ipv6/mcast.c:1804
add_grhead+0x6c/0x260 net/ipv6/mcast.c:1901
add_grec+0xf9c/0x1214 net/ipv6/mcast.c:2040
mld_send_cr net/ipv6/mcast.c:2165 [inline]
mld_ifc_work+0x620/0x9e4 net/ipv6/mcast.c:2711
process_one_work kernel/workqueue.c:3387 [inline]
process_scheduled_works+0x91c/0x1250 kernel/workqueue.c:3470
worker_thread+0x798/0xbd0 kernel/workqueue.c:3551
kthread+0x304/0x3d4 kernel/kthread.c:436
ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:838
pool 2:
task:kworker/0:5 state:R running task stack:0 pid:4805 tgid:4805 ppid:2 task_flags:0x4208060 flags:0x00000010
Workqueue: wg-crypt-wg0 wg_packet_encrypt_worker
Call trace:
__switch_to+0x2b4/0x6e4 arch/arm64/kernel/process.c:775 (T)
context_switch kernel/sched/core.c:5520 [inline]
__schedule+0x1370/0x2d80 kernel/sched/core.c:7270
preempt_schedule_common+0xd4/0x190 kernel/sched/core.c:7449
preempt_schedule kernel/sched/core.c:7473 [inline]
class_preempt_destructor include/linux/preempt.h:500 [inline]
try_to_wake_up+0x5f8/0xdc4 kernel/sched/core.c:4432
wake_up_process+0x18/0x24 kernel/sched/core.c:4557
process_one_work kernel/workqueue.c:3355 [inline]
process_scheduled_works+0x744/0x1250 kernel/workqueue.c:3470
worker_thread+0x798/0xbd0 kernel/workqueue.c:3551
kthread+0x304/0x3d4 kernel/kthread.c:436
ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:838
pool 2:
task:kworker/0:2 state:R running task stack:0 pid:870 tgid:870 ppid:2 task_flags:0x4208060 flags:0x00000010
Workqueue: wg-crypt-wg2 wg_packet_encrypt_worker
Call trace:
__switch_to+0x2b4/0x6e4 arch/arm64/kernel/process.c:775 (T)
context_switch kernel/sched/core.c:5520 [inline]
__schedule+0x1370/0x2d80 kernel/sched/core.c:7270
preempt_schedule_common+0xd4/0x190 kernel/sched/core.c:7449
preempt_schedule+0x60/0x78 kernel/sched/core.c:7473
__local_bh_enable_ip+0x20c/0x35c kernel/softirq.c:480
__raw_spin_unlock_bh include/linux/spinlock_api_smp.h:237 [inline]
_raw_spin_unlock_bh+0x3c/0x4c kernel/locking/spinlock.c:245
spin_unlock_bh include/linux/spinlock.h:407 [inline]
ptr_ring_consume_bh include/linux/ptr_ring.h:399 [inline]
wg_packet_encrypt_worker+0x25c/0x1290 drivers/net/wireguard/send.c:293
process_one_work kernel/workqueue.c:3387 [inline]
process_scheduled_works+0x91c/0x1250 kernel/workqueue.c:3470
worker_thread+0x798/0xbd0 kernel/workqueue.c:3551
kthread+0x304/0x3d4 kernel/kthread.c:436
ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:838
BUG: workqueue lockup - pool cpus=0 node=0 flags=0x0 nice=0 stuck for 90s!
BUG: workqueue lockup - pool cpus=0 node=0 flags=0x2 nice=-20 stuck for 47s!
Showing busy workqueues and worker pools:
workqueue events: flags=0x100
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=25 refcnt=26
in-flight: 4798:xfrm_state_gc_task for 91s
pending: psi_avgs_work, vmstat_shepherd, 10*nsim_dev_hwstats_traffic_work, psi_avgs_work, free_obj_work, 8*ovs_dp_masks_rebalance, key_garbage_collector, psi_avgs_work
workqueue events_unbound: flags=0x2
pwq 8: cpus=0-1 flags=0x6 nice=0 active=43 refcnt=44
in-flight: 12:cfg80211_wiphy_work for 125s ,3772:nsim_dev_trap_report_work for 109s ,3729:linkwatch_event for 113s ,64:cfg80211_wiphy_work for 130s cfg80211_wiphy_work ,14:call_usermodehelper_exec_work for 134s ,602:cfg80211_wiphy_work for 114s ,54:cfg80211_wiphy_work for 129s cfg80211_wiphy_work ,40:cfg80211_wiphy_work for 131s cfg80211_wiphy_work ,3658:cfg80211_wiphy_work for 118s ,3438:cfg80211_wiphy_work for 129s ,3108:cfg80211_wiphy_work for 115s ,623:cfg80211_wiphy_work for 131s cfg80211_wiphy_work ,94:cfg80211_wiphy_work for 104s
pending: nsim_dev_trap_report_work, 2*cfg80211_wiphy_work, 3*macvlan_process_broadcast, 3*cfg80211_wiphy_work, macvlan_process_broadcast, 7*nsim_dev_trap_report_work, macvlan_process_broadcast, flush_memcg_stats_dwork, cfg80211_wiphy_work, nsim_dev_trap_report_work, macvlan_process_broadcast, crng_reseed, macvlan_process_broadcast, toggle_allocation_gate, macvlan_process_broadcast
workqueue events_power_efficient: flags=0x82
pwq 8: cpus=0-1 flags=0x6 nice=0 active=17 refcnt=18
pending: check_lifetime, neigh_periodic_work, gc_worker, wg_ratelimiter_gc_entries, fb_flashcursor, do_cache_clean, neigh_managed_work, neigh_periodic_work, reg_check_chans_work, neigh_managed_work, 7*check_lifetime
workqueue events_dfl_long: flags=0x2
pwq 8: cpus=0-1 flags=0x6 nice=0 active=8 refcnt=9
pending: 8*defense_work_handler
workqueue rcu_gp: flags=0x108
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: process_srcu
workqueue netns: flags=0x6000a
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=6 MAYDAY
in-flight: 8(RESCUER):cleanup_net for 90s
pending: mayday_cursor_func
workqueue mm_percpu_wq: flags=0x108
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: vmstat_update
workqueue writeback: flags=0x4a
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
in-flight: 6176:wb_workfn for 93s
workqueue kblockd: flags=0x118
pwq 3: cpus=0 node=0 flags=0x2 nice=-20 active=2 refcnt=5 MAYDAY
in-flight: 38(RESCUER):blk_mq_timeout_work for 47s ,4389:blk_mq_timeout_work for 70s
pending: mayday_cursor_func
workqueue mld: flags=0x40108
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=14
in-flight: 4314:mld_ifc_work for 90s
inactive: 5*mld_ifc_work, 2*mld_dad_work, 5*mld_ifc_work
workqueue ipv6_addrconf: flags=0x6000a
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=51
in-flight: 3318:addrconf_dad_work for 133s
inactive: 39*addrconf_dad_work, 8*addrconf_verify_work
workqueue bat_events: flags=0x6000a
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=58 MAYDAY
in-flight: 2854(RESCUER):batadv_tt_purge for 3s
pending: mayday_cursor_func
inactive: batadv_tt_purge, batadv_dat_purge, batadv_bla_periodic_work, 2*batadv_tt_purge, batadv_bla_periodic_work, 2*batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, 2*batadv_bla_periodic_work, batadv_dat_purge, 7*batadv_mcast_mla_update, 6*batadv_iv_send_outstanding_bat_ogm_packet, 5*batadv_purge_orig, 10*batadv_iv_send_outstanding_bat_ogm_packet, batadv_purge_orig, batadv_iv_send_outstanding_bat_ogm_packet, batadv_purge_orig, 2*batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_tt_purge, batadv_dat_purge, batadv_bla_periodic_work
workqueue loop3: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
pending: loop_rootcg_workfn
workqueue wg-kex-wg0: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
pending: wg_packet_decrypt_worker, wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
pending: 2*wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-crypt-wg0: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-crypt-wg2: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
in-flight: 870:wg_packet_encrypt_worker for 103s
pending: wg_packet_encrypt_worker
workqueue wg-kex-wg0: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
in-flight: 143:wg_packet_handshake_send_worker for 70s
workqueue wg-crypt-wg0: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
pending: 2*wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
pending: 2*wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
pending: 2*wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_encrypt_worker
workqueue bond1: flags=0x2000a
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=6 MAYDAY
pending: bond_mii_monitor
inactive: bond_3ad_state_machine_handler
workqueue hci3: flags=0x20012
pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
in-flight: 4713:hci_cmd_sync_work for 83s
workqueue bond0: flags=0x2000a
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=5 MAYDAY
pending: bond_arp_monitor
workqueue wg-kex-wg0: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
pending: wg_packet_decrypt_worker, wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-crypt-wg1: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker
workqueue wg-crypt-wg2: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_encrypt_worker
workqueue hci0: flags=0x20012
pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
in-flight: 4722:hci_cmd_sync_work for 67s
workqueue wg-kex-wg0: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
in-flight: 4889:wg_packet_decrypt_worker for 101s ,4805:wg_packet_encrypt_worker for 90s
pending: wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg2: flags=0x6
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker
workqueue bond1: flags=0x2000a
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=6 MAYDAY
pending: bond_mii_monitor
inactive: bond_3ad_state_machine_handler
workqueue bond6: flags=0x2000a
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=6 MAYDAY
pending: bond_mii_monitor
inactive: bond_3ad_state_machine_handler
workqueue bond7: flags=0x2000a
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=6 MAYDAY
pending: bond_mii_monitor
inactive: bond_3ad_state_machine_handler
workqueue hci2: flags=0x20012
pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
in-flight: 50:hci_cmd_sync_work for 21s
workqueue bond2: flags=0x2000a
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=6 MAYDAY
pending: bond_3ad_state_machine_handler
inactive: bond_mii_monitor
workqueue bond1: flags=0x2000a
pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=6 MAYDAY
pending: bond_alb_monitor
inactive: bond_mii_monitor
pool 2: cpus=0 node=0 flags=0x0 nice=0 hung=90s workers=9 idle: 10 9 4813 4890
pool 3: cpus=0 node=0 flags=0x2 nice=-20 hung=47s workers=2 manager: 11
pool 8: cpus=0-1 flags=0x6 nice=0 hung=1s workers=17 manager: 533
pool 9: cpus=0-1 node=0 flags=0x4 nice=-20 hung=0s workers=6 idle: 4719 4721 4720
Showing backtraces of busy workers in stalled worker pools:
pool 2:
task:kworker/0:7 state:R running task stack:0 pid:4889 tgid:4889 ppid:2 task_flags:0x4208060 flags:0x00000010
Workqueue: wg-crypt-wg0 wg_packet_decrypt_worker
Call trace:
__switch_to+0x2b4/0x6e4 arch/arm64/kernel/process.c:775 (T)
context_switch kernel/sched/core.c:5520 [inline]
__schedule+0x1370/0x2d80 kernel/sched/core.c:7270
preempt_schedule_common+0xd4/0x190 kernel/sched/core.c:7449
preempt_schedule+0x60/0x78 kernel/sched/core.c:7473
__local_bh_enable_ip+0x20c/0x35c kernel/softirq.c:480
__raw_spin_unlock_bh include/linux/spinlock_api_smp.h:237 [inline]
_raw_spin_unlock_bh+0x3c/0x4c kernel/locking/spinlock.c:245
spin_unlock_bh include/linux/spinlock.h:407 [inline]
ptr_ring_consume_bh include/linux/ptr_ring.h:399 [inline]
wg_packet_decrypt_worker+0x980/0xa00 drivers/net/wireguard/receive.c:499
process_one_work kernel/workqueue.c:3387 [inline]
process_scheduled_works+0x91c/0x1250 kernel/workqueue.c:3470
worker_thread+0x798/0xbd0 kernel/workqueue.c:3551
kthread+0x304/0x3d4 kernel/kthread.c:436
ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:838
pool 2:
task:kworker/0:3 state:R running task stack:0 pid:4314 tgid:4314 ppid:2 task_flags:0x4208060 flags:0x00000010
Workqueue: mld mld_ifc_work
Call trace:
__switch_to+0x2b4/0x6e4 arch/arm64/kernel/process.c:775 (T)
context_switch kernel/sched/core.c:5520 [inline]
__schedule+0x1370/0x2d80 kernel/sched/core.c:7270
preempt_schedule_common+0xd4/0x190 kernel/sched/core.c:7449
preempt_schedule+0x60/0x78 kernel/sched/core.c:7473
__local_bh_enable_ip+0x20c/0x35c kernel/softirq.c:480
__raw_read_unlock_bh include/linux/rwlock_api_smp.h:283 [inline]
_raw_read_unlock_bh+0x3c/0x4c kernel/locking/spinlock.c:319
ipv6_get_lladdr+0x284/0x3c0 net/ipv6/addrconf.c:1951
mld_newpack+0x36c/0x7b4 net/ipv6/mcast.c:1804
add_grhead+0x6c/0x260 net/ipv6/mcast.c:1901
add_grec+0xf9c/0x1214 net/ipv6/mcast.c:2040
mld_send_cr net/ipv6/mcast.c:2165 [inline]
mld_ifc_work+0x620/0x9e4 net/ipv6/mcast.c:2711
process_one_work kernel/workqueue.c:3387 [inline]
process_scheduled_works+0x91c/0x1250 kernel/workqueue.c:3470
worker_thread+0x798/0xbd0 kernel/workqueue.c:3551
kthread+0x304/0x3d4 kernel/kthread.c:436
ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:838
pool 2:
task:kworker/0:5 state:R running task stack:0 pid:4805 tgid:4805 ppid:2 task_flags:0x4208060 flags:0x00000010
Workqueue: wg-crypt-wg0 wg_packet_encrypt_worker
Call trace:
__switch_to+0x2b4/0x6e4 arch/arm64/kernel/process.c:775 (T)
context_switch kernel/sched/core.c:5520 [inline]
__schedule+0x1370/0x2d80 kernel/sched/core.c:7270
preempt_schedule_common+0xd4/0x190 kernel/sched/core.c:7449
preempt_schedule kernel/sched/core.c:7473 [inline]
class_preempt_destructor include/linux/preempt.h:500 [inline]
try_to_wake_up+0x5f8/0xdc4 kernel/sched/core.c:4432
wake_up_process+0x18/0x24 kernel/sched/core.c:4557
process_one_work kernel/workqueue.c:3355 [inline]
process_scheduled_works+0x744/0x1250 kernel/workqueue.c:3470
worker_thread+0x798/0xbd0 kernel/workqueue.c:3551
kthread+0x304/0x3d4 kernel/kthread.c:436
ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:838
pool 2:
task:kworker/0:2 state:R running task stack:0 pid:870 tgid:870 ppid:2 task_flags:0x4208060 flags:0x00000010
Workqueue: wg-crypt-wg2 wg_packet_encrypt_worker
Call trace:
__switch_to+0x2b4/0x6e4 arch/arm64/kernel/process.c:775 (T)
context_switch kernel/sched/core.c:5520 [inline]
__schedule+0x1370/0x2d80 kernel/sched/core.c:7270
preempt_schedule_common+0xd4/0x190 kernel/sched/core.c:7449
preempt_schedule+0x60/0x78 kernel/sched/core.c:7473
__local_bh_enable_ip+0x20c/0x35c kernel/softirq.c:480
__raw_spin_unlock_bh include/linux/spinlock_api_smp.h:237 [inline]
_raw_spin_unlock_bh+0x3c/0x4c kernel/locking/spinlock.c:245
spin_unlock_bh include/linux/spinlock.h:407 [inline]
ptr_ring_consume_bh include/linux/ptr_ring.h:399 [inline]
wg_packet_encrypt_worker+0x25c/0x1290 drivers/net/wireguard/send.c:293
process_one_work kernel/workqueue.c:3387 [inline]
process_scheduled_works+0x91c/0x1250 kernel/workqueue.c:3470
worker_thread+0x798/0xbd0 kernel/workqueue.c:3551
kthread+0x304/0x3d4 kernel/kthread.c:436
ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:838
pool 3: no worker in running state, cpu=0 is busy (nr_workers=2 nr_idle=0)
The pool might have trouble waking an idle worker.
Backtrace of last woken worker:
task:kworker/0:0H state:D stack:0 pid:11 tgid:11 ppid:2 task_flags:0x4208060 flags:0x00000010
Workqueue: 0x0 (kblockd)
Call trace:
__switch_to+0x2b4/0x6e4 arch/arm64/kernel/process.c:775 (T)
context_switch kernel/sched/core.c:5520 [inline]
__schedule+0x1370/0x2d80 kernel/sched/core.c:7270
__schedule_loop kernel/sched/core.c:7347 [inline]
schedule+0x13c/0x20c kernel/sched/core.c:7362
schedule_timeout+0xc0/0x28c kernel/time/sleep_timeout.c:75
do_wait_for_common kernel/sched/completion.c:100 [inline]
__wait_for_common kernel/sched/completion.c:121 [inline]
wait_for_common+0x220/0x394 kernel/sched/completion.c:132
wait_for_completion_killable+0x18/0x30 kernel/sched/completion.c:259
__kthread_create_on_node+0x29c/0x3b0 kernel/kthread.c:507
kthread_create_on_node+0xe4/0x134 kernel/kthread.c:559
create_worker+0x15c/0x52c kernel/workqueue.c:2910
maybe_create_worker kernel/workqueue.c:3177 [inline]
manage_workers kernel/workqueue.c:3254 [inline]
worker_thread+0x2ac/0xbd0 kernel/workqueue.c:3526
kthread+0x304/0x3d4 kernel/kthread.c:436
ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:838
NMI backtrace for cpu 0
CPU: 0 UID: 0 PID: 2854 Comm: kworker/R-bat_e Not tainted syzkaller #0 PREEMPT
Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 08/14/2026
Workqueue: bat_events batadv_tt_purge
Call trace:
show_stack+0x2c/0x3c arch/arm64/kernel/stacktrace.c:499 (C)
__dump_stack+0x30/0x40 lib/dump_stack.c:94
dump_stack_lvl+0xd8/0x12c lib/dump_stack.c:120
dump_stack+0x1c/0x28 lib/dump_stack.c:129
nmi_cpu_backtrace+0x270/0x2b8 lib/nmi_backtrace.c:123
nmi_trigger_cpumask_backtrace+0x1c0/0x430 lib/nmi_backtrace.c:66
arch_trigger_cpumask_backtrace+0x30/0x40 arch/arm64/kernel/smp.c:972
cpumask_backtrace+0x28/0x38 lib/nmi_backtrace.c:136
trigger_single_cpu_backtrace include/linux/nmi.h:182 [inline]
show_cpu_pool_busy_workers kernel/workqueue.c:7876 [inline]
show_cpu_pools_busy_workers+0x640/0x830 kernel/workqueue.c:7890
wq_watchdog_timer_fn+0x648/0x740 kernel/workqueue.c:8006
call_timer_fn+0x194/0xa4c kernel/time/timer.c:1748
expire_timers kernel/time/timer.c:1799 [inline]
__run_timers kernel/time/timer.c:2374 [inline]
__run_timer_base+0x530/0x744 kernel/time/timer.c:2386
run_timer_base+0x84/0x1b8 kernel/time/timer.c:2395
run_timer_softirq+0x28/0x48 kernel/time/timer.c:2406
handle_softirqs+0x2ec/0xd98 kernel/softirq.c:645
__do_softirq+0x14/0x20 kernel/softirq.c:679
____do_softirq+0x14/0x20 arch/arm64/kernel/irq.c:78
call_on_irq_stack+0x30/0x48 arch/arm64/kernel/entry.S:885
do_softirq_own_stack+0x20/0x2c arch/arm64/kernel/irq.c:83
do_softirq+0x90/0xf8 kernel/softirq.c:546
__local_bh_enable_ip+0x240/0x35c kernel/softirq.c:473
__raw_spin_unlock_bh include/linux/spinlock_api_smp.h:237 [inline]
_raw_spin_unlock_bh+0x3c/0x4c kernel/locking/spinlock.c:245
spin_unlock_bh include/linux/spinlock.h:407 [inline]
batadv_tt_local_purge+0x274/0x2f0 net/batman-adv/translation-table.c:1402
batadv_tt_purge+0x44/0x7b4 net/batman-adv/translation-table.c:3753
process_one_work kernel/workqueue.c:3387 [inline]
process_scheduled_works+0x91c/0x1250 kernel/workqueue.c:3470
rescuer_thread+0x68c/0xd90 kernel/workqueue.c:3694
kthread+0x304/0x3d4 kernel/kthread.c:436
ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:838
net_ratelimit: 10 callbacks suppressed
rcu: INFO: rcu_preempt detected stalls on CPUs/tasks:
rcu: Tasks blocked on level-0 rcu_node (CPUs 0-1): P6176/1:b..l
rcu: (detected by 0, t=10502 jiffies, g=49001, q=1279042 ncpus=1)
task:kworker/u8:16 state:R running task stack:0 pid:6176 tgid:6176 ppid:2 task_flags:0x24208060 flags:0x00000010
Workqueue: writeback wb_workfn (flush-259:0)
Call trace:
__switch_to+0x2b4/0x6e4 arch/arm64/kernel/process.c:775 (T)
context_switch kernel/sched/core.c:5520 [inline]
__schedule+0x1370/0x2d80 kernel/sched/core.c:7270
preempt_schedule_notrace+0xb4/0x154 kernel/sched/core.c:7542
rcu_is_watching+0xf4/0x134 kernel/rcu/tree.c:754
trace_lock_acquire include/trace/events/lock.h:23 [inline]
lock_acquire+0x80/0x370 kernel/locking/lockdep.c:5871
rcu_read_lock include/linux/rcupdate.h:850 [inline]
percpu_ref_put_many+0x4c/0x33c include/linux/percpu-refcount.h:331
percpu_ref_put include/linux/percpu-refcount.h:352 [inline]
blk_mq_dispatch_list+0xa48/0xb2c block/blk-mq.c:2922
blk_mq_flush_plug_list+0x3ac/0x494 block/blk-mq.c:2966
__blk_flush_plug+0x364/0x440 block/blk-core.c:1270
blk_flush_plug include/linux/blk_plug.h:51 [inline]
io_schedule_prepare+0x74/0xa4 kernel/sched/core.c:8159
io_schedule+0x14/0xe0 kernel/sched/core.c:8189
rq_qos_wait+0x1e8/0x2c8 block/blk-rq-qos.c:307
__wbt_wait block/blk-wbt.c:598 [inline]
wbt_wait+0x124/0x2c0 block/blk-wbt.c:658
__rq_qos_throttle+0x6c/0xb4 block/blk-rq-qos.c:66
rq_qos_throttle block/blk-rq-qos.h:164 [inline]
blk_mq_get_new_requests block/blk-mq.c:3032 [inline]
blk_mq_submit_bio+0x12b8/0x24c0 block/blk-mq.c:3170
__submit_bio+0x94/0x2dc block/blk-core.c:673
__submit_bio_noacct_mq block/blk-core.c:756 [inline]
submit_bio_noacct_nocheck+0x288/0xaac block/blk-core.c:790
submit_bio_noacct+0xd68/0x17dc block/blk-core.c:925
submit_bio+0x38c/0x528 block/blk-core.c:966
blk_crypto_submit_bio include/linux/blk-crypto.h:212 [inline]
__bh_submit+0x5e0/0x6f0 fs/buffer.c:1132
__block_write_full_folio+0x790/0xccc fs/buffer.c:1817
block_write_full_folio+0x2a0/0x348 include/linux/highmem.h:-1
blkdev_writepages+0xec/0x184 block/fops.c:487
do_writepages+0x284/0x47c mm/page-writeback.c:2560
__writeback_single_inode+0x140/0x1ae4 fs/fs-writeback.c:1788
writeback_sb_inodes+0x798/0x1720 fs/fs-writeback.c:2099
__writeback_inodes_wb+0xec/0x210 fs/fs-writeback.c:2175
wb_writeback+0x3d0/0x10cc fs/fs-writeback.c:2286
wb_check_old_data_flush fs/fs-writeback.c:2390 [inline]
wb_do_writeback fs/fs-writeback.c:2469 [inline]
wb_workfn+0xa78/0xda8 fs/fs-writeback.c:2497
process_one_work kernel/workqueue.c:3387 [inline]
process_scheduled_works+0x91c/0x1250 kernel/workqueue.c:3470
worker_thread+0x798/0xbd0 kernel/workqueue.c:3551
kthread+0x304/0x3d4 kernel/kthread.c:436
ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:838
rcu: rcu_preempt kthread starved for 9165 jiffies! g49001 f0x2 RCU_GP_WAIT_FQS(5) ->state=R ->cpu=0
rcu: Unless rcu_preempt kthread gets sufficient CPU time, OOM is now expected behavior.
rcu: RCU grace-period kthread stack dump:
task:rcu_preempt state:R running task stack:0 pid:16 tgid:16 ppid:2 task_flags:0x208040 flags:0x00000010
Call trace:
__switch_to+0x2b4/0x6e4 arch/arm64/kernel/process.c:775 (T)
context_switch kernel/sched/core.c:5520 [inline]
__schedule+0x1370/0x2d80 kernel/sched/core.c:7270
__schedule_loop kernel/sched/core.c:7347 [inline]
schedule+0x13c/0x20c kernel/sched/core.c:7362
schedule_timeout+0x13c/0x28c kernel/time/sleep_timeout.c:99
rcu_gp_fqs_loop+0x284/0x13c4 kernel/rcu/tree.c:2122
rcu_gp_kthread+0x148/0x230 kernel/rcu/tree.c:2330
kthread+0x304/0x3d4 kernel/kthread.c:436
ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:838
rcu: Stack dump where RCU GP kthread last ran:
CPU: 0 UID: 0 PID: 1595 Comm: aoe_tx0 Not tainted syzkaller #0 PREEMPT
Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 08/14/2026
pstate: 83400005 (Nzcv daif +PAN -UAO +TCO +DIT -SSBS BTYPE=--)
pc : __raw_spin_unlock_irqrestore include/linux/spinlock_api_smp.h:211 [inline]
pc : _raw_spin_unlock_irqrestore+0x44/0x98 kernel/locking/spinlock.c:221
lr : __raw_spin_unlock_irqrestore include/linux/spinlock_api_smp.h:210 [inline]
lr : _raw_spin_unlock_irqrestore+0x38/0x98 kernel/locking/spinlock.c:221
sp : ffff80008ec36b00
x29: ffff80008ec36b00 x28: dfff800000000000 x27: 1fffe0001f7860ea
x26: ffff0000fbc30750 x25: dfff800000000000 x24: ffff700011d86d64
x23: ffff000076677388 x22: ffff000076677380 x21: ffff0000766773c0
x20: ffff0000fbc30700 x19: 0000000000000000 x18: 000000001998eb66
x17: ffff800125154000 x16: ffff80008ec30000 x15: 0000000000000000
x14: 00000000850ade10 x13: 0000000000000001 x12: 0000000000000000
x11: ffff80008015610c x10: 0000000000000003 x9 : 0000000000000000
x8 : 00000000000000c0 x7 : 0000000000000000 x6 : 0000000000000000
x5 : 0000000000000001 x4 : 0000000000000008 x3 : ffff8000801561b4
x2 : 0000000000000000 x1 : ffff0000ca551d80 x0 : ffff800125154000
Call trace:
__daif_local_irq_restore arch/arm64/include/asm/irqflags.h:175 [inline] (P)
arch_local_irq_restore arch/arm64/include/asm/irqflags.h:195 [inline] (P)
__raw_spin_unlock_irqrestore include/linux/spinlock_api_smp.h:210 [inline] (P)
_raw_spin_unlock_irqrestore+0x44/0x98 kernel/locking/spinlock.c:221 (P)
spin_unlock_irqrestore include/linux/spinlock.h:425 [inline]
ref_tracker_alloc+0x258/0x3fc lib/ref_tracker.c:281
__netdev_tracker_alloc include/linux/netdevice.h:4528 [inline]
netdev_hold include/linux/netdevice.h:4557 [inline]
dst_init+0xbc/0x42c net/core/dst.c:52
dst_alloc+0x118/0x15c net/core/dst.c:94
rt_dst_alloc+0x4c/0x240 net/ipv4/route.c:1681
__mkroute_output net/ipv4/route.c:2684 [inline]
ip_route_output_key_hash_rcu+0x108c/0x1c20 net/ipv4/route.c:2904
ip_route_output_key_hash+0x180/0x2a0 net/ipv4/route.c:2734
__ip_route_output_key include/net/route.h:169 [inline]
ip_route_output_flow+0x3c/0x158 net/ipv4/route.c:2961
ip_route_output_key include/net/route.h:179 [inline]
ip_route_me_harder+0x5d0/0xd80 net/ipv4/netfilter.c:54
synproxy_send_tcp+0x2d4/0x5a4 net/netfilter/nf_synproxy_core.c:434
synproxy_send_client_synack+0x3b0/0x640 net/netfilter/nf_synproxy_core.c:487
nft_synproxy_eval_v4+0x16c/0x278 net/netfilter/nft_synproxy.c:59
nft_synproxy_do_eval+0x2d8/0x4a4 net/netfilter/nft_synproxy.c:141
nft_synproxy_eval+0x34/0x48 net/netfilter/nft_synproxy.c:247
expr_call_ops_eval net/netfilter/nf_tables_core.c:237 [inline]
nft_do_chain+0x418/0x13c8 net/netfilter/nf_tables_core.c:285
nft_do_chain_inet+0x284/0x3d4 net/netfilter/nft_chain_filter.c:162
nf_hook_entry_hookfn include/linux/netfilter.h:165 [inline]
nf_hook_slow+0xc8/0x224 net/netfilter/core.c:619
nf_hook include/linux/netfilter.h:280 [inline]
NF_HOOK+0x1f0/0x398 include/linux/netfilter.h:323
ip_local_deliver+0x128/0x188 net/ipv4/ip_input.c:262
dst_input include/net/dst.h:480 [inline]
ip_rcv_finish+0x20c/0x230 net/ipv4/ip_input.c:492
NF_HOOK+0x308/0x398 include/linux/netfilter.h:325
ip_rcv+0x7c/0x9c net/ipv4/ip_input.c:612
__netif_receive_skb_one_core+0xc8/0x110 net/core/dev.c:6264
__netif_receive_skb+0x34/0x128 net/core/dev.c:6377
process_backlog+0x5f4/0x16f8 net/core/dev.c:6728
__napi_poll+0xb4/0x328 net/core/dev.c:7787
napi_poll net/core/dev.c:7850 [inline]
net_rx_action+0x5b0/0xc14 net/core/dev.c:8007
handle_softirqs+0x2ec/0xd98 kernel/softirq.c:645
__do_softirq+0x14/0x20 kernel/softirq.c:679
____do_softirq+0x14/0x20 arch/arm64/kernel/irq.c:78
call_on_irq_stack+0x30/0x48 arch/arm64/kernel/entry.S:885
do_softirq_own_stack+0x20/0x2c arch/arm64/kernel/irq.c:83
do_softirq+0x90/0xf8 kernel/softirq.c:546
__local_bh_enable_ip+0x240/0x35c kernel/softirq.c:473
local_bh_enable include/linux/bottom_half.h:33 [inline]
rcu_read_unlock_bh include/linux/rcupdate.h:923 [inline]
__dev_queue_xmit+0x14d4/0x2bc0 net/core/dev.c:4961
dev_queue_xmit include/linux/netdevice.h:3461 [inline]
tx+0xa0/0x1cc drivers/block/aoe/aoenet.c:62
kthread+0x178/0x340 drivers/block/aoe/aoecmd.c:1241
kthread+0x304/0x3d4 kernel/kthread.c:436
ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:838