BUG: workqueue lockup - pool cpus=0 node=0 flags=0x0 nice=0 stuck for 177s!
Showing busy workqueues and worker pools:
workqueue events: flags=0x0
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=9 refcnt=10
    pending: 3*nsim_dev_hwstats_traffic_work, psi_avgs_work, 2*ovs_dp_masks_rebalance, vmstat_shepherd, delayed_vfree_work, free_obj_work
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=12 refcnt=13
    in-flight: 5876:reg_todo
    pending: 2*nsim_dev_hwstats_traffic_work, 5*psi_avgs_work, 4*ovs_dp_masks_rebalance
workqueue events_long: flags=0x0
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=4 refcnt=5
    pending: 4*defense_work_handler
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: 2*defense_work_handler
workqueue events_unbound: flags=0x2
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=16 refcnt=17
    in-flight: 6040:cfg80211_wiphy_work ,36:fsnotify_mark_destroy_workfn fsnotify_mark_destroy_workfn ,1159:cfg80211_wiphy_work ,6105:cfg80211_wiphy_work cfg80211_wiphy_work ,13:fsnotify_connector_destroy_workfn fsnotify_connector_destroy_workfn ,1119:cfg80211_wiphy_work ,6115:cfg80211_wiphy_work cfg80211_wiphy_work
    pending: 4*cfg80211_wiphy_work, macvlan_process_broadcast, cfg80211_wiphy_work
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=15 refcnt=16
    in-flight: 1141:cfg80211_wiphy_work cfg80211_wiphy_work ,6106:nsim_dev_trap_report_work ,6113:nsim_dev_trap_report_work ,5018:cfg80211_wiphy_work cfg80211_wiphy_work ,6102:cfg80211_wiphy_work cfg80211_wiphy_work
    pending: flush_memcg_stats_dwork, 2*nsim_dev_trap_report_work, toggle_allocation_gate, nsim_dev_trap_report_work, macvlan_process_broadcast
workqueue events_freezable: flags=0x4
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: update_balloon_stats_func
workqueue events_power_efficient: flags=0x80
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=4 refcnt=5
    pending: wg_ratelimiter_gc_entries, 3*check_lifetime
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=6 refcnt=7
    in-flight: 5971:neigh_periodic_work ,5894:gc_worker ,5908:reg_check_chans_work
    pending: do_cache_clean, 2*neigh_managed_work
workqueue rcu_gp: flags=0x8
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    in-flight: 5934:process_srcu
    pending: process_srcu
workqueue kvfree_rcu_reclaim: flags=0xa
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=3 MAYDAY
    pending: kfree_rcu_monitor
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=3 refcnt=5 MAYDAY
    in-flight: 49:kfree_rcu_work ,12:kfree_rcu_work
    pending: kfree_rcu_monitor
workqueue mm_percpu_wq: flags=0x8
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: vmstat_update
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: vmstat_update
workqueue writeback: flags=0x4a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    in-flight: 64:wb_workfn
workqueue kblockd: flags=0x18
  pwq 7: cpus=1 node=0 flags=0x0 nice=-20 active=1 refcnt=2
    pending: blk_mq_timeout_work
workqueue mld: flags=0x40008
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=3
    pending: mld_dad_work
    inactive: mld_dad_work
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=13
    pending: mld_dad_work
    inactive: 11*mld_dad_work
workqueue ipv6_addrconf: flags=0x6000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=9
    in-flight: 6108:addrconf_verify_work
    inactive: 5*addrconf_verify_work
workqueue bat_events: flags=0x6000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=48
    in-flight: 6109:batadv_iv_send_outstanding_bat_ogm_packet
    inactive: 2*batadv_purge_orig, 6*batadv_iv_send_outstanding_bat_ogm_packet, batadv_nc_worker, 5*batadv_tt_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, 4*batadv_nc_worker, 5*batadv_mcast_mla_update, 5*batadv_iv_send_outstanding_bat_ogm_packet, 3*batadv_purge_orig, 3*batadv_iv_send_outstanding_bat_ogm_packet
workqueue ext4-rsv-conversion: flags=0xa
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    in-flight: 6111:ext4_end_io_rsv_work
workqueue hci0: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
    in-flight: 5154:hci_cmd_sync_work
workqueue hci1: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
    in-flight: 5838:hci_cmd_sync_work
workqueue hci2: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
    in-flight: 5837:hci_cmd_sync_work
workqueue hci3: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
    in-flight: 5853:hci_cmd_sync_work
workqueue hci4: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
    in-flight: 5848:hci_cmd_sync_work
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
    pending: 2*wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_tx_worker
workqueue wg-crypt-wg2: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg0: flags=0x24
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg0: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_tx_worker
workqueue wg-kex-wg0: flags=0x24
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_decrypt_worker, wg_packet_tx_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x24
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_decrypt_worker, wg_packet_tx_worker
workqueue wg-crypt-wg2: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x24
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg2: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg0: flags=0x24
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
    pending: 2*wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_decrypt_worker
workqueue wg-crypt-wg1: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_tx_worker
workqueue wg-kex-wg2: flags=0x24
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg0: flags=0x24
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
    pending: 2*wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_decrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x24
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
    pending: 2*wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_tx_worker
workqueue wg-kex-wg2: flags=0x24
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_tx_worker
pool 2: cpus=0 node=0 flags=0x0 nice=0 hung=179s workers=7 idle: 9 5922 10 5952 5901 977
pool 6: cpus=1 node=0 flags=0x0 nice=0 hung=14s workers=8 idle: 24 1213 43 5921
pool 8: cpus=0-1 flags=0x6 nice=0 hung=12s workers=19 manager: 6118
pool 9: cpus=0-1 node=0 flags=0x4 nice=-20 hung=0s workers=10 idle: 5851 51 5843 5845 5852
Showing backtraces of running workers in stalled CPU-bound worker pools:
pool 2:
task:kworker/0:5     state:R  running task     stack:24040 pid:5934  tgid:5934  ppid:2      task_flags:0x4208060 flags:0x00004000
Workqueue: rcu_gp process_srcu
Call Trace:
 <TASK>
 context_switch kernel/sched/core.c:5396 [inline]
 __schedule+0x16f5/0x4d00 kernel/sched/core.c:6785
 preempt_schedule_irq+0xb5/0x150 kernel/sched/core.c:7108
 irqentry_exit+0x6f/0x90 kernel/entry/common.c:307
 asm_sysvec_apic_timer_interrupt+0x1a/0x20 arch/x86/include/asm/idtentry.h:702
RIP: 0010:queue_delayed_work_on+0x201/0x280 kernel/workqueue.c:2566
Code: 4d 85 e4 75 16 e8 df 7f 35 00 eb 15 e8 d8 7f 35 00 e8 23 6c db 09 4d 85 e4 74 ea e8 c9 7f 35 00 fb 48 c7 44 24 40 0e 36 e0 45 <4b> c7 04 2f 00 00 00 00 66 43 c7 44 2f 09 00 00 43 c6 44 2f 0b 00
RSP: 0018:ffffc9000542f980 EFLAGS: 00000293
RAX: ffffffff818adc77 RBX: ffff888140eeda01 RCX: ffff88802955da00
RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000000
RBP: ffffc9000542fa70 R08: ffffffff8fa10df7 R09: 1ffffffff1f421be
R10: dffffc0000000000 R11: fffffbfff1f421bf R12: 0000000000000200
R13: dffffc0000000000 R14: 0000000000000000 R15: 1ffff92000a85f38
 process_one_work kernel/workqueue.c:3238 [inline]
 process_scheduled_works+0xae1/0x17b0 kernel/workqueue.c:3321
 worker_thread+0x8a0/0xda0 kernel/workqueue.c:3402
 kthread+0x70e/0x8a0 kernel/kthread.c:464
 ret_from_fork+0x3f9/0x770 arch/x86/kernel/process.c:148
 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245
 </TASK>
----------------
Code disassembly (best guess):
   0:	4d 85 e4             	test   %r12,%r12
   3:	75 16                	jne    0x1b
   5:	e8 df 7f 35 00       	call   0x357fe9
   a:	eb 15                	jmp    0x21
   c:	e8 d8 7f 35 00       	call   0x357fe9
  11:	e8 23 6c db 09       	call   0x9db6c39
  16:	4d 85 e4             	test   %r12,%r12
  19:	74 ea                	je     0x5
  1b:	e8 c9 7f 35 00       	call   0x357fe9
  20:	fb                   	sti
  21:	48 c7 44 24 40 0e 36 	movq   $0x45e0360e,0x40(%rsp)
  28:	e0 45
* 2a:	4b c7 04 2f 00 00 00 	movq   $0x0,(%r15,%r13,1) <-- trapping instruction
  31:	00
  32:	66 43 c7 44 2f 09 00 	movw   $0x0,0x9(%r15,%r13,1)
  39:	00
  3a:	43 c6 44 2f 0b 00    	movb   $0x0,0xb(%r15,%r13,1)