BUG: workqueue lockup - pool cpus=0 node=0 flags=0x0 nice=0 stuck for 188s!
Showing busy workqueues and worker pools:
workqueue events: flags=0x0
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=13 refcnt=14
    in-flight: 10:nsim_fib_event_work nsim_fib_event_work
    pending: 3*nsim_dev_hwstats_traffic_work, 2*psi_avgs_work, 4*ovs_dp_masks_rebalance, vmstat_shepherd, ovs_dp_masks_rebalance
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=10 refcnt=11
    pending: 2*nsim_dev_hwstats_traffic_work, free_ipc, 4*psi_avgs_work, ovs_dp_masks_rebalance, delayed_vfree_work, bpf_prog_free_deferred
workqueue events_highpri: flags=0x10
  pwq 3: cpus=0 node=0 flags=0x0 nice=-20 active=1 refcnt=3
    pending: flush_backlog BAR(6026)
  pwq 7: cpus=1 node=0 flags=0x0 nice=-20 active=1 refcnt=2
    pending: fill_page_cache_func
workqueue events_long: flags=0x0
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: 3*defense_work_handler
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: 3*defense_work_handler
workqueue events_unbound: flags=0x2
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=16 refcnt=17
    in-flight: 6035:cfg80211_wiphy_work cfg80211_wiphy_work ,1168:cfg80211_wiphy_work cfg80211_wiphy_work ,2915:cfg80211_wiphy_work cfg80211_wiphy_work
    pending: 5*cfg80211_wiphy_work, 8*macvlan_process_broadcast
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=23 refcnt=24
    in-flight: 49:cfg80211_wiphy_work cfg80211_wiphy_work ,198:nsim_dev_trap_report_work ,36:linkwatch_event linkwatch_event ,12:cfg80211_wiphy_work ,4784:fsnotify_mark_destroy_workfn fsnotify_mark_destroy_workfn ,78:fsnotify_connector_destroy_workfn fsnotify_connector_destroy_workfn ,6047:toggle_allocation_gate
    pending: 4*nsim_dev_trap_report_work, cfg80211_wiphy_work, flush_memcg_stats_dwork, crng_reseed, 2*macvlan_process_broadcast
workqueue events_freezable: flags=0x4
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: update_balloon_stats_func
workqueue events_power_efficient: flags=0x80
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=8 refcnt=9
    pending: neigh_managed_work, crda_timeout_work, neigh_periodic_work, reg_check_chans_work, 4*check_lifetime
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=7 refcnt=8
    in-flight: 24:gc_worker
    pending: wg_ratelimiter_gc_entries, neigh_managed_work, neigh_periodic_work, do_cache_clean, 2*check_lifetime
workqueue rcu_gp: flags=0x8
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: srcu_invoke_callbacks
workqueue kvfree_rcu_reclaim: flags=0xa
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=3 MAYDAY
    pending: kfree_rcu_monitor
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=5 refcnt=7 MAYDAY
    in-flight: 13:kfree_rcu_monitor
    pending: kfree_rcu_monitor, 3*kfree_rcu_work
workqueue mm_percpu_wq: flags=0x8
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: vmstat_update
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: vmstat_update
workqueue writeback: flags=0x4a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=3 MAYDAY
    pending: wb_workfn
workqueue kblockd: flags=0x18
  pwq 7: cpus=1 node=0 flags=0x0 nice=-20 active=1 refcnt=2
    pending: blk_mq_timeout_work
workqueue mld: flags=0x40008
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=18
    pending: mld_dad_work
    inactive: 16*mld_dad_work
workqueue ipv6_addrconf: flags=0x6000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=10 MAYDAY
    pending: addrconf_verify_work
    inactive: 5*addrconf_verify_work
workqueue krxrpcd: flags=0x2001a
  pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=9
    pending: rxrpc_peer_keepalive_worker
    inactive: 5*rxrpc_peer_keepalive_worker
workqueue bat_events: flags=0x6000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=48
    in-flight: 6046:batadv_nc_worker
    inactive: 4*batadv_nc_worker, 5*batadv_mcast_mla_update, 9*batadv_iv_send_outstanding_bat_ogm_packet, batadv_purge_orig, batadv_iv_send_outstanding_bat_ogm_packet, 3*batadv_purge_orig, 4*batadv_iv_send_outstanding_bat_ogm_packet, batadv_purge_orig, batadv_iv_send_outstanding_bat_ogm_packet, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, 2*batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, 5*batadv_tt_purge
workqueue hci0: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
    pending: hci_conn_timeout
workqueue hci1: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
    pending: hci_conn_timeout
workqueue hci2: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
    pending: hci_conn_timeout
workqueue hci4: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
    pending: hci_conn_timeout
workqueue wg-kex-wg0: flags=0x24
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    in-flight: 2154:wg_packet_handshake_receive_worker
workqueue wg-crypt-wg0: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg0: flags=0x24
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg0: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg1: flags=0x24
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg1: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker, wg_packet_tx_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x24
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x24
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg1: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x24
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-crypt-wg0: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
    pending: 2*wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_tx_worker, wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_tx_worker, wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-crypt-wg0: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_decrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x24
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg1: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x24
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-crypt-wg0: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-crypt-wg1: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_decrypt_worker, wg_packet_tx_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x24
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg2: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
pool 2: cpus=0 node=0 flags=0x0 nice=0 hung=190s workers=8 idle: 978 5835 9 5848 5892 5894 5847
pool 6: cpus=1 node=0 flags=0x0 nice=0 hung=105s workers=7 idle: 5988 5930 43 5891 5890
pool 8: cpus=0-1 flags=0x6 nice=0 hung=104s workers=13 manager: 6048
Showing backtraces of running workers in stalled CPU-bound worker pools:
pool 2:
task:kworker/0:1     state:R  running task     stack:23760 pid:10    tgid:10    ppid:2      task_flags:0x4208060 flags:0x00004000
Workqueue: events nsim_fib_event_work
Call Trace:
 <TASK>
 context_switch kernel/sched/core.c:5396 [inline]
 __schedule+0x16f5/0x4d00 kernel/sched/core.c:6785
 __schedule_loop kernel/sched/core.c:6863 [inline]
 schedule+0x165/0x360 kernel/sched/core.c:6878
 schedule_timeout+0x12b/0x270 kernel/time/sleep_timeout.c:99
 schedule_timeout_uninterruptible kernel/time/sleep_timeout.c:158 [inline]
 msleep+0xa2/0x100 kernel/time/sleep_timeout.c:318
 nsim_fib6_rt_add drivers/net/netdevsim/fib.c:693 [inline]
 nsim_fib6_rt_insert drivers/net/netdevsim/fib.c:759 [inline]
 nsim_fib6_event drivers/net/netdevsim/fib.c:856 [inline]
 nsim_fib_event drivers/net/netdevsim/fib.c:889 [inline]
 nsim_fib_event_work+0x269f/0x3180 drivers/net/netdevsim/fib.c:1493
 process_one_work kernel/workqueue.c:3238 [inline]
 process_scheduled_works+0xae1/0x17b0 kernel/workqueue.c:3321
 worker_thread+0x8a0/0xda0 kernel/workqueue.c:3402
 kthread+0x70e/0x8a0 kernel/kthread.c:464
 ret_from_fork+0x3f9/0x770 arch/x86/kernel/process.c:148
 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245
 </TASK>