BUG: workqueue lockup - pool cpus=0 node=[ 1003.112330][    C0] BUG: workqueue lockup - pool cpus=0 node=0 flags=0x0 nice=0 stuck for 242s!
BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=0 stuck for 242s!
BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=-20 stuck for 241s!
BUG: workqueue lockup - pool cpus=0-1 flags=0x4 nice=0 stuck for 242s!
Showing busy workqueues and worker pools:
workqueue events: flags=0x100
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=18 refcnt=19
    pending: 4*nsim_dev_hwstats_traffic_work, vmstat_shepherd, 2*psi_avgs_work, 6*ovs_dp_masks_rebalance, 3*psi_avgs_work, destroy_super_work, delayed_vfree_work
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=3 refcnt=4
    in-flight: 6831:request_firmware_work_func
    pending: nsim_dev_hwstats_traffic_work, psi_avgs_work
workqueue events_highpri: flags=0x110
  pwq 7: cpus=1 node=0 flags=0x0 nice=-20 active=1 refcnt=2
    pending: fill_page_cache_func
workqueue events_long: flags=0x100
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=8 refcnt=9
    pending: 5*defense_work_handler, 3*br_fdb_cleanup
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=4 refcnt=5
    pending: defense_work_handler, 3*br_fdb_cleanup
workqueue events_unbound: flags=0x2
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
    pending: toggle_allocation_gate, flush_memcg_stats_dwork
workqueue events_unbound: flags=0x2
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=14 refcnt=15
    in-flight: 1403:nsim_dev_trap_report_work
    pending: nsim_dev_trap_report_work, 7*cfg80211_wiphy_work, 3*nsim_dev_trap_report_work, 2*macvlan_process_broadcast
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=5 refcnt=6
    pending: 3*cfg80211_wiphy_work, crng_reseed, idle_cull_fn
workqueue events_freezable: flags=0x104
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: update_balloon_stats_func
workqueue events_power_efficient: flags=0x180
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=9 refcnt=10
    pending: wg_ratelimiter_gc_entries, neigh_managed_work, gc_worker, neigh_periodic_work, do_cache_clean, 2*check_lifetime, 2*hash_ip6_gc
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=9 refcnt=10
    pending: neigh_managed_work, 2*hash_ip6_gc, neigh_periodic_work, 4*check_lifetime, hash_ip6_gc
workqueue kvfree_rcu_reclaim: flags=0xa
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
    pending: kfree_rcu_monitor
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
    pending: kfree_rcu_monitor
workqueue mm_percpu_wq: flags=0x8
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: vmstat_update
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: vmstat_update
workqueue writeback: flags=0x4a
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
    pending: wb_workfn
workqueue kblockd: flags=0x18
  pwq 7: cpus=1 node=0 flags=0x0 nice=-20 active=1 refcnt=2
    pending: blk_mq_timeout_work
workqueue usb_hub_wq: flags=0x104
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    in-flight: 6448:hub_event
workqueue ipv6_addrconf: flags=0x6000a
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=9
    pending: addrconf_verify_work
    inactive: 5*addrconf_verify_work
workqueue bat_events: flags=0x6000a
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=47
    pending: batadv_mcast_mla_update
    inactive: 4*batadv_mcast_mla_update, 2*batadv_iv_send_outstanding_bat_ogm_packet, 5*batadv_purge_orig, 17*batadv_iv_send_outstanding_bat_ogm_packet, 3*batadv_tt_purge, batadv_dat_purge, 3*batadv_bla_periodic_work, 2*batadv_dat_purge, 2*batadv_bla_periodic_work, 2*batadv_tt_purge, 2*batadv_dat_purge
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=4 refcnt=5
    pending: 2*wg_packet_handshake_send_worker, 2*wg_queued_expired_zero_key_material
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=3 refcnt=4
    pending: 2*wg_packet_handshake_send_worker, wg_queued_expired_zero_key_material
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
    pending: wg_queued_expired_zero_key_material
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
    pending: wg_packet_handshake_send_worker, wg_queued_expired_zero_key_material
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
    pending: wg_packet_handshake_send_worker, wg_queued_expired_zero_key_material
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=3 refcnt=4
    pending: 2*wg_packet_handshake_send_worker, wg_queued_expired_zero_key_material
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
    pending: wg_queued_expired_zero_key_material
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
    pending: wg_queued_expired_zero_key_material
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=3 refcnt=4
    pending: 2*wg_packet_handshake_send_worker, wg_queued_expired_zero_key_material
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
    pending: 2*wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
    pending: wg_packet_handshake_send_worker, wg_queued_expired_zero_key_material
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
    pending: wg_packet_handshake_send_worker, wg_queued_expired_zero_key_material
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
    pending: wg_packet_handshake_send_worker, wg_queued_expired_zero_key_material
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
    pending: wg_packet_handshake_send_worker, wg_queued_expired_zero_key_material
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
    pending: wg_queued_expired_zero_key_material
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=3 refcnt=4
    pending: 2*wg_packet_handshake_send_worker, wg_queued_expired_zero_key_material
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
    pending: 2*wg_packet_handshake_send_worker
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
    pending: wg_packet_handshake_send_worker, wg_queued_expired_zero_key_material
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
    pending: wg_packet_handshake_send_worker, wg_queued_expired_zero_key_material
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=3 refcnt=4
    pending: 2*wg_packet_handshake_send_worker, wg_queued_expired_zero_key_material
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
    pending: wg_queued_expired_zero_key_material
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
    pending: wg_packet_handshake_send_worker, wg_queued_expired_zero_key_material
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
    pending: wg_packet_handshake_send_worker, wg_queued_expired_zero_key_material
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
    pending: wg_packet_handshake_send_worker, wg_queued_expired_zero_key_material
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
    pending: wg_packet_handshake_send_worker, wg_queued_expired_zero_key_material
pool 6: cpus=1 node=0 flags=0x0 nice=0 hung=242s workers=4 idle: 5889 5968
pool 8: cpus=0-1 flags=0x4 nice=0 hung=242s workers=28 idle: 2128 8087 1121 12 1424 1125 10627 44 8958 1108 58 9214 10650 3553 7303 10536 152 3502 3469 1016 13 69 85 5933 6139 1398 6116
Showing backtraces of running workers in stalled CPU-bound worker pools:
pool 6:
task:kworker/1:10    state:R  running task     stack:21240 pid:6448  tgid:6448  ppid:2      task_flags:0x4208060 flags:0x00080000
Workqueue: usb_hub_wq hub_event
Call Trace:
 <TASK>
 context_switch kernel/sched/core.c:5256 [inline]
 __schedule+0x145f/0x5070 kernel/sched/core.c:6863
 __schedule_loop kernel/sched/core.c:6945 [inline]
 schedule_rtlock+0x30/0x70 kernel/sched/core.c:7024
 rtlock_slowlock_locked+0x3819/0x4010 kernel/locking/rtmutex.c:1868
 rtlock_slowlock kernel/locking/rtmutex.c:1895 [inline]
 rtlock_lock kernel/locking/spinlock_rt.c:43 [inline]
 __rt_spin_lock kernel/locking/spinlock_rt.c:49 [inline]
 rt_spin_lock+0x158/0x3e0 kernel/locking/spinlock_rt.c:57
 spin_lock include/linux/spinlock_rt.h:44 [inline]
 dummy_hub_control+0xaf/0x1a30 drivers/usb/gadget/udc/dummy_hcd.c:2122
 rh_call_control drivers/usb/core/hcd.c:656 [inline]
 rh_urb_enqueue drivers/usb/core/hcd.c:821 [inline]
 usb_hcd_submit_urb+0xdbe/0x1b60 drivers/usb/core/hcd.c:1542
 usb_start_wait_urb+0x115/0x4f0 drivers/usb/core/message.c:59
 usb_internal_control_msg drivers/usb/core/message.c:103 [inline]
 usb_control_msg+0x232/0x3e0 drivers/usb/core/message.c:154
 get_port_status drivers/usb/core/hub.c:607 [inline]
 hub_ext_port_status+0x116/0x820 drivers/usb/core/hub.c:624
 usb_hub_port_status drivers/usb/core/hub.c:674 [inline]
 port_event drivers/usb/core/hub.c:5761 [inline]
 hub_event+0x778/0x4f30 drivers/usb/core/hub.c:5953
 process_one_work kernel/workqueue.c:3257 [inline]
 process_scheduled_works+0xad1/0x1770 kernel/workqueue.c:3340
 worker_thread+0x8a0/0xda0 kernel/workqueue.c:3421
 kthread+0x711/0x8a0 kernel/kthread.c:463
 ret_from_fork+0x510/0xa50 arch/x86/kernel/process.c:158
 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:246
 </TASK>