R13: 00007fcf013e6038 R14: 00007fcf013e5fa0 R15: 00007ffebae2f0b8 BUG: workqueue lockup - pool cpus=0 node=0 flags=0x0 nice=0 stuck for 298s! Showing busy workqueues and worker pools: workqueue events: flags=0x100 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=14 refcnt=15 pending: 3*nsim_dev_hwstats_traffic_work, vmstat_shepherd, 3*psi_avgs_work, 4*ovs_dp_masks_rebalance, key_garbage_collector, 2*rht_deferred_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=16 refcnt=17 pending: 7*nsim_dev_hwstats_traffic_work, 4*psi_avgs_work, 4*ovs_dp_masks_rebalance, switchdev_deferred_process_work workqueue events_highpri: flags=0x110 pwq 3: cpus=0 node=0 flags=0x0 nice=-20 active=1 refcnt=2 pending: fill_page_cache_func pwq 7: cpus=1 node=0 flags=0x0 nice=-20 active=1 refcnt=2 in-flight: 25:fill_page_cache_func workqueue events_long: flags=0x100 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=12 refcnt=13 pending: 7*defense_work_handler, 2*br_fdb_cleanup, br_multicast_gc_work, 2*br_fdb_cleanup pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=4 refcnt=5 pending: defense_work_handler, br_multicast_gc_work, 2*br_fdb_cleanup workqueue events_unbound: flags=0x2 pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3 in-flight: 5992:toggle_allocation_gate pending: flush_memcg_stats_dwork workqueue events_unbound: flags=0x2 pwq 8: cpus=0-1 flags=0x4 nice=0 active=19 refcnt=20 in-flight: 6061:nsim_dev_trap_report_work ,37:nsim_dev_trap_report_work ,6305:nsim_dev_trap_report_work pending: 4*cfg80211_wiphy_work, 4*nsim_dev_trap_report_work, macvlan_process_broadcast, cfg80211_wiphy_work, 2*macvlan_process_broadcast, 4*idle_cull_fn pwq 8: cpus=0-1 flags=0x4 nice=0 active=17 refcnt=18 in-flight: 1304:cfg80211_wiphy_work cfg80211_wiphy_work ,6244:cfg80211_wiphy_work pending: 3*cfg80211_wiphy_work, nsim_dev_trap_report_work, 2*cfg80211_wiphy_work, 2*nsim_dev_trap_report_work, 3*macvlan_process_broadcast, crng_reseed, 2*macvlan_process_broadcast workqueue events_power_efficient: flags=0x180 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=8 refcnt=9 pending: neigh_managed_work, hash_net4_gc, neigh_periodic_work, do_cache_clean, 4*check_lifetime pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=10 refcnt=11 pending: neigh_managed_work, wg_ratelimiter_gc_entries, hash_net4_gc, neigh_periodic_work, reg_check_chans_work, gc_worker, 4*check_lifetime workqueue kvfree_rcu_reclaim: flags=0xa pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: kfree_rcu_monitor pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: kfree_rcu_monitor workqueue mm_percpu_wq: flags=0x8 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: vmstat_update pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: vmstat_update workqueue writeback: flags=0x4a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wb_workfn workqueue kblockd: flags=0x18 pwq 3: cpus=0 node=0 flags=0x0 nice=-20 active=1 refcnt=2 in-flight: 56:blk_mq_timeout_work workqueue usb_hub_wq: flags=0x104 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3 in-flight: 5965:hub_event hub_event workqueue ipv6_addrconf: flags=0x6000a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=11 pending: addrconf_verify_work inactive: 7*addrconf_verify_work workqueue krxrpcd: flags=0x2001a pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=11 pending: rxrpc_peer_keepalive_worker inactive: 7*rxrpc_peer_keepalive_worker workqueue bat_events: flags=0x6000a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=59 in-flight: 14299:batadv_mcast_mla_update inactive: 2*batadv_mcast_mla_update, 7*batadv_iv_send_outstanding_bat_ogm_packet, 4*batadv_mcast_mla_update, 11*batadv_iv_send_outstanding_bat_ogm_packet, 5*batadv_purge_orig, 3*batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_purge_orig, 7*batadv_tt_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, 2*batadv_dat_purge, batadv_bla_periodic_work, 2*batadv_dat_purge, 2*batadv_bla_periodic_work, batadv_dat_purge, 2*batadv_bla_periodic_work, batadv_dat_purge workqueue wg-crypt-wg0: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-crypt-wg1: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-crypt-wg2: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg0: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-kex-wg1: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3 pending: 2*wg_packet_handshake_send_worker pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_queued_expired_zero_key_material workqueue wg-kex-wg2: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=4 refcnt=5 pending: 2*wg_packet_handshake_send_worker, 2*wg_queued_expired_zero_key_material workqueue wg-crypt-wg0: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-crypt-wg1: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-crypt-wg2: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg0: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg0: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg1: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg1: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg2: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg2: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue hci6: flags=0x20012 pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4 pending: hci_conn_timeout workqueue hci4: flags=0x20012 pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4 pending: hci_conn_timeout workqueue wg-kex-wg0: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg0: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg1: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg1: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg2: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg2: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg0: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg0: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg1: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg1: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg2: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg2: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pool 2: cpus=0 node=0 flags=0x0 nice=0 hung=299s workers=14 idle: 12508 5922 10 6001 13771 789 6987 13847 9 5988 14154 6332 13806 pool 3: cpus=0 node=0 flags=0x0 nice=-20 hung=177s workers=3 idle: 11 15690 pool 7: cpus=1 node=0 flags=0x0 nice=-20 hung=0s workers=3 idle: 95 15983 pool 8: cpus=0-1 flags=0x4 nice=0 hung=299s workers=30 idle: 13 3016 6104 1087 3729 7751 72 6073 8773 5982 7632 3560 6054 5999 6988 13391 62 6027 12933 6830 6680 12312 50 Showing backtraces of running workers in stalled CPU-bound worker pools: pool 2: task:kworker/0:7 state:R running task stack:23496 pid:5965 tgid:5965 ppid:2 task_flags:0x4208060 flags:0x00080000 Workqueue: usb_hub_wq hub_event Call Trace: context_switch kernel/sched/core.c:5256 [inline] __schedule+0x1139/0x6150 kernel/sched/core.c:6863 __schedule_loop kernel/sched/core.c:6945 [inline] schedule+0xe7/0x3a0 kernel/sched/core.c:6960 schedule_timeout+0x123/0x290 kernel/time/sleep_timeout.c:99 do_wait_for_common kernel/sched/completion.c:100 [inline] __wait_for_common+0x2fc/0x4e0 kernel/sched/completion.c:121 usb_start_wait_urb+0x145/0x4e0 drivers/usb/core/message.c:64 usb_internal_control_msg drivers/usb/core/message.c:103 [inline] usb_control_msg+0x326/0x4a0 drivers/usb/core/message.c:154 get_bMaxPacketSize0.constprop.0+0xd0/0x1f0 drivers/usb/core/hub.c:4851 hub_port_init+0x690/0x3a90 drivers/usb/core/hub.c:5047 hub_port_connect drivers/usb/core/hub.c:5496 [inline] hub_port_connect_change drivers/usb/core/hub.c:5707 [inline] port_event drivers/usb/core/hub.c:5871 [inline] hub_event+0x2ee7/0x52f0 drivers/usb/core/hub.c:5953 process_one_work+0x9ba/0x1b20 kernel/workqueue.c:3257 process_scheduled_works kernel/workqueue.c:3340 [inline] worker_thread+0x6c8/0xf10 kernel/workqueue.c:3421 kthread+0x3c5/0x780 kernel/kthread.c:463 ret_from_fork+0x983/0xb10 arch/x86/kernel/process.c:158 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:246