BUG: workqueue lockup - pool cpus=0 node=0 flags=0x0 nice=0 stuck for 58s! BUG: workqueue lockup - pool cpus=0 node=0 flags=0x0 nice=-20 stuck for 58s! BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=0 stuck for 58s! BUG: workqueue lockup - pool cpus=0-1 flags=0x4 nice=0 stuck for 58s! BUG: workqueue lockup - pool cpus=0-1 node=0 flags=0x4 nice=-20 stuck for 57s! Showing busy workqueues and worker pools: workqueue events: flags=0x0 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=24 refcnt=25 in-flight: 2056:request_firmware_work_func ,5961:nsim_fib_event_work nsim_fib_event_work ,8:nsim_fib_event_work nsim_fib_event_work ,7:nsim_fib_event_work nsim_fib_event_work pending: nsim_dev_trap_report_work, 2*nsim_dev_hwstats_traffic_work, nsim_dev_trap_report_work, nsim_dev_hwstats_traffic_work, vmstat_shepherd, 3*l2cap_info_timeout, 3*ovs_dp_masks_rebalance, kfree_rcu_monitor, 3*psi_avgs_work, ovs_dp_masks_rebalance pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=15 refcnt=16 in-flight: 6395:linkwatch_event ,1600:nsim_fib_event_work nsim_fib_event_work ,24:nsim_fib_event_work nsim_fib_event_work pending: 2*nsim_dev_hwstats_traffic_work, 3*nsim_dev_trap_report_work, 2*l2cap_info_timeout, 2*ovs_dp_masks_rebalance, kfree_rcu_monitor workqueue events_long: flags=0x0 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: defense_work_handler pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=5 refcnt=6 pending: 5*defense_work_handler workqueue events_unbound: flags=0x2 pwq 8: cpus=0-1 flags=0x4 nice=0 active=3 refcnt=4 pending: toggle_allocation_gate, cfg80211_wiphy_work, crng_reseed pwq 8: cpus=0-1 flags=0x4 nice=0 active=4 refcnt=5 pending: 3*cfg80211_wiphy_work, flush_memcg_stats_dwork workqueue events_power_efficient: flags=0x82 pwq 8: cpus=0-1 flags=0x4 nice=0 active=4 refcnt=5 pending: wg_ratelimiter_gc_entries, neigh_managed_work, neigh_periodic_work, gc_worker pwq 8: cpus=0-1 flags=0x4 nice=0 active=3 refcnt=4 pending: neigh_managed_work, neigh_periodic_work, do_cache_clean workqueue mm_percpu_wq: flags=0x8 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: vmstat_update pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: vmstat_update workqueue writeback: flags=0x4a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wb_workfn workqueue kblockd: flags=0x18 pwq 3: cpus=0 node=0 flags=0x0 nice=-20 active=1 refcnt=2 pending: blk_mq_timeout_work workqueue dm_bufio_cache: flags=0x8 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: work_fn workqueue mld: flags=0x40008 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=56 pending: mld_ifc_work inactive: 2*mld_ifc_work, mld_dad_work, 2*mld_ifc_work, mld_dad_work, 3*mld_ifc_work, mld_dad_work, 3*mld_ifc_work, mld_dad_work, 2*mld_ifc_work, mld_dad_work, 12*mld_ifc_work, mld_dad_work, 6*mld_ifc_work, 2*mld_dad_work, 3*mld_ifc_work, mld_dad_work, 5*mld_ifc_work, 4*mld_dad_work, 3*mld_ifc_work pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=40 pending: mld_ifc_work inactive: mld_ifc_work, 2*mld_dad_work, 4*mld_ifc_work, mld_dad_work, 2*mld_ifc_work, mld_dad_work, 13*mld_ifc_work, mld_dad_work, 2*mld_ifc_work, 2*mld_dad_work, 3*mld_ifc_work, 2*mld_dad_work, 2*mld_ifc_work, mld_dad_work, mld_ifc_work workqueue ipv6_addrconf: flags=0x6000a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=238 in-flight: 264:addrconf_dad_work inactive: 234*addrconf_dad_work workqueue krxrpcd: flags=0x2001a pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=9 pending: rxrpc_peer_keepalive_worker inactive: 5*rxrpc_peer_keepalive_worker workqueue bat_events: flags=0x6000a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=45 in-flight: 11:batadv_mcast_mla_update inactive: batadv_mcast_mla_update, 4*batadv_nc_worker, 2*batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_mcast_mla_update, batadv_nc_worker, 4*batadv_iv_send_outstanding_bat_ogm_packet, batadv_mcast_mla_update, 5*batadv_purge_orig, 6*batadv_iv_send_outstanding_bat_ogm_packet, 5*batadv_tt_purge, 2*batadv_dat_purge, 2*batadv_bla_periodic_work, 2*batadv_dat_purge, 2*batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work workqueue hci0: flags=0x20012 pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4 pending: hci_cmd_timeout workqueue hci1: flags=0x20012 pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4 pending: hci_cmd_timeout workqueue hci2: flags=0x20012 pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4 pending: hci_cmd_timeout workqueue hci3: flags=0x20012 pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4 pending: hci_cmd_timeout workqueue hci4: flags=0x20012 pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4 pending: hci_cmd_timeout workqueue wg-kex-wg0: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg0: flags=0x28 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-crypt-wg1: flags=0x28 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg2: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg2: flags=0x28 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg0: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg0: flags=0x28 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg1: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg1: flags=0x28 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg2: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg2: flags=0x28 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pool 2: cpus=0 node=0 flags=0x0 nice=0 hung=58s workers=8 idle: 6421 5947 5974 6379 pool 6: cpus=1 node=0 flags=0x0 nice=0 hung=58s workers=5 idle: 26 6396 pool 8: cpus=0-1 flags=0x4 nice=0 hung=58s workers=10 idle: 41 208 10 221 146 289 42 1791 Showing backtraces of running workers in stalled CPU-bound worker pools: pool 2: task:kworker/0:2 state:R running task stack:0 pid:2056 tgid:2056 ppid:2 flags:0x00000008 Workqueue: events request_firmware_work_func Call trace: __switch_to+0x314/0x560 arch/arm64/kernel/process.c:553 context_switch kernel/sched/core.c:5409 [inline] __schedule+0x14ac/0x24f4 kernel/sched/core.c:6746 __schedule_loop kernel/sched/core.c:6823 [inline] schedule+0xbc/0x238 kernel/sched/core.c:6838 schedule_timeout+0x1d8/0x348 kernel/time/timer.c:2582 do_wait_for_common+0x30c/0x468 kernel/sched/completion.c:95 __wait_for_common kernel/sched/completion.c:116 [inline] wait_for_common kernel/sched/completion.c:127 [inline] wait_for_completion_killable_timeout+0x50/0x74 kernel/sched/completion.c:288 __fw_state_wait_common drivers/base/firmware_loader/firmware.h:104 [inline] fw_sysfs_wait_timeout drivers/base/firmware_loader/fallback.c:44 [inline] fw_load_sysfs_fallback drivers/base/firmware_loader/fallback.c:110 [inline] fw_load_from_user_helper drivers/base/firmware_loader/fallback.c:162 [inline] firmware_fallback_sysfs+0x528/0x918 drivers/base/firmware_loader/fallback.c:238 _request_firmware+0xf70/0x1220 drivers/base/firmware_loader/main.c:914 request_firmware_work_func+0xfc/0x214 drivers/base/firmware_loader/main.c:1165 process_one_work+0x7b8/0x15d4 kernel/workqueue.c:3254 process_scheduled_works kernel/workqueue.c:3335 [inline] worker_thread+0x938/0xef4 kernel/workqueue.c:3416 kthread+0x288/0x310 kernel/kthread.c:388 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:860 pool 2: task:kworker/0:4 state:R running task stack:0 pid:5961 tgid:5961 ppid:2 flags:0x00000008 Workqueue: events nsim_fib_event_work Call trace: __switch_to+0x314/0x560 arch/arm64/kernel/process.c:553 context_switch kernel/sched/core.c:5409 [inline] __schedule+0x14ac/0x24f4 kernel/sched/core.c:6746 __schedule_loop kernel/sched/core.c:6823 [inline] schedule+0xbc/0x238 kernel/sched/core.c:6838 schedule_timeout+0x1d8/0x348 kernel/time/timer.c:2582 schedule_timeout_uninterruptible+0x74/0xac kernel/time/timer.c:2616 msleep+0x48/0x6c kernel/time/timer.c:2737 nsim_fib6_rt_add drivers/net/netdevsim/fib.c:693 [inline] nsim_fib6_rt_insert drivers/net/netdevsim/fib.c:759 [inline] nsim_fib6_event drivers/net/netdevsim/fib.c:856 [inline] nsim_fib_event drivers/net/netdevsim/fib.c:889 [inline] nsim_fib_event_work+0x247c/0x32bc drivers/net/netdevsim/fib.c:1492 process_one_work+0x7b8/0x15d4 kernel/workqueue.c:3254 process_scheduled_works kernel/workqueue.c:3335 [inline] worker_thread+0x970/0xef4 kernel/workqueue.c:3416 kthread+0x288/0x310 kernel/kthread.c:388 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:860 pool 2: task:kworker/0:1 state:R running task stack:0 pid:8 tgid:8 ppid:2 flags:0x00000008 Workqueue: events nsim_fib_event_work Call trace: __switch_to+0x314/0x560 arch/arm64/kernel/process.c:553 context_switch kernel/sched/core.c:5409 [inline] __schedule+0x14ac/0x24f4 kernel/sched/core.c:6746 __schedule_loop kernel/sched/core.c:6823 [inline] schedule+0xbc/0x238 kernel/sched/core.c:6838 schedule_timeout+0x1d8/0x348 kernel/time/timer.c:2582 schedule_timeout_uninterruptible+0x74/0xac kernel/time/timer.c:2616 msleep+0x48/0x6c kernel/time/timer.c:2737 nsim_fib6_rt_add drivers/net/netdevsim/fib.c:693 [inline] nsim_fib6_rt_insert drivers/net/netdevsim/fib.c:759 [inline] nsim_fib6_event drivers/net/netdevsim/fib.c:856 [inline] nsim_fib_event drivers/net/netdevsim/fib.c:889 [inline] nsim_fib_event_work+0x247c/0x32bc drivers/net/netdevsim/fib.c:1492 process_one_work+0x7b8/0x15d4 kernel/workqueue.c:3254 process_scheduled_works kernel/workqueue.c:3335 [inline] worker_thread+0x938/0xef4 kernel/workqueue.c:3416 kthread+0x288/0x310 kernel/kthread.c:388 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:860 pool 2: task:kworker/0:0 state:R running task stack:0 pid:7 tgid:7 ppid:2 flags:0x00000008 Workqueue: events nsim_fib_event_work Call trace: __switch_to+0x314/0x560 arch/arm64/kernel/process.c:553 context_switch kernel/sched/core.c:5409 [inline] __schedule+0x14ac/0x24f4 kernel/sched/core.c:6746 __schedule_loop kernel/sched/core.c:6823 [inline] schedule+0xbc/0x238 kernel/sched/core.c:6838 schedule_timeout+0x1d8/0x348 kernel/time/timer.c:2582 schedule_timeout_uninterruptible+0x74/0xac kernel/time/timer.c:2616 msleep+0x48/0x6c kernel/time/timer.c:2737 nsim_fib6_rt_add drivers/net/netdevsim/fib.c:693 [inline] nsim_fib6_rt_insert drivers/net/netdevsim/fib.c:759 [inline] nsim_fib6_event drivers/net/netdevsim/fib.c:856 [inline] nsim_fib_event drivers/net/netdevsim/fib.c:889 [inline] nsim_fib_event_work+0x247c/0x32bc drivers/net/netdevsim/fib.c:1492 process_one_work+0x7b8/0x15d4 kernel/workqueue.c:3254 process_scheduled_works kernel/workqueue.c:3335 [inline] worker_thread+0x970/0xef4 kernel/workqueue.c:3416 kthread+0x288/0x310 kernel/kthread.c:388 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:860 pool 6: task:kworker/1:3 state:R running task stack:0 pid:6395 tgid:6395 ppid:2 flags:0x00000008 Workqueue: events linkwatch_event Call trace: __switch_to+0x314/0x560 arch/arm64/kernel/process.c:553 context_switch kernel/sched/core.c:5409 [inline] __schedule+0x14ac/0x24f4 kernel/sched/core.c:6746 __schedule_loop kernel/sched/core.c:6823 [inline] schedule+0xbc/0x238 kernel/sched/core.c:6838 schedule_preempt_disabled+0x18/0x2c kernel/sched/core.c:6895 __mutex_lock_common+0xbd8/0x21a0 kernel/locking/mutex.c:684 __mutex_lock kernel/locking/mutex.c:752 [inline] mutex_lock_nested+0x2c/0x38 kernel/locking/mutex.c:804 rtnl_lock+0x20/0x2c net/core/rtnetlink.c:79 linkwatch_event+0x14/0x68 net/core/link_watch.c:276 process_one_work+0x7b8/0x15d4 kernel/workqueue.c:3254 process_scheduled_works kernel/workqueue.c:3335 [inline] worker_thread+0x938/0xef4 kernel/workqueue.c:3416 kthread+0x288/0x310 kernel/kthread.c:388 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:860 pool 6: task:kworker/1:2 state:R running task stack:0 pid:1600 tgid:1600 ppid:2 flags:0x00000008 Workqueue: events nsim_fib_event_work Call trace: __switch_to+0x314/0x560 arch/arm64/kernel/process.c:553 context_switch kernel/sched/core.c:5409 [inline] __schedule+0x14ac/0x24f4 kernel/sched/core.c:6746 __schedule_loop kernel/sched/core.c:6823 [inline] schedule+0xbc/0x238 kernel/sched/core.c:6838 schedule_timeout+0x1d8/0x348 kernel/time/timer.c:2582 schedule_timeout_uninterruptible+0x74/0xac kernel/time/timer.c:2616 msleep+0x48/0x6c kernel/time/timer.c:2737 nsim_fib6_rt_add drivers/net/netdevsim/fib.c:693 [inline] nsim_fib6_rt_insert drivers/net/netdevsim/fib.c:759 [inline] nsim_fib6_event drivers/net/netdevsim/fib.c:856 [inline] nsim_fib_event drivers/net/netdevsim/fib.c:889 [inline] nsim_fib_event_work+0x247c/0x32bc drivers/net/netdevsim/fib.c:1492 process_one_work+0x7b8/0x15d4 kernel/workqueue.c:3254 process_scheduled_works kernel/workqueue.c:3335 [inline] worker_thread+0x970/0xef4 kernel/workqueue.c:3416 kthread+0x288/0x310 kernel/kthread.c:388 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:860 pool 6: task:kworker/1:0 state:R running task stack:0 pid:24 tgid:24 ppid:2 flags:0x00000008 Workqueue: events nsim_fib_event_work Call trace: __switch_to+0x314/0x560 arch/arm64/kernel/process.c:553 context_switch kernel/sched/core.c:5409 [inline] __schedule+0x14ac/0x24f4 kernel/sched/core.c:6746 __schedule_loop kernel/sched/core.c:6823 [inline] schedule+0xbc/0x238 kernel/sched/core.c:6838 schedule_timeout+0x1d8/0x348 kernel/time/timer.c:2582 schedule_timeout_uninterruptible+0x74/0xac kernel/time/timer.c:2616 msleep+0x48/0x6c kernel/time/timer.c:2737 nsim_fib4_rt_add drivers/net/netdevsim/fib.c:369 [inline] nsim_fib4_rt_insert drivers/net/netdevsim/fib.c:432 [inline] nsim_fib4_event drivers/net/netdevsim/fib.c:464 [inline] nsim_fib_event drivers/net/netdevsim/fib.c:884 [inline] nsim_fib_event_work+0x18f0/0x32bc drivers/net/netdevsim/fib.c:1492 process_one_work+0x7b8/0x15d4 kernel/workqueue.c:3254 process_scheduled_works kernel/workqueue.c:3335 [inline] worker_thread+0x970/0xef4 kernel/workqueue.c:3416 kthread+0x288/0x310 kernel/kthread.c:388 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:860