BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=0 stuck for 46s! Showing busy workqueues and worker pools: workqueue events: flags=0x100 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=12 refcnt=13 in-flight: 2370:nsim_fib_event_work nsim_fib_event_work pending: vmstat_shepherd, 4*nsim_dev_hwstats_traffic_work, 2*psi_avgs_work, 3*ovs_dp_masks_rebalance pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=31 refcnt=32 in-flight: 24:nsim_fib_event_work ,6592:nsim_fib_event_work nsim_fib_event_work ,6618:nsim_fib_event_work ,26:nsim_fib_event_work nsim_fib_event_work ,1857:request_firmware_work_func pending: drm_fb_helper_damage_work, ovs_dp_masks_rebalance, 2*psi_avgs_work, rht_deferred_worker, nsim_dev_hwstats_traffic_work, psi_avgs_work, 5*ovs_dp_masks_rebalance, 2*l2cap_info_timeout, 2*ovs_dp_masks_rebalance, 2*l2cap_info_timeout, 2*ovs_dp_masks_rebalance, free_obj_work, rht_deferred_worker, 2*nsim_fib_event_work workqueue events_long: flags=0x100 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=5 refcnt=6 pending: 5*defense_work_handler pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=7 refcnt=8 pending: 7*defense_work_handler workqueue events_unbound: flags=0x2 pwq 8: cpus=0-1 flags=0x4 nice=0 active=3 refcnt=4 in-flight: 41:cfg80211_wiphy_work cfg80211_wiphy_work pwq 8: cpus=0-1 flags=0x4 nice=0 active=7 refcnt=8 in-flight: 5859:cfg80211_wiphy_work cfg80211_wiphy_work ,2274:cfg80211_wiphy_work cfg80211_wiphy_work ,15:cfg80211_wiphy_work cfg80211_wiphy_work pending: flush_memcg_stats_dwork, toggle_allocation_gate workqueue events_unbound: flags=0x2 pwq 8: cpus=0-1 flags=0x4 nice=0 active=3 refcnt=4 pending: 3*macvlan_process_broadcast pwq 8: cpus=0-1 flags=0x4 nice=0 active=8 refcnt=9 in-flight: 42:linkwatch_event pending: macvlan_process_broadcast, nsim_dev_trap_report_work, macvlan_process_broadcast, 4*nsim_dev_trap_report_work workqueue events_power_efficient: flags=0x182 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 in-flight: 4685:neigh_periodic_work pwq 8: cpus=0-1 flags=0x4 nice=0 active=5 refcnt=6 in-flight: 6771:gc_worker pending: do_cache_clean, wg_ratelimiter_gc_entries, fb_flashcursor, neigh_managed_work workqueue netns: flags=0x6000a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=4 in-flight: 680:cleanup_net workqueue mm_percpu_wq: flags=0x8 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: vmstat_update pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: vmstat_update workqueue writeback: flags=0x4a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 in-flight: 6770:wb_workfn workqueue mld: flags=0x40108 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=5 in-flight: 2362:mld_dad_work inactive: 3*mld_dad_work pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=181 pending: mld_dad_work inactive: 8*mld_ifc_work, 2*mld_dad_work, 9*mld_ifc_work, mld_dad_work, 8*mld_ifc_work, 6*mld_dad_work, 7*mld_ifc_work, mld_dad_work, 2*mld_ifc_work, mld_dad_work, 2*mld_ifc_work, 9*mld_dad_work, mld_ifc_work, mld_dad_work, 10*mld_ifc_work, 2*mld_dad_work, mld_ifc_work, 2*mld_dad_work, 4*mld_ifc_work, mld_dad_work, 6*mld_ifc_work, 3*mld_dad_work, mld_ifc_work, mld_dad_work, mld_ifc_work, mld_dad_work, 2*mld_ifc_work, 2*mld_dad_work, mld_ifc_work, mld_dad_work, 2*mld_ifc_work, mld_dad_work, 2*mld_ifc_work, mld_dad_work, mld_ifc_work, mld_dad_work, 3*mld_ifc_work, 3*mld_dad_work, mld_ifc_work, 2*mld_dad_work, 6*mld_ifc_work, 2*mld_dad_work, 3*mld_ifc_work, 3*mld_dad_work, mld_ifc_work, mld_dad_work, 4*mld_ifc_work, mld_dad_work, 2*mld_ifc_work, mld_dad_work, mld_ifc_work, mld_dad_work, 2*mld_ifc_work, mld_dad_work, 17*mld_ifc_work, 18*mld_dad_work, mld_ifc_work workqueue ipv6_addrconf: flags=0x6000a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=282 pending: addrconf_dad_work inactive: 278*addrconf_dad_work workqueue bat_events: flags=0x6000a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=49 in-flight: 4311(RESCUER):batadv_tt_purge inactive: 3*batadv_tt_purge, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, 6*batadv_mcast_mla_update, 6*batadv_iv_send_outstanding_bat_ogm_packet, 4*batadv_purge_orig, 4*batadv_iv_send_outstanding_bat_ogm_packet, batadv_purge_orig, 2*batadv_iv_send_outstanding_bat_ogm_packet, batadv_purge_orig, 4*batadv_iv_send_outstanding_bat_ogm_packet, batadv_tt_purge workqueue wg-kex-wg0: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg0: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_encrypt_worker, wg_packet_tx_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_encrypt_worker, wg_packet_tx_worker workqueue wg-kex-wg1: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 in-flight: 12:wg_packet_handshake_send_worker workqueue wg-crypt-wg1: flags=0x128 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_encrypt_worker, wg_packet_tx_worker workqueue wg-crypt-wg2: flags=0x128 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_encrypt_worker, wg_packet_tx_worker workqueue wg-crypt-wg0: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_encrypt_worker, wg_packet_tx_worker workqueue wg-kex-wg1: flags=0x124 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-crypt-wg1: flags=0x128 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=3 refcnt=4 pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker workqueue wg-kex-wg2: flags=0x124 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-crypt-wg2: flags=0x128 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=3 refcnt=4 pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker workqueue wg-crypt-wg0: flags=0x128 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_encrypt_worker, wg_packet_tx_worker workqueue wg-crypt-wg1: flags=0x128 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=3 refcnt=4 pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker workqueue wg-kex-wg2: flags=0x124 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-crypt-wg2: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3 in-flight: 6760:wg_packet_encrypt_worker wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=3 refcnt=4 pending: wg_packet_encrypt_worker, wg_packet_decrypt_worker, wg_packet_tx_worker workqueue wg-kex-wg0: flags=0x124 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-kex-wg0: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 in-flight: 6769:wg_packet_handshake_send_worker workqueue wg-kex-wg1: flags=0x124 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-kex-wg1: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 in-flight: 571:wg_packet_handshake_send_worker pool 2: cpus=0 node=0 flags=0x0 nice=0 hung=9s workers=6 idle: 9 6773 11 pool 6: cpus=1 node=0 flags=0x0 nice=0 hung=46s workers=8 idle: 6735 6730 6663 pool 8: cpus=0-1 flags=0x4 nice=0 hung=1s workers=13 idle: 6772 Showing backtraces of running workers in stalled CPU-bound worker pools: pool 6: task:kworker/1:0 state:R running task stack:0 pid:24 tgid:24 ppid:2 task_flags:0x4208060 flags:0x00000010 Workqueue: events nsim_fib_event_work Call trace: __switch_to+0x418/0x87c arch/arm64/kernel/process.c:741 (T) context_switch kernel/sched/core.c:5325 [inline] __schedule+0x13b0/0x2864 kernel/sched/core.c:6929 __schedule_loop kernel/sched/core.c:7011 [inline] schedule+0xb4/0x230 kernel/sched/core.c:7026 schedule_timeout+0x128/0x270 kernel/time/sleep_timeout.c:99 schedule_timeout_uninterruptible+0x7c/0xc0 kernel/time/sleep_timeout.c:158 msleep+0x3c/0x68 kernel/time/sleep_timeout.c:318 nsim_fib4_rt_add drivers/net/netdevsim/fib.c:369 [inline] nsim_fib4_rt_insert drivers/net/netdevsim/fib.c:432 [inline] nsim_fib4_event drivers/net/netdevsim/fib.c:464 [inline] nsim_fib_event+0x5d5c/0x6fc0 drivers/net/netdevsim/fib.c:884 nsim_fib_event_work+0x1c8/0x308 drivers/net/netdevsim/fib.c:1493 process_one_work+0x7e8/0x155c kernel/workqueue.c:3263 process_scheduled_works kernel/workqueue.c:3346 [inline] worker_thread+0x990/0xed8 kernel/workqueue.c:3427 kthread+0x5fc/0x75c kernel/kthread.c:463 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:844 pool 6: task:kworker/1:3 state:R running task stack:0 pid:6592 tgid:6592 ppid:2 task_flags:0x4208060 flags:0x00000010 Workqueue: events nsim_fib_event_work Call trace: __switch_to+0x418/0x87c arch/arm64/kernel/process.c:741 (T) context_switch kernel/sched/core.c:5325 [inline] __schedule+0x13b0/0x2864 kernel/sched/core.c:6929 preempt_schedule_common+0xe0/0x194 kernel/sched/core.c:7113 preempt_schedule+0x60/0x80 kernel/sched/core.c:7137 __raw_spin_unlock_irqrestore include/linux/spinlock_api_smp.h:152 [inline] _raw_spin_unlock_irqrestore+0x94/0x98 kernel/locking/spinlock.c:194 __debug_check_no_obj_freed lib/debugobjects.c:1108 [inline] debug_check_no_obj_freed+0x454/0x470 lib/debugobjects.c:1129 free_pages_prepare mm/page_alloc.c:1401 [inline] __free_frozen_pages+0x4b8/0xcac mm/page_alloc.c:2906 free_frozen_pages+0x14/0x20 mm/page_alloc.c:2944 __free_slab+0x108/0x1c0 mm/slub.c:3297 free_slab+0x24/0x120 mm/slub.c:3320 discard_slab mm/slub.c:3326 [inline] __put_partials+0x188/0x1f4 mm/slub.c:3872 put_cpu_partial+0x1d0/0x244 mm/slub.c:3947 __slab_free+0x198/0x240 mm/slub.c:5921 do_slab_free mm/slub.c:6570 [inline] ___cache_free+0x150/0x170 mm/slub.c:6696 qlink_free+0x48/0x94 mm/kasan/quarantine.c:163 qlist_free_all+0x44/0xb8 mm/kasan/quarantine.c:179 kasan_quarantine_reduce+0x118/0x124 mm/kasan/quarantine.c:286 __kasan_kmalloc+0x2c/0xb4 mm/kasan/common.c:408 kasan_kmalloc include/linux/kasan.h:262 [inline] __kmalloc_cache_noprof+0x3a4/0x65c mm/slub.c:5763 kmalloc_noprof include/linux/slab.h:957 [inline] kzalloc_noprof include/linux/slab.h:1094 [inline] nsim_fib6_rt_create drivers/net/netdevsim/fib.c:547 [inline] nsim_fib6_rt_insert drivers/net/netdevsim/fib.c:752 [inline] nsim_fib6_event drivers/net/netdevsim/fib.c:856 [inline] nsim_fib_event+0x9a4/0x6fc0 drivers/net/netdevsim/fib.c:889 nsim_fib_event_work+0x1c8/0x308 drivers/net/netdevsim/fib.c:1493 process_one_work+0x7e8/0x155c kernel/workqueue.c:3263 process_scheduled_works kernel/workqueue.c:3346 [inline] worker_thread+0x990/0xed8 kernel/workqueue.c:3427 kthread+0x5fc/0x75c kernel/kthread.c:463 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:844 pool 6: task:kworker/1:4 state:R running task stack:0 pid:6618 tgid:6618 ppid:2 task_flags:0x4208060 flags:0x00000010 Workqueue: events nsim_fib_event_work Call trace: __switch_to+0x418/0x87c arch/arm64/kernel/process.c:741 (T) context_switch kernel/sched/core.c:5325 [inline] __schedule+0x13b0/0x2864 kernel/sched/core.c:6929 __schedule_loop kernel/sched/core.c:7011 [inline] schedule+0xb4/0x230 kernel/sched/core.c:7026 schedule_timeout+0x128/0x270 kernel/time/sleep_timeout.c:99 schedule_timeout_uninterruptible+0x7c/0xc0 kernel/time/sleep_timeout.c:158 msleep+0x3c/0x68 kernel/time/sleep_timeout.c:318 nsim_fib4_rt_add drivers/net/netdevsim/fib.c:369 [inline] nsim_fib4_rt_insert drivers/net/netdevsim/fib.c:432 [inline] nsim_fib4_event drivers/net/netdevsim/fib.c:464 [inline] nsim_fib_event+0x5d5c/0x6fc0 drivers/net/netdevsim/fib.c:884 nsim_fib_event_work+0x1c8/0x308 drivers/net/netdevsim/fib.c:1493 process_one_work+0x7e8/0x155c kernel/workqueue.c:3263 process_scheduled_works kernel/workqueue.c:3346 [inline] worker_thread+0x990/0xed8 kernel/workqueue.c:3427 kthread+0x5fc/0x75c kernel/kthread.c:463 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:844 pool 6: task:kworker/1:1 state:R running task stack:0 pid:26 tgid:26 ppid:2 task_flags:0x4208060 flags:0x00000010 Workqueue: events nsim_fib_event_work Call trace: __switch_to+0x418/0x87c arch/arm64/kernel/process.c:741 (T) context_switch kernel/sched/core.c:5325 [inline] __schedule+0x13b0/0x2864 kernel/sched/core.c:6929 preempt_schedule_common+0xe0/0x194 kernel/sched/core.c:7113 preempt_schedule+0x60/0x80 kernel/sched/core.c:7137 __raw_spin_unlock_irqrestore include/linux/spinlock_api_smp.h:152 [inline] _raw_spin_unlock_irqrestore+0x94/0x98 kernel/locking/spinlock.c:194 __debug_check_no_obj_freed lib/debugobjects.c:1108 [inline] debug_check_no_obj_freed+0x454/0x470 lib/debugobjects.c:1129 free_pages_prepare mm/page_alloc.c:1401 [inline] __free_frozen_pages+0x4b8/0xcac mm/page_alloc.c:2906 free_frozen_pages+0x14/0x20 mm/page_alloc.c:2944 __free_slab+0x108/0x1c0 mm/slub.c:3297 free_slab+0x24/0x120 mm/slub.c:3320 discard_slab mm/slub.c:3326 [inline] __put_partials+0x188/0x1f4 mm/slub.c:3872 put_cpu_partial+0x1d0/0x244 mm/slub.c:3947 __slab_free+0x198/0x240 mm/slub.c:5921 do_slab_free mm/slub.c:6570 [inline] ___cache_free+0x150/0x170 mm/slub.c:6696 qlink_free+0x48/0x94 mm/kasan/quarantine.c:163 qlist_free_all+0x44/0xb8 mm/kasan/quarantine.c:179 kasan_quarantine_reduce+0x118/0x124 mm/kasan/quarantine.c:286 __kasan_kmalloc+0x2c/0xb4 mm/kasan/common.c:408 kasan_kmalloc include/linux/kasan.h:262 [inline] __kmalloc_cache_noprof+0x3a4/0x65c mm/slub.c:5763 kmalloc_noprof include/linux/slab.h:957 [inline] kzalloc_noprof include/linux/slab.h:1094 [inline] nsim_fib6_rt_nh_add+0x64/0x218 drivers/net/netdevsim/fib.c:500 nsim_fib6_rt_create drivers/net/netdevsim/fib.c:562 [inline] nsim_fib6_rt_insert drivers/net/netdevsim/fib.c:752 [inline] nsim_fib6_event drivers/net/netdevsim/fib.c:856 [inline] nsim_fib_event+0x1710/0x6fc0 drivers/net/netdevsim/fib.c:889 nsim_fib_event_work+0x1c8/0x308 drivers/net/netdevsim/fib.c:1493 process_one_work+0x7e8/0x155c kernel/workqueue.c:3263 process_scheduled_works kernel/workqueue.c:3346 [inline] worker_thread+0x990/0xed8 kernel/workqueue.c:3427 kthread+0x5fc/0x75c kernel/kthread.c:463 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:844 pool 6: task:kworker/1:2 state:R running task stack:0 pid:1857 tgid:1857 ppid:2 task_flags:0x4288060 flags:0x00000010 Workqueue: events request_firmware_work_func Call trace: __switch_to+0x418/0x87c arch/arm64/kernel/process.c:741 (T) context_switch kernel/sched/core.c:5325 [inline] __schedule+0x13b0/0x2864 kernel/sched/core.c:6929 preempt_schedule_common+0xe0/0x194 kernel/sched/core.c:7113 preempt_schedule+0x60/0x80 kernel/sched/core.c:7137 __raw_spin_unlock_irqrestore include/linux/spinlock_api_smp.h:152 [inline] _raw_spin_unlock_irqrestore+0x94/0x98 kernel/locking/spinlock.c:194 __debug_check_no_obj_freed lib/debugobjects.c:1108 [inline] debug_check_no_obj_freed+0x454/0x470 lib/debugobjects.c:1129 free_pages_prepare mm/page_alloc.c:1401 [inline] __free_frozen_pages+0x4b8/0xcac mm/page_alloc.c:2906 free_frozen_pages+0x14/0x20 mm/page_alloc.c:2944 __free_slab+0x108/0x1c0 mm/slub.c:3297 free_slab+0x24/0x120 mm/slub.c:3320 discard_slab mm/slub.c:3326 [inline] __put_partials+0x188/0x1f4 mm/slub.c:3872 put_cpu_partial+0x1d0/0x244 mm/slub.c:3947 __slab_free+0x198/0x240 mm/slub.c:5921 do_slab_free mm/slub.c:6570 [inline] ___cache_free+0x150/0x170 mm/slub.c:6696 qlink_free+0x48/0x94 mm/kasan/quarantine.c:163 qlist_free_all+0x44/0xb8 mm/kasan/quarantine.c:179 kasan_quarantine_reduce+0x118/0x124 mm/kasan/quarantine.c:286 __kasan_slab_alloc+0x2c/0x88 mm/kasan/common.c:352 kasan_slab_alloc include/linux/kasan.h:252 [inline] slab_post_alloc_hook mm/slub.c:4970 [inline] slab_alloc_node mm/slub.c:5280 [inline] __kmalloc_cache_noprof+0x338/0x65c mm/slub.c:5758 kmalloc_noprof include/linux/slab.h:957 [inline] kzalloc_noprof include/linux/slab.h:1094 [inline] kobject_uevent_env+0x258/0x884 lib/kobject_uevent.c:540 kobject_uevent+0x2c/0x3c lib/kobject_uevent.c:656 device_del+0x684/0x808 drivers/base/core.c:3896 fw_load_sysfs_fallback drivers/base/firmware_loader/fallback.c:124 [inline] fw_load_from_user_helper drivers/base/firmware_loader/fallback.c:162 [inline] firmware_fallback_sysfs+0x32c/0x8b4 drivers/base/firmware_loader/fallback.c:238 _request_firmware+0xfd8/0x12d8 drivers/base/firmware_loader/main.c:906 request_firmware_work_func+0xa8/0x19c drivers/base/firmware_loader/main.c:1159 process_one_work+0x7e8/0x155c kernel/workqueue.c:3263 process_scheduled_works kernel/workqueue.c:3346 [inline] worker_thread+0x958/0xed8 kernel/workqueue.c:3427 kthread+0x5fc/0x75c kernel/kthread.c:463 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:844