BUG: workqueue lockup - pool cpus=0 node=0 flags=0x0 nice=0 stuck for 199s!
Showing busy workqueues and worker pools:
workqueue events: flags=0x100
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=12 refcnt=13
    in-flight: 977:request_firmware_work_func
    pending: psi_avgs_work, 4*nsim_dev_hwstats_traffic_work, vmstat_shepherd, psi_avgs_work, 2*bpf_prog_free_deferred, ovs_dp_masks_rebalance, delayed_vfree_work
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=10 refcnt=11
    in-flight: 5907:xfrm_state_gc_task
    pending: 5*ovs_dp_masks_rebalance, free_obj_work, nsim_dev_hwstats_traffic_work, psi_avgs_work, nsim_fib_event_work
workqueue events_long: flags=0x100
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: 3*defense_work_handler
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=3 refcnt=4
    pending: 3*defense_work_handler
workqueue events_unbound: flags=0x2
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
    pending: flush_memcg_stats_dwork, toggle_allocation_gate
workqueue events_unbound: flags=0x2
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
    in-flight: 3574:cfg80211_wiphy_work cfg80211_wiphy_work
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=11 refcnt=12
    in-flight: 3831:cfg80211_wiphy_work cfg80211_wiphy_work ,3896:nsim_dev_trap_report_work ,5979:nsim_dev_trap_report_work ,5975:nsim_dev_trap_report_work ,36:cfg80211_wiphy_work cfg80211_wiphy_work ,139:nsim_dev_trap_report_work ,1006:nsim_dev_trap_report_work ,1929:cfg80211_wiphy_work cfg80211_wiphy_work
workqueue events_freezable: flags=0x104
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: update_balloon_stats_func
workqueue events_power_efficient: flags=0x180
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=10 refcnt=11
    pending: hash_netnet6_gc, 2*neigh_managed_work, 2*neigh_periodic_work, do_cache_clean, gc_worker, reg_check_chans_work, 2*check_lifetime
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=5 refcnt=6
    pending: wg_ratelimiter_gc_entries, 4*check_lifetime
workqueue kvfree_rcu_reclaim: flags=0xa
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=4 MAYDAY
    pending: 2*kfree_rcu_monitor
workqueue mm_percpu_wq: flags=0x8
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: vmstat_update
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=3 MAYDAY
    pending: vmstat_update
workqueue writeback: flags=0x4a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    in-flight: 2990:wb_workfn
workqueue mld: flags=0x40108
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=3
    in-flight: 980:mld_dad_work
    inactive: mld_ifc_work
workqueue ipv6_addrconf: flags=0x6000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=406
    in-flight: 3184(RESCUER):addrconf_dad_work
    inactive: 395*addrconf_dad_work, 6*addrconf_verify_work
workqueue bat_events: flags=0x6000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=43
    in-flight: 5980:batadv_dat_purge
    inactive: batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, 3*batadv_tt_purge, 2*batadv_iv_send_outstanding_bat_ogm_packet, batadv_dat_purge, batadv_bla_periodic_work, 2*batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_mcast_mla_update, 2*batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_purge_orig, 2*batadv_iv_send_outstanding_bat_ogm_packet, 3*batadv_mcast_mla_update, 3*batadv_purge_orig, 7*batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_tt_purge, batadv_dat_purge, batadv_bla_periodic_work
workqueue udp_tunnel_nic: flags=0x20002
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=4
    in-flight: 50:udp_tunnel_nic_device_sync_work
workqueue hci0: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
    in-flight: 5851:hci_cmd_sync_work
workqueue hci1: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
    in-flight: 5846:hci_cmd_sync_work
workqueue hci2: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
    in-flight: 5845:hci_cmd_sync_work
workqueue hci3: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
    in-flight: 5152:hci_cmd_sync_work
workqueue hci4: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
    in-flight: 5848:hci_cmd_sync_work
workqueue wg-kex-wg0: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=2 refcnt=3
    in-flight: 48:wg_packet_handshake_receive_worker
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    in-flight: 13:wg_packet_handshake_send_worker
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    in-flight: 5978:wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg1: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=2 refcnt=3
    in-flight: 5883:wg_packet_handshake_receive_worker
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
    in-flight: 4522:wg_packet_handshake_send_worker wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg0: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=2 refcnt=3
    in-flight: 5951:wg_packet_handshake_receive_worker
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg2: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=2 refcnt=3
    in-flight: 5950:wg_packet_handshake_receive_worker
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
    in-flight: 12:wg_packet_handshake_send_worker wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    in-flight: 72:wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_decrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg0: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    in-flight: 5977:wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg2: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=2 refcnt=3
    in-flight: 24:wg_packet_handshake_receive_worker
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_tx_worker, wg_packet_decrypt_worker, wg_packet_encrypt_worker
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    in-flight: 1131:wg_packet_handshake_send_worker
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    in-flight: 5981:wg_packet_handshake_send_worker
pool 2: cpus=0 node=0 flags=0x0 nice=0 hung=200s workers=9 idle: 10 5844 5929 5970 5921 9 5894
pool 6: cpus=1 node=0 flags=0x2 nice=0 hung=0s workers=7 manager: 793
pool 8: cpus=0-1 flags=0x6 nice=0 hung=0s workers=22 manager: 5976 idle: 3422
pool 9: cpus=0-1 node=0 flags=0x4 nice=-20 hung=0s workers=11 idle: 52 5849 5839 5836 5837 5850
Showing backtraces of running workers in stalled CPU-bound worker pools:
pool 2:
task:kworker/0:3     state:R  running task     stack:23416 pid:980   tgid:980   ppid:2      task_flags:0x4208060 flags:0x00080000
Workqueue: mld mld_dad_work
Call Trace:
 <TASK>
 context_switch kernel/sched/core.c:5256 [inline]
 __schedule+0x14bc/0x5000 kernel/sched/core.c:6863
 preempt_schedule_irq+0xb5/0x150 kernel/sched/core.c:7190
 irqentry_exit+0x5d8/0x660 kernel/entry/common.c:216
 asm_sysvec_apic_timer_interrupt+0x1a/0x20 arch/x86/include/asm/idtentry.h:697
RIP: 0010:__kernel_text_address+0x0/0x40 kernel/extable.c:78
Code: c1 03 38 c1 7c c1 48 c7 c7 80 e1 81 8f e8 58 2e 9b 00 eb b3 66 0f 1f 44 00 00 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 <0f> 1f 40 d6 53 48 89 fb e8 43 00 00 00 85 c0 0f 95 c0 48 81 fb 00
RSP: 0018:ffffc900039d70e0 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffffc900039d7150 RCX: 0af332aa60ad9700
RDX: 0000000000000004 RSI: ffffffff8d979ceb RDI: ffffffff82286187
RBP: ffffc900039d7190 R08: ffffc900039d7548 R09: 0000000000000000
R10: ffffc900039d7158 R11: fffff5200073ae2d R12: ffff888025729e80
R13: 1ffff9200073ae4c R14: dffffc0000000000 R15: 1ffff9200073ae2a
 unwind_get_return_address+0x4d/0x90 arch/x86/kernel/unwind_orc.c:385
 arch_stack_walk+0xfc/0x150 arch/x86/kernel/stacktrace.c:26
 stack_trace_save+0x9c/0xe0 kernel/stacktrace.c:122
 save_stack+0xf5/0x1f0 mm/page_owner.c:165
 __reset_page_owner+0x71/0x1f0 mm/page_owner.c:320
 reset_page_owner include/linux/page_owner.h:25 [inline]
 free_pages_prepare mm/page_alloc.c:1395 [inline]
 __free_frozen_pages+0xbc8/0xd30 mm/page_alloc.c:2943
 discard_slab mm/slub.c:3346 [inline]
 __put_partials+0x146/0x170 mm/slub.c:3886
 put_cpu_partial+0x1f2/0x2d0 mm/slub.c:3961
 __slab_free+0x288/0x2a0 mm/slub.c:5952
 qlink_free mm/kasan/quarantine.c:163 [inline]
 qlist_free_all+0x97/0x100 mm/kasan/quarantine.c:179
 kasan_quarantine_reduce+0x148/0x160 mm/kasan/quarantine.c:286
 __kasan_slab_alloc+0x22/0x80 mm/kasan/common.c:349
 kasan_slab_alloc include/linux/kasan.h:252 [inline]
 slab_post_alloc_hook mm/slub.c:4953 [inline]
 slab_alloc_node mm/slub.c:5263 [inline]
 __do_kmalloc_node mm/slub.c:5656 [inline]
 __kmalloc_node_track_caller_noprof+0x526/0x820 mm/slub.c:5764
 kmalloc_reserve+0x136/0x290 net/core/skbuff.c:608
 __alloc_skb+0x27e/0x430 net/core/skbuff.c:690
 alloc_skb include/linux/skbuff.h:1383 [inline]
 mld_newpack+0x13c/0xc40 net/ipv6/mcast.c:1775
 add_grhead+0x5a/0x2a0 net/ipv6/mcast.c:1886
 add_grec+0x1452/0x1740 net/ipv6/mcast.c:2025
 mld_send_initial_cr+0x288/0x550 net/ipv6/mcast.c:2268
 mld_dad_work+0x46/0x490 net/ipv6/mcast.c:2294
 process_one_work kernel/workqueue.c:3257 [inline]
 process_scheduled_works+0xad1/0x1770 kernel/workqueue.c:3340
 worker_thread+0x8a0/0xda0 kernel/workqueue.c:3421
 kthread+0x711/0x8a0 kernel/kthread.c:463
 ret_from_fork+0x599/0xb30 arch/x86/kernel/process.c:158
 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:246
 </TASK>
pool 2:
task:kworker/0:2     state:R  running task     stack:24600 pid:977   tgid:977   ppid:2      task_flags:0x4288060 flags:0x00080000
Workqueue: events request_firmware_work_func
Call Trace:
 <TASK>
 context_switch kernel/sched/core.c:5256 [inline]
 __schedule+0x14bc/0x5000 kernel/sched/core.c:6863
 preempt_schedule_irq+0xb5/0x150 kernel/sched/core.c:7190
 irqentry_exit+0x5d8/0x660 kernel/entry/common.c:216
 asm_sysvec_apic_timer_interrupt+0x1a/0x20 arch/x86/include/asm/idtentry.h:697
RIP: 0010:unwind_next_frame+0x484/0x23d0 arch/x86/kernel/unwind_orc.c:520
Code: d2 84 c0 48 0f 45 d1 48 89 15 98 be 10 11 48 85 d2 0f 84 40 02 00 00 4c 8d 62 04 4c 8d 6a 05 4c 89 e0 48 c1 e8 03 0f b6 04 28 <84> c0 0f 85 6d 18 00 00 4c 89 e8 48 c1 e8 03 0f b6 04 28 84 c0 0f
RSP: 0018:ffffc90003a2efd8 EFLAGS: 00000a06
RAX: 0000000000000000 RBX: ffffffff8fa26628 RCX: ffffffff8fa26630
RDX: ffffffff90220936 RSI: ffffffff90220936 RDI: ffffffff8bc08680
RBP: dffffc0000000000 R08: 000000000000000c R09: ffffffff8df41a20
R10: ffffc90003a2f0f8 R11: ffffffff81ada100 R12: ffffffff9022093a
R13: ffffffff9022093b R14: ffffc90003a2f0a8 R15: ffffffff8fa2662c
 arch_stack_walk+0x11c/0x150 arch/x86/kernel/stacktrace.c:25
 stack_trace_save+0x9c/0xe0 kernel/stacktrace.c:122
 save_stack+0xf5/0x1f0 mm/page_owner.c:165
 __reset_page_owner+0x71/0x1f0 mm/page_owner.c:320
 reset_page_owner include/linux/page_owner.h:25 [inline]
 free_pages_prepare mm/page_alloc.c:1395 [inline]
 __free_frozen_pages+0xbc8/0xd30 mm/page_alloc.c:2943
 discard_slab mm/slub.c:3346 [inline]
 __put_partials+0x146/0x170 mm/slub.c:3886
 put_cpu_partial+0x1f2/0x2d0 mm/slub.c:3961
 __slab_free+0x288/0x2a0 mm/slub.c:5952
 qlink_free mm/kasan/quarantine.c:163 [inline]
 qlist_free_all+0x97/0x100 mm/kasan/quarantine.c:179
 kasan_quarantine_reduce+0x148/0x160 mm/kasan/quarantine.c:286
 __kasan_slab_alloc+0x22/0x80 mm/kasan/common.c:349
 kasan_slab_alloc include/linux/kasan.h:252 [inline]
 slab_post_alloc_hook mm/slub.c:4953 [inline]
 slab_alloc_node mm/slub.c:5263 [inline]
 __do_kmalloc_node mm/slub.c:5656 [inline]
 __kmalloc_noprof+0x3cf/0x800 mm/slub.c:5669
 kmalloc_noprof include/linux/slab.h:961 [inline]
 kzalloc_noprof include/linux/slab.h:1094 [inline]
 kobject_get_path+0xc5/0x2f0 lib/kobject.c:161
 kobject_uevent_env+0x2a1/0x9f0 lib/kobject_uevent.c:545
 device_del+0x73a/0x8e0 drivers/base/core.c:3896
 fw_load_sysfs_fallback drivers/base/firmware_loader/fallback.c:124 [inline]
 fw_load_from_user_helper drivers/base/firmware_loader/fallback.c:162 [inline]
 firmware_fallback_sysfs+0x38d/0xa40 drivers/base/firmware_loader/fallback.c:238
 _request_firmware+0xfa4/0x1680 drivers/base/firmware_loader/main.c:898
 request_firmware_work_func+0xaf/0x1c0 drivers/base/firmware_loader/main.c:1150
 process_one_work kernel/workqueue.c:3257 [inline]
 process_scheduled_works+0xad1/0x1770 kernel/workqueue.c:3340
 worker_thread+0x8a0/0xda0 kernel/workqueue.c:3421
 kthread+0x711/0x8a0 kernel/kthread.c:463
 ret_from_fork+0x599/0xb30 arch/x86/kernel/process.c:158
 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:246
 </TASK>
----------------
Code disassembly (best guess):
   0:	c1 03 38             	roll   $0x38,(%rbx)
   3:	c1 7c c1 48 c7       	sarl   $0xc7,0x48(%rcx,%rax,8)
   8:	c7 80 e1 81 8f e8 58 	movl   $0x9b2e58,-0x17707e1f(%rax)
   f:	2e 9b 00
  12:	eb b3                	jmp    0xffffffc7
  14:	66 0f 1f 44 00 00    	nopw   0x0(%rax,%rax,1)
  1a:	90                   	nop
  1b:	90                   	nop
  1c:	90                   	nop
  1d:	90                   	nop
  1e:	90                   	nop
  1f:	90                   	nop
  20:	90                   	nop
  21:	90                   	nop
  22:	90                   	nop
  23:	90                   	nop
  24:	90                   	nop
  25:	90                   	nop
  26:	90                   	nop
  27:	90                   	nop
  28:	90                   	nop
  29:	90                   	nop
* 2a:	0f 1f 40 d6          	nopl   -0x2a(%rax) <-- trapping instruction
  2e:	53                   	push   %rbx
  2f:	48 89 fb             	mov    %rdi,%rbx
  32:	e8 43 00 00 00       	call   0x7a
  37:	85 c0                	test   %eax,%eax
  39:	0f 95 c0             	setne  %al
  3c:	48                   	rex.W
  3d:	81                   	.byte 0x81
  3e:	fb                   	sti