BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=0 stuck for 141s!
Showing busy workqueues and worker pools:
workqueue events: flags=0x0
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=33/256 refcnt=34
in-flight: 54:cfg80211_wiphy_work, 23:cfg80211_wiphy_work
pending: 2*cfg80211_wiphy_work, 3*ovs_dp_masks_rebalance, nsim_dev_trap_report_work, 3*nsim_dev_hwstats_traffic_work, nsim_dev_trap_report_work, nsim_dev_hwstats_traffic_work, nsim_dev_trap_report_work, 2*cfg80211_wiphy_work, 4*psi_avgs_work, 3*bpf_prog_free_deferred, psi_avgs_work, cfg80211_wiphy_work, ovs_dp_masks_rebalance, delayed_vfree_work, 2*cfg80211_wiphy_work, kfree_rcu_monitor, kfree_rcu_work, cfg80211_wiphy_work, ima_keys_handler
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=21/256 refcnt=22
in-flight: 5146:cfg80211_wiphy_work cfg80211_wiphy_work, 922:cfg80211_wiphy_work cfg80211_wiphy_work, 9:cfg80211_wiphy_work cfg80211_wiphy_work, 5086:kfree_rcu_work, 5147:cfg80211_wiphy_work cfg80211_wiphy_work
pending: 3*nsim_dev_trap_report_work, 2*nsim_dev_hwstats_traffic_work, kfree_rcu_work, free_obj_work, vmstat_shepherd, 3*ovs_dp_masks_rebalance, kfree_rcu_monitor
workqueue events_long: flags=0x0
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=2/256 refcnt=3
pending: 2*defense_work_handler
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=5/256 refcnt=6
pending: 5*defense_work_handler
workqueue events_unbound: flags=0x2
pwq 4: cpus=0-1 flags=0x4 nice=0 active=2/512 refcnt=3
in-flight: 61:fsnotify_mark_destroy_workfn fsnotify_mark_destroy_workfn, 2421:fsnotify_connector_destroy_workfn fsnotify_connector_destroy_workfn
pwq 4: cpus=0-1 flags=0x4 nice=0 active=3/512 refcnt=4
pending: toggle_allocation_gate
workqueue events_freezable: flags=0x4
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: update_balloon_stats_func
workqueue events_power_efficient: flags=0x80
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=4/256 refcnt=5
pending: neigh_periodic_work, do_cache_clean, neigh_managed_work, reg_check_chans_work
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=4/256 refcnt=5
in-flight: 5149:neigh_periodic_work
pending: wg_ratelimiter_gc_entries, gc_worker, neigh_managed_work
workqueue rcu_gp: flags=0x8
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=2/256 refcnt=3
pending: wait_rcu_exp_gp, process_srcu
workqueue mm_percpu_wq: flags=0x8
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=2/256 refcnt=4
pending: vmstat_update, lru_add_drain_per_cpu BAR(35)
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: vmstat_update
workqueue kblockd: flags=0x18
pwq 3: cpus=1 node=0 flags=0x0 nice=-20 active=1/256 refcnt=2
pending: blk_mq_requeue_work
pwq 1: cpus=0 node=0 flags=0x0 nice=-20 active=1/256 refcnt=2
pending: blk_mq_timeout_work
workqueue dm_bufio_cache: flags=0x8
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: work_fn
workqueue ipv6_addrconf: flags=0x40008
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/1 refcnt=4
pending: addrconf_verify_work
inactive: 2*addrconf_verify_work
workqueue bat_events: flags=0xe000a
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/1 refcnt=57
in-flight: 2398:batadv_tt_purge
inactive: 3*batadv_tt_purge, 2*batadv_dat_purge, 2*batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, 6*batadv_nc_worker, 11*batadv_iv_send_outstanding_bat_ogm_packet, 6*batadv_mcast_mla_update, batadv_iv_send_outstanding_bat_ogm_packet, batadv_purge_orig, 2*batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_purge_orig, 4*batadv_iv_send_outstanding_bat_ogm_packet, 3*batadv_purge_orig, 2*batadv_tt_purge, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work
workqueue hci0: flags=0xa0012
pwq 5: cpus=0-1 node=0 flags=0x4 nice=-20 active=1/1 refcnt=4
in-flight: 5089:hci_cmd_sync_work
workqueue hci1: flags=0xa0012
pwq 5: cpus=0-1 node=0 flags=0x4 nice=-20 active=1/1 refcnt=4
in-flight: 5103:hci_cmd_sync_work
workqueue hci2: flags=0xa0012
pwq 5: cpus=0-1 node=0 flags=0x4 nice=-20 active=1/1 refcnt=4
in-flight: 5094:hci_cmd_sync_work
workqueue hci3: flags=0xa0012
pwq 5: cpus=0-1 node=0 flags=0x4 nice=-20 active=1/1 refcnt=4
in-flight: 5104:hci_cmd_sync_work
workqueue hci4: flags=0xa0012
pwq 5: cpus=0-1 node=0 flags=0x4 nice=-20 active=1/1 refcnt=4
in-flight: 5093:hci_cmd_sync_work
workqueue hci5: flags=0xa0012
pwq 5: cpus=0-1 node=0 flags=0x4 nice=-20 active=1/1 refcnt=4
in-flight: 5084:hci_cmd_sync_work
workqueue wg-kex-wg0: flags=0x24
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg0: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=2/256 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x24
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg2: flags=0x24
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg2: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg0: flags=0x24
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=2/256 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x24
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
pending: wg_packet_encrypt_worker, wg_packet_decrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg2: flags=0x24
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_receive_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg2: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
pending: wg_packet_tx_worker, wg_packet_encrypt_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg0: flags=0x24
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=2/256 refcnt=3
pending: wg_packet_tx_worker, wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x24
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_receive_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg2: flags=0x24
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg2: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker, wg_packet_tx_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_decrypt_worker
workqueue wg-kex-wg0: flags=0x24
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=2/256 refcnt=3
pending: wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x24
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_receive_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg0: flags=0x24
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=2/256 refcnt=3
pending: wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg0: flags=0x24
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_receive_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg0: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=2/256 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg2: flags=0x24
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_receive_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg2: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
pending: wg_packet_tx_worker, wg_packet_encrypt_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg1: flags=0x24
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg1: flags=0x24
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_receive_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg1: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
pending: wg_packet_encrypt_worker, wg_packet_decrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg2: flags=0x24
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_receive_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg2: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
pending: wg_packet_tx_worker, wg_packet_encrypt_worker, wg_packet_decrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_decrypt_worker
workqueue wg-kex-wg2: flags=0x24
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg2: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
pending: wg_packet_tx_worker, wg_packet_encrypt_worker, wg_packet_decrypt_worker
pool 0: cpus=0 node=0 flags=0x0 nice=0 hung=3s workers=11 idle: 5154 5188 8 5083 5145
pool 2: cpus=1 node=0 flags=0x0 nice=0 hung=143s workers=9 idle: 5202 27 5097 5201 5148 4790 5144
pool 4: cpus=0-1 flags=0x4 nice=0 hung=2s workers=8 idle: 1250 11 2418 59 2428
pool 5: cpus=0-1 node=0 flags=0x4 nice=-20 hung=0s workers=10 idle: 4451 50 5096 5100
Showing backtraces of running workers in stalled CPU-bound worker pools:
pool 2:
task:kworker/1:2 state:R running task stack:25408 pid:54 ppid:2 flags:0x00004000
Workqueue: events cfg80211_wiphy_work
Call Trace:
context_switch kernel/sched/core.c:5382 [inline]
__schedule+0xee1/0x5a10 kernel/sched/core.c:6695
preempt_schedule_irq+0x52/0x90 kernel/sched/core.c:7007
irqentry_exit+0x35/0x80 kernel/entry/common.c:432
asm_sysvec_apic_timer_interrupt+0x1a/0x20 arch/x86/include/asm/idtentry.h:645
RIP: 0010:clear_page_erms+0xb/0x10 arch/x86/lib/clear_page_64.S:50
Code: 48 89 47 20 48 89 47 28 48 89 47 30 48 89 47 38 48 8d 7f 40 75 d9 90 c3 0f 1f 80 00 00 00 00 f3 0f 1e fa b9 00 10 00 00 31 c0 aa c3 66 90 f3 0f 1e fa 48 83 f9 40 73 36 83 f9 08 73 0f 85 c9
RSP: 0018:ffffc90000bf7318 EFLAGS: 00010246
RAX: 0000000000000000 RBX: 0000000000000003 RCX: 0000000000000f40
RDX: ffffea000081bb80 RSI: ffff888000000000 RDI: ffff8880206ee0c0
RBP: ffffea000081ba00 R08: 0000160000000000 R09: 0000000000000000
R10: ffffed10040dd000 R11: dffffc0000000000 R12: 0000000000000000
R13: ffffea000081bc00 R14: 0000000000000001 R15: 0000000000000000
clear_page arch/x86/include/asm/page_64.h:53 [inline]
clear_highpage_kasan_tagged include/linux/highmem.h:248 [inline]
kernel_init_pages mm/page_alloc.c:1071 [inline]
post_alloc_hook+0x1a3/0x340 mm/page_alloc.c:1534
prep_new_page mm/page_alloc.c:1543 [inline]
get_page_from_freelist+0xee0/0x2f20 mm/page_alloc.c:3170
__alloc_pages+0x1d0/0x4a0 mm/page_alloc.c:4426
alloc_pages+0x1a9/0x270 mm/mempolicy.c:2298
alloc_slab_page mm/slub.c:1870 [inline]
allocate_slab+0x251/0x380 mm/slub.c:2017
new_slab mm/slub.c:2070 [inline]
___slab_alloc+0x8c7/0x1580 mm/slub.c:3223
__slab_alloc.constprop.0+0x56/0xa0 mm/slub.c:3322
__slab_alloc_node mm/slub.c:3375 [inline]
slab_alloc_node mm/slub.c:3468 [inline]
__kmem_cache_alloc_node+0x131/0x340 mm/slub.c:3517
__do_kmalloc_node mm/slab_common.c:1022 [inline]
__kmalloc+0x4f/0x100 mm/slab_common.c:1036
kmalloc include/linux/slab.h:603 [inline]
kzalloc include/linux/slab.h:720 [inline]
ieee802_11_parse_elems_full+0xc7/0x1380 net/mac80211/util.c:1602
ieee802_11_parse_elems_crc net/mac80211/ieee80211_i.h:2326 [inline]
ieee802_11_parse_elems net/mac80211/ieee80211_i.h:2333 [inline]
ieee80211_rx_mgmt_probe_beacon net/mac80211/ibss.c:1609 [inline]
ieee80211_ibss_rx_queued_mgmt+0xc65/0x2ed0 net/mac80211/ibss.c:1642
ieee80211_iface_process_skb net/mac80211/iface.c:1604 [inline]
ieee80211_iface_work+0xa1f/0xd40 net/mac80211/iface.c:1658
cfg80211_wiphy_work+0x24e/0x330 net/wireless/core.c:435
process_one_work+0x884/0x15c0 kernel/workqueue.c:2630
process_scheduled_works kernel/workqueue.c:2703 [inline]
worker_thread+0x8b9/0x1290 kernel/workqueue.c:2784
kthread+0x33c/0x440 kernel/kthread.c:388
ret_from_fork+0x45/0x80 arch/x86/kernel/process.c:147
ret_from_fork_asm+0x11/0x20 arch/x86/entry/entry_64.S:304
pool 2:
task:kworker/1:0 state:R running task stack:24912 pid:23 ppid:2 flags:0x00004000
Workqueue: events cfg80211_wiphy_work
Call Trace:
context_switch kernel/sched/core.c:5382 [inline]
__schedule+0xee1/0x5a10 kernel/sched/core.c:6695
preempt_schedule_irq+0x52/0x90 kernel/sched/core.c:7007
irqentry_exit+0x35/0x80 kernel/entry/common.c:432
asm_sysvec_apic_timer_interrupt+0x1a/0x20 arch/x86/include/asm/idtentry.h:645
RIP: 0010:kcov_remote_start+0x3f8/0x6c0 kernel/kcov.c:923
Code: 8e e8 2c 78 cc 02 e9 4a ff ff ff e8 e2 a9 06 00 e9 95 fc ff ff e8 68 a9 06 00 9c 58 f6 c4 02 0f 85 99 02 00 00 fb 48 83 c4 18 <5b> 5d 41 5c 41 5d 41 5e 41 5f c3 be 02 00 00 00 4c 89 e7 e8 80 8b
RSP: 0018:ffffc900001d7c08 EFLAGS: 00000282
RAX: 0000000000000006 RBX: ffff8880b992c528 RCX: 1ffffffff2343c3a
RDX: 0000000000000000 RSI: ffffffff8a8cab20 RDI: ffffffff8ae938e0
RBP: ffff888017a6d940 R08: 0000000000000001 R09: fffffbfff233e3dd
R10: ffffffff919f1eef R11: ffffffff8187cfa7 R12: 0000000000000000
R13: 0000000000000200 R14: 0000000000000000 R15: ffff888017a6d940
kcov_remote_start_common include/linux/kcov.h:48 [inline]
ieee80211_iface_work+0x39d/0xd40 net/mac80211/iface.c:1653
cfg80211_wiphy_work+0x24e/0x330 net/wireless/core.c:435
process_one_work+0x884/0x15c0 kernel/workqueue.c:2630
process_scheduled_works kernel/workqueue.c:2703 [inline]
worker_thread+0x8b9/0x1290 kernel/workqueue.c:2784
kthread+0x33c/0x440 kernel/kthread.c:388
ret_from_fork+0x45/0x80 arch/x86/kernel/process.c:147
ret_from_fork_asm+0x11/0x20 arch/x86/entry/entry_64.S:304
----------------
Code disassembly (best guess):
0: 48 89 47 20 mov %rax,0x20(%rdi)
4: 48 89 47 28 mov %rax,0x28(%rdi)
8: 48 89 47 30 mov %rax,0x30(%rdi)
c: 48 89 47 38 mov %rax,0x38(%rdi)
10: 48 8d 7f 40 lea 0x40(%rdi),%rdi
14: 75 d9 jne 0xffffffef
16: 90 nop
17: c3 ret
18: 0f 1f 80 00 00 00 00 nopl 0x0(%rax)
1f: f3 0f 1e fa endbr64
23: b9 00 10 00 00 mov $0x1000,%ecx
28: 31 c0 xor %eax,%eax
* 2a: f3 aa rep stos %al,%es:(%rdi) <-- trapping instruction
2c: c3 ret
2d: 66 90 xchg %ax,%ax
2f: f3 0f 1e fa endbr64
33: 48 83 f9 40 cmp $0x40,%rcx
37: 73 36 jae 0x6f
39: 83 f9 08 cmp $0x8,%ecx
3c: 73 0f jae 0x4d
3e: 85 c9 test %ecx,%ecx