BUG: workqueue lockup - pool cpus=0 node=0 flags=0x0 nice=0 stuck for 224s!
Showing busy workqueues and worker pools:
workqueue events: flags=0x0
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
pending: psi_avgs_work, vmstat_shepherd, kfree_rcu_monitor
workqueue events_unbound: flags=0x2
pwq 4: cpus=0-1 flags=0x5 nice=0 active=1/512 refcnt=3
in-flight: 329:toggle_allocation_gate
workqueue events_power_efficient: flags=0x80
pwq 2: cpus=1 node=0 flags=0x1 nice=0 active=2/256 refcnt=3
pending: gc_worker, check_lifetime
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=4/256 refcnt=5
in-flight: 15:wg_ratelimiter_gc_entries
pending: neigh_periodic_work, neigh_periodic_work, reg_check_chans_work
workqueue mm_percpu_wq: flags=0x8
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: vmstat_update
workqueue netns: flags=0xe000a
pwq 4: cpus=0-1 flags=0x5 nice=0 active=1/1 refcnt=4
in-flight: 9:cleanup_net
delayed: cleanup_net
workqueue writeback: flags=0x4a
pwq 4: cpus=0-1 flags=0x5 nice=0 active=1/256 refcnt=3
in-flight: 424:wb_workfn
workqueue dm_bufio_cache: flags=0x8
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: work_fn
workqueue ipv6_addrconf: flags=0x40008
pwq 2: cpus=1 node=0 flags=0x1 nice=0 active=1/1 refcnt=63
in-flight: 441:addrconf_dad_work
delayed: addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work
, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/1 refcnt=77
in-flight: 5:addrconf_dad_work
delayed: addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work
, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_verify_work
workqueue wg-crypt-wg0: flags=0x28
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=2/256 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=1 node=0 flags=0x1 nice=0 active=1/256 refcnt=2
in-flight: 360:wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=2/256 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-crypt-wg2: flags=0x28
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=2/256 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=1 node=0 flags=0x1 nice=0 active=1/256 refcnt=4 MAYDAY
in-flight: 375(RESCUER):wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
pending: wg_packet_encrypt_worker, wg_packet_decrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x6
pwq 4: cpus=0-1 flags=0x5 nice=0 active=1/256 refcnt=3
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=1 node=0 flags=0x1 nice=0 active=3/256 refcnt=6 MAYDAY
in-flight: 376(RESCUER):wg_packet_decrypt_worker, 362:wg_packet_encrypt_worker
pending: wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-crypt-wg2: flags=0x28
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=2/256 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg0: flags=0x6
pwq 4: cpus=0-1 flags=0x5 nice=0 active=1/256 refcnt=3
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=1 node=0 flags=0x1 nice=0 active=1/256 refcnt=2
in-flight: 20:wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=1 node=0 flags=0x1 nice=0 active=1/256 refcnt=2
in-flight: 52:wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg2: flags=0x24
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg2: flags=0x6
pwq 4: cpus=0-1 flags=0x5 nice=0 active=1/256 refcnt=3
in-flight: 418:wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x28
pwq 2: cpus=1 node=0 flags=0x1 nice=0 active=1/256 refcnt=4 MAYDAY
in-flight: 381(RESCUER):wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=2/256 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg0: flags=0x6
pwq 4: cpus=0-1 flags=0x5 nice=0 active=1/256 refcnt=3
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=1 node=0 flags=0x1 nice=0 active=1/256 refcnt=4 MAYDAY
in-flight: 382(RESCUER):wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
pending: wg_packet_encrypt_worker, wg_packet_decrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x24
pwq 2: cpus=1 node=0 flags=0x1 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg1: flags=0x6
pwq 4: cpus=0-1 flags=0x5 nice=0 active=1/256 refcnt=3
in-flight: 7:wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=1 node=0 flags=0x1 nice=0 active=3/256 refcnt=6 MAYDAY
in-flight: 361:wg_packet_encrypt_worker, 383(RESCUER):wg_packet_decrypt_worker
pending: wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg2: flags=0x6
pwq 4: cpus=0-1 flags=0x5 nice=0 active=2/256 refcnt=4
in-flight: 112:wg_packet_handshake_send_worker
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x28
pwq 2: cpus=1 node=0 flags=0x1 nice=0 active=2/256 refcnt=5 MAYDAY
in-flight: 384(RESCUER):wg_packet_encrypt_worker
pending: wg_packet_tx_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=2/256 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg0: flags=0x6
pwq 4: cpus=0-1 flags=0x5 nice=0 active=1/256 refcnt=3
in-flight: 442:wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=1 node=0 flags=0x1 nice=0 active=1/256 refcnt=4 MAYDAY
in-flight: 385(RESCUER):wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
pending: wg_packet_encrypt_worker, wg_packet_decrypt_worker, wg_packet_tx_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=1 node=0 flags=0x1 nice=0 active=3/256 refcnt=6 MAYDAY
in-flight: 386(RESCUER):wg_packet_encrypt_worker wg_packet_decrypt_worker
pending: wg_packet_tx_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg2: flags=0x6
pwq 4: cpus=0-1 flags=0x5 nice=0 active=1/256 refcnt=3
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x28
pwq 2: cpus=1 node=0 flags=0x1 nice=0 active=1/256 refcnt=2
in-flight: 25:wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=2/256 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg0: flags=0x6
pwq 4: cpus=0-1 flags=0x5 nice=0 active=1/256 refcnt=3
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=1 node=0 flags=0x1 nice=0 active=2/256 refcnt=5 MAYDAY
in-flight: 388(RESCUER):wg_packet_encrypt_worker
pending: wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
pending: wg_packet_encrypt_worker, wg_packet_decrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x6
pwq 4: cpus=0-1 flags=0x5 nice=0 active=1/256 refcnt=3
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=1 node=0 flags=0x1 nice=0 active=3/256 refcnt=6 MAYDAY
in-flight: 389(RESCUER):wg_packet_encrypt_worker
pending: wg_packet_decrypt_worker, wg_packet_tx_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
pending: wg_packet_tx_worker, wg_packet_encrypt_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg2: flags=0x24
pwq 2: cpus=1 node=0 flags=0x1 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_receive_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg2: flags=0x28
pwq 2: cpus=1 node=0 flags=0x1 nice=0 active=1/256 refcnt=4 MAYDAY
in-flight: 390(RESCUER):wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=2/256 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
pool 0: cpus=0 node=0 flags=0x0 nice=0 hung=225s workers=22 idle: 397 400 398 401 402 403 404 406 405 399 391 392 393 394 395 396 409 306 408 407
pool 2: cpus=1 node=0 flags=0x1 nice=0 hung=93s workers=8 manager: 425
pool 4: cpus=0-1 flags=0x5 nice=0 hung=108s workers=8 manager: 358
rcu: INFO: rcu_preempt detected stalls on CPUs/tasks:
rcu: Tasks blocked on level-0 rcu_node (CPUs 0-1): P20/1:b..l
(detected by 0, t=10002 jiffies, g=1513, q=85)
task:kworker/1:0 state:R running task stack: 0 pid: 20 ppid: 2 flags:0x00004000
Workqueue: wg-crypt-wg0 wg_packet_encrypt_worker
Call Trace:
context_switch kernel/sched/core.c:4048 [inline]
__schedule+0xc03/0x13e0 kernel/sched/core.c:4806
preempt_schedule_irq+0xaa/0x120 kernel/sched/core.c:5067
irqentry_exit_cond_resched kernel/entry/common.c:365 [inline]
irqentry_exit+0x56/0x60 kernel/entry/common.c:395
sysvec_apic_timer_interrupt+0xcb/0xe0 arch/x86/kernel/apic/apic.c:1106
asm_sysvec_apic_timer_interrupt+0x12/0x20 arch/x86/include/asm/idtentry.h:635
RIP: 0010:__skb_flow_dissect+0x3df/0x5670 net/core/flow_dissector.c:992
Code: 00 48 89 df e8 92 b9 02 fe e9 e1 03 00 00 48 81 c3 40 0d 00 00 48 89 d8 48 c1 e8 03 48 b9 00 00 00 00 00 fc ff df 80 3c 08 00 <74> 08 48 89 df e8 67 b9 02 fe 4c 8b 2b 4d 85 ed 0f 84 82 06 00 00
RSP: 0018:ffffc90000157120 EFLAGS: 00000246
RAX: 1ffff11021ddab48 RBX: ffff88810eed5a40 RCX: dffffc0000000000
RDX: 0000000000000000 RSI: ffff888116141000 RDI: ffffffff86c3c562
RBP: ffffc90000157630 R08: 0000000000000000 R09: 0000000000000000
R10: dffffc0000000000 R11: fffff5200002aeea R12: 1ffffffff0d878ac
R13: 0000000000000000 R14: dffffc0000000000 R15: 0000000000000000
skb_flow_dissect_flow_keys include/linux/skbuff.h:1354 [inline]
___skb_get_hash+0x58/0x680 net/core/flow_dissector.c:1558
__skb_get_hash+0xd2/0x270 net/core/flow_dissector.c:1623
skb_get_hash include/linux/skbuff.h:1396 [inline]
encrypt_packet drivers/net/wireguard/send.c:173 [inline]
wg_packet_encrypt_worker+0x3ab/0x1470 drivers/net/wireguard/send.c:297
process_one_work+0x6fd/0xbc0 kernel/workqueue.c:2301
worker_thread+0xa8e/0x13c0 kernel/workqueue.c:2447
kthread+0x324/0x3b0 kernel/kthread.c:313
ret_from_fork+0x1f/0x30 arch/x86/entry/entry_64.S:298
rcu: rcu_preempt kthread starved for 9827 jiffies! g1513 f0x0 RCU_GP_WAIT_FQS(5) ->state=0x0 ->cpu=0
rcu: Unless rcu_preempt kthread gets sufficient CPU time, OOM is now expected behavior.
rcu: RCU grace-period kthread stack dump:
task:rcu_preempt state:R running task stack: 0 pid: 13 ppid: 2 flags:0x00004000
Call Trace:
context_switch kernel/sched/core.c:4048 [inline]
__schedule+0xc03/0x13e0 kernel/sched/core.c:4806
preempt_schedule_irq+0xaa/0x120 kernel/sched/core.c:5067
irqentry_exit_cond_resched kernel/entry/common.c:365 [inline]
irqentry_exit+0x56/0x60 kernel/entry/common.c:395
sysvec_apic_timer_interrupt+0xcb/0xe0 arch/x86/kernel/apic/apic.c:1106
asm_sysvec_apic_timer_interrupt+0x12/0x20 arch/x86/include/asm/idtentry.h:635
RIP: 0010:del_timer_sync+0x90/0x1a0 kernel/time/timer.c:1409
Code: 01 00 00 45 8b 36 44 89 f6 81 e6 00 00 20 00 31 ff e8 24 7e 0f 00 41 81 e6 00 00 20 00 0f 84 db 00 00 00 e8 62 7a 0f 00 eb 05 <e8> 5b 7a 0f 00 48 c7 45 c8 00 00 00 00 48 8d 75 c8 48 89 df e8 77
RSP: 0018:ffffc900000d7c38 EFLAGS: 00000246
RAX: ffffffff815539fe RBX: ffffc900000d7ca0 RCX: 0000000000000000
RDX: ffff8881002962c0 RSI: 0000000000000000 RDI: 0000000000000000
RBP: ffffc900000d7c70 R08: ffff8881002962c7 R09: 1ffff11020052c58
R10: dffffc0000000000 R11: ffffed1020052c59 R12: dffffc0000000000
R13: dffffc0000000000 R14: 0000000000000000 R15: ffffc900000d7ca0
schedule_timeout+0x161/0x330 kernel/time/timer.c:1914
rcu_gp_fqs_loop kernel/rcu/tree.c:1943 [inline]
rcu_gp_kthread+0x109a/0x2810 kernel/rcu/tree.c:2117
kthread+0x324/0x3b0 kernel/kthread.c:313
ret_from_fork+0x1f/0x30 arch/x86/entry/entry_64.S:298
sched: RT throttling activated
----------------
Code disassembly (best guess), 1 bytes skipped:
0: 48 89 df mov %rbx,%rdi
3: e8 92 b9 02 fe call 0xfe02b99a
8: e9 e1 03 00 00 jmp 0x3ee
d: 48 81 c3 40 0d 00 00 add $0xd40,%rbx
14: 48 89 d8 mov %rbx,%rax
17: 48 c1 e8 03 shr $0x3,%rax
1b: 48 b9 00 00 00 00 00 movabs $0xdffffc0000000000,%rcx
22: fc ff df
25: 80 3c 08 00 cmpb $0x0,(%rax,%rcx,1)
* 29: 74 08 je 0x33 <-- trapping instruction
2b: 48 89 df mov %rbx,%rdi
2e: e8 67 b9 02 fe call 0xfe02b99a
33: 4c 8b 2b mov (%rbx),%r13
36: 4d 85 ed test %r13,%r13
39: 0f 84 82 06 00 00 je 0x6c1