EEVDF scheduling fail, picking leftmost ====================================================== WARNING: possible circular locking dependency detected 6.6.0-rc4-next-20231004-syzkaller #0 Not tainted ------------------------------------------------------ kworker/0:2/780 is trying to acquire lock: ffffffff8cab8520 (console_owner){-...}-{0:0}, at: console_lock_spinning_enable kernel/printk/printk.c:1855 [inline] ffffffff8cab8520 (console_owner){-...}-{0:0}, at: console_emit_next_record kernel/printk/printk.c:2888 [inline] ffffffff8cab8520 (console_owner){-...}-{0:0}, at: console_flush_all+0x49a/0xd50 kernel/printk/printk.c:2960 but task is already holding lock: ffff8880b983c718 (&rq->__lock){-.-.}-{2:2}, at: raw_spin_rq_lock_nested+0x29/0x130 kernel/sched/core.c:558 which lock already depends on the new lock. the existing dependency chain (in reverse order) is: -> #4 (&rq->__lock){-.-.}-{2:2}: _raw_spin_lock_nested+0x31/0x40 kernel/locking/spinlock.c:378 raw_spin_rq_lock_nested+0x29/0x130 kernel/sched/core.c:558 raw_spin_rq_lock kernel/sched/sched.h:1357 [inline] rq_lock kernel/sched/sched.h:1671 [inline] task_fork_fair+0x70/0x240 kernel/sched/fair.c:12399 sched_cgroup_fork+0x3cf/0x510 kernel/sched/core.c:4799 copy_process+0x4580/0x74b0 kernel/fork.c:2609 kernel_clone+0xfd/0x920 kernel/fork.c:2907 user_mode_thread+0xb4/0xf0 kernel/fork.c:2985 rest_init+0x27/0x2b0 init/main.c:691 arch_call_rest_init+0x13/0x30 init/main.c:823 start_kernel+0x39f/0x480 init/main.c:1068 x86_64_start_reservations+0x18/0x30 arch/x86/kernel/head64.c:556 x86_64_start_kernel+0xb2/0xc0 arch/x86/kernel/head64.c:537 secondary_startup_64_no_verify+0x166/0x16b -> #3 (&p->pi_lock){-.-.}-{2:2}: __raw_spin_lock_irqsave include/linux/spinlock_api_smp.h:110 [inline] _raw_spin_lock_irqsave+0x3a/0x50 kernel/locking/spinlock.c:162 class_raw_spinlock_irqsave_constructor include/linux/spinlock.h:518 [inline] try_to_wake_up+0xb0/0x15d0 kernel/sched/core.c:4213 __wake_up_common+0x140/0x5a0 kernel/sched/wait.c:107 __wake_up_common_lock+0xd6/0x140 kernel/sched/wait.c:138 tty_port_default_wakeup+0x2a/0x40 drivers/tty/tty_port.c:69 serial8250_tx_chars+0x542/0xf60 drivers/tty/serial/8250/8250_port.c:1842 serial8250_handle_irq+0x606/0xbe0 drivers/tty/serial/8250/8250_port.c:1949 serial8250_default_handle_irq+0x94/0x210 drivers/tty/serial/8250/8250_port.c:1969 serial8250_interrupt+0xfc/0x200 drivers/tty/serial/8250/8250_core.c:127 __handle_irq_event_percpu+0x22a/0x740 kernel/irq/handle.c:158 handle_irq_event_percpu kernel/irq/handle.c:193 [inline] handle_irq_event+0xab/0x1e0 kernel/irq/handle.c:210 handle_edge_irq+0x261/0xcf0 kernel/irq/chip.c:831 generic_handle_irq_desc include/linux/irqdesc.h:161 [inline] handle_irq arch/x86/kernel/irq.c:238 [inline] __common_interrupt+0xdb/0x240 arch/x86/kernel/irq.c:257 common_interrupt+0xa9/0xd0 arch/x86/kernel/irq.c:247 asm_common_interrupt+0x26/0x40 arch/x86/include/asm/idtentry.h:636 native_safe_halt arch/x86/include/asm/irqflags.h:48 [inline] arch_safe_halt arch/x86/include/asm/irqflags.h:86 [inline] acpi_safe_halt+0x1b/0x20 drivers/acpi/processor_idle.c:112 acpi_idle_enter+0xc5/0x160 drivers/acpi/processor_idle.c:707 cpuidle_enter_state+0x82/0x500 drivers/cpuidle/cpuidle.c:267 cpuidle_enter+0x4e/0xa0 drivers/cpuidle/cpuidle.c:388 cpuidle_idle_call kernel/sched/idle.c:215 [inline] do_idle+0x315/0x3f0 kernel/sched/idle.c:282 cpu_startup_entry+0x50/0x60 kernel/sched/idle.c:380 start_secondary+0x20e/0x2a0 arch/x86/kernel/smpboot.c:326 secondary_startup_64_no_verify+0x166/0x16b -> #2 (&tty->write_wait){-.-.}-{2:2}: __raw_spin_lock_irqsave include/linux/spinlock_api_smp.h:110 [inline] _raw_spin_lock_irqsave+0x3a/0x50 kernel/locking/spinlock.c:162 __wake_up_common_lock+0xbb/0x140 kernel/sched/wait.c:137 tty_port_default_wakeup+0x2a/0x40 drivers/tty/tty_port.c:69 serial8250_tx_chars+0x542/0xf60 drivers/tty/serial/8250/8250_port.c:1842 serial8250_handle_irq+0x606/0xbe0 drivers/tty/serial/8250/8250_port.c:1949 serial8250_default_handle_irq+0x94/0x210 drivers/tty/serial/8250/8250_port.c:1969 serial8250_interrupt+0xfc/0x200 drivers/tty/serial/8250/8250_core.c:127 __handle_irq_event_percpu+0x22a/0x740 kernel/irq/handle.c:158 handle_irq_event_percpu kernel/irq/handle.c:193 [inline] handle_irq_event+0xab/0x1e0 kernel/irq/handle.c:210 handle_edge_irq+0x261/0xcf0 kernel/irq/chip.c:831 generic_handle_irq_desc include/linux/irqdesc.h:161 [inline] handle_irq arch/x86/kernel/irq.c:238 [inline] __common_interrupt+0xdb/0x240 arch/x86/kernel/irq.c:257 common_interrupt+0xa9/0xd0 arch/x86/kernel/irq.c:247 asm_common_interrupt+0x26/0x40 arch/x86/include/asm/idtentry.h:636 __raw_spin_unlock_irqrestore include/linux/spinlock_api_smp.h:151 [inline] _raw_spin_unlock_irqrestore+0x31/0x70 kernel/locking/spinlock.c:194 spin_unlock_irqrestore include/linux/spinlock.h:406 [inline] uart_port_unlock_irqrestore include/linux/serial_core.h:667 [inline] serial_port_runtime_resume+0x2ae/0x330 drivers/tty/serial/serial_port.c:41 __rpm_callback+0xc5/0x4c0 drivers/base/power/runtime.c:392 rpm_callback+0x1da/0x220 drivers/base/power/runtime.c:446 rpm_resume+0xdb5/0x1980 drivers/base/power/runtime.c:912 pm_runtime_work+0x12e/0x180 drivers/base/power/runtime.c:977 process_one_work+0x884/0x15c0 kernel/workqueue.c:2630 process_scheduled_works kernel/workqueue.c:2703 [inline] worker_thread+0x8b9/0x1290 kernel/workqueue.c:2784 kthread+0x33c/0x440 kernel/kthread.c:388 ret_from_fork+0x45/0x80 arch/x86/kernel/process.c:147 ret_from_fork_asm+0x11/0x20 arch/x86/entry/entry_64.S:304 -> #1 (&port_lock_key){-.-.}-{2:2}: __raw_spin_lock_irqsave include/linux/spinlock_api_smp.h:110 [inline] _raw_spin_lock_irqsave+0x3a/0x50 kernel/locking/spinlock.c:162 uart_port_lock_irqsave include/linux/serial_core.h:616 [inline] serial8250_console_write+0xa7c/0x1060 drivers/tty/serial/8250/8250_port.c:3410 console_emit_next_record kernel/printk/printk.c:2894 [inline] console_flush_all+0x4d5/0xd50 kernel/printk/printk.c:2960 console_unlock+0x10c/0x260 kernel/printk/printk.c:3029 vprintk_emit+0x17f/0x5f0 kernel/printk/printk.c:2304 vprintk+0x7b/0x90 kernel/printk/printk_safe.c:45 _printk+0xc8/0x100 kernel/printk/printk.c:2329 register_console+0xb30/0x1210 kernel/printk/printk.c:3535 univ8250_console_init+0x35/0x50 drivers/tty/serial/8250/8250_core.c:717 console_init+0xba/0x5c0 kernel/printk/printk.c:3681 start_kernel+0x25a/0x480 init/main.c:1004 x86_64_start_reservations+0x18/0x30 arch/x86/kernel/head64.c:556 x86_64_start_kernel+0xb2/0xc0 arch/x86/kernel/head64.c:537 secondary_startup_64_no_verify+0x166/0x16b -> #0 (console_owner){-...}-{0:0}: check_prev_add kernel/locking/lockdep.c:3134 [inline] check_prevs_add kernel/locking/lockdep.c:3253 [inline] validate_chain kernel/locking/lockdep.c:3868 [inline] __lock_acquire+0x2e3d/0x5de0 kernel/locking/lockdep.c:5136 lock_acquire kernel/locking/lockdep.c:5753 [inline] lock_acquire+0x1ae/0x510 kernel/locking/lockdep.c:5718 console_lock_spinning_enable kernel/printk/printk.c:1855 [inline] console_emit_next_record kernel/printk/printk.c:2888 [inline] console_flush_all+0x4af/0xd50 kernel/printk/printk.c:2960 console_unlock+0x10c/0x260 kernel/printk/printk.c:3029 vprintk_emit+0x17f/0x5f0 kernel/printk/printk.c:2304 vprintk+0x7b/0x90 kernel/printk/printk_safe.c:45 _printk+0xc8/0x100 kernel/printk/printk.c:2329 pick_eevdf kernel/sched/fair.c:963 [inline] check_preempt_wakeup_fair+0x553/0x640 kernel/sched/fair.c:8108 wakeup_preempt+0x125/0x320 kernel/sched/core.c:2217 ttwu_do_activate+0x24f/0xd90 kernel/sched/core.c:3767 ttwu_queue kernel/sched/core.c:4008 [inline] try_to_wake_up+0x6bc/0x15d0 kernel/sched/core.c:4329 hrtimer_wakeup+0x47/0x60 kernel/time/hrtimer.c:1942 __run_hrtimer kernel/time/hrtimer.c:1688 [inline] __hrtimer_run_queues+0x203/0xc10 kernel/time/hrtimer.c:1752 hrtimer_interrupt+0x31b/0x800 kernel/time/hrtimer.c:1814 local_apic_timer_interrupt arch/x86/kernel/apic/apic.c:1063 [inline] __sysvec_apic_timer_interrupt+0x105/0x3f0 arch/x86/kernel/apic/apic.c:1080 sysvec_apic_timer_interrupt+0x8e/0xc0 arch/x86/kernel/apic/apic.c:1074 asm_sysvec_apic_timer_interrupt+0x1a/0x20 arch/x86/include/asm/idtentry.h:645 __raw_spin_unlock_irqrestore include/linux/spinlock_api_smp.h:151 [inline] _raw_spin_unlock_irqrestore+0x31/0x70 kernel/locking/spinlock.c:194 __mod_timer+0x8fa/0xea0 kernel/time/timer.c:1134 mod_peer_timer+0x139/0x2c0 drivers/net/wireguard/timers.c:37 wg_timers_any_authenticated_packet_traversal+0x96/0xc0 drivers/net/wireguard/timers.c:215 wg_packet_create_data_done drivers/net/wireguard/send.c:247 [inline] wg_packet_tx_worker+0x13f/0x810 drivers/net/wireguard/send.c:276 process_one_work+0x884/0x15c0 kernel/workqueue.c:2630 process_scheduled_works kernel/workqueue.c:2703 [inline] worker_thread+0x8b9/0x1290 kernel/workqueue.c:2784 kthread+0x33c/0x440 kernel/kthread.c:388 ret_from_fork+0x45/0x80 arch/x86/kernel/process.c:147 ret_from_fork_asm+0x11/0x20 arch/x86/entry/entry_64.S:304 other info that might help us debug this: Chain exists of: console_owner --> &p->pi_lock --> &rq->__lock Possible unsafe locking scenario: CPU0 CPU1 ---- ---- lock(&rq->__lock); lock(&p->pi_lock); lock(&rq->__lock); lock(console_owner); *** DEADLOCK *** 7 locks held by kworker/0:2/780: #0: ffff8881433f6d38 ((wq_completion)wg-crypt-wg1#2){+.+.}-{0:0}, at: process_one_work+0x787/0x15c0 kernel/workqueue.c:2605 #1: ffffc90003edfd80 ((work_completion)(&peer->transmit_packet_work)){+.+.}-{0:0}, at: process_one_work+0x7e9/0x15c0 kernel/workqueue.c:2606 #2: ffffffff8cbab180 (rcu_read_lock_bh){....}-{1:2}, at: local_bh_disable include/linux/bottom_half.h:20 [inline] #2: ffffffff8cbab180 (rcu_read_lock_bh){....}-{1:2}, at: rcu_read_lock_bh include/linux/rcupdate.h:801 [inline] #2: ffffffff8cbab180 (rcu_read_lock_bh){....}-{1:2}, at: mod_peer_timer+0x18/0x2c0 drivers/net/wireguard/timers.c:34 #3: ffff888018fda7b8 (&p->pi_lock){-.-.}-{2:2}, at: class_raw_spinlock_irqsave_constructor include/linux/spinlock.h:518 [inline] #3: ffff888018fda7b8 (&p->pi_lock){-.-.}-{2:2}, at: try_to_wake_up+0xb0/0x15d0 kernel/sched/core.c:4213 #4: ffff8880b983c718 (&rq->__lock){-.-.}-{2:2}, at: raw_spin_rq_lock_nested+0x29/0x130 kernel/sched/core.c:558 #5: ffffffff8cb98960 (console_lock){+.+.}-{0:0}, at: console_trylock_spinning kernel/printk/printk.c:1924 [inline] #5: ffffffff8cb98960 (console_lock){+.+.}-{0:0}, at: vprintk_emit+0x162/0x5f0 kernel/printk/printk.c:2303 #6: ffffffff8cb989d0 (console_srcu){....}-{0:0}, at: rcu_lock_acquire include/linux/rcupdate.h:303 [inline] #6: ffffffff8cb989d0 (console_srcu){....}-{0:0}, at: srcu_read_lock_nmisafe include/linux/srcu.h:232 [inline] #6: ffffffff8cb989d0 (console_srcu){....}-{0:0}, at: console_srcu_read_lock kernel/printk/printk.c:286 [inline] #6: ffffffff8cb989d0 (console_srcu){....}-{0:0}, at: console_flush_all+0x12a/0xd50 kernel/printk/printk.c:2952 stack backtrace: CPU: 0 PID: 780 Comm: kworker/0:2 Not tainted 6.6.0-rc4-next-20231004-syzkaller #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 09/06/2023 Workqueue: wg-crypt-wg1 wg_packet_tx_worker Call Trace: __dump_stack lib/dump_stack.c:88 [inline] dump_stack_lvl+0xd9/0x1b0 lib/dump_stack.c:106 check_noncircular+0x311/0x3f0 kernel/locking/lockdep.c:2187 check_prev_add kernel/locking/lockdep.c:3134 [inline] check_prevs_add kernel/locking/lockdep.c:3253 [inline] validate_chain kernel/locking/lockdep.c:3868 [inline] __lock_acquire+0x2e3d/0x5de0 kernel/locking/lockdep.c:5136 lock_acquire kernel/locking/lockdep.c:5753 [inline] lock_acquire+0x1ae/0x510 kernel/locking/lockdep.c:5718 console_lock_spinning_enable kernel/printk/printk.c:1855 [inline] console_emit_next_record kernel/printk/printk.c:2888 [inline] console_flush_all+0x4af/0xd50 kernel/printk/printk.c:2960 console_unlock+0x10c/0x260 kernel/printk/printk.c:3029 vprintk_emit+0x17f/0x5f0 kernel/printk/printk.c:2304 vprintk+0x7b/0x90 kernel/printk/printk_safe.c:45 _printk+0xc8/0x100 kernel/printk/printk.c:2329 pick_eevdf kernel/sched/fair.c:963 [inline] check_preempt_wakeup_fair+0x553/0x640 kernel/sched/fair.c:8108 wakeup_preempt+0x125/0x320 kernel/sched/core.c:2217 ttwu_do_activate+0x24f/0xd90 kernel/sched/core.c:3767 ttwu_queue kernel/sched/core.c:4008 [inline] try_to_wake_up+0x6bc/0x15d0 kernel/sched/core.c:4329 hrtimer_wakeup+0x47/0x60 kernel/time/hrtimer.c:1942 __run_hrtimer kernel/time/hrtimer.c:1688 [inline] __hrtimer_run_queues+0x203/0xc10 kernel/time/hrtimer.c:1752 hrtimer_interrupt+0x31b/0x800 kernel/time/hrtimer.c:1814 local_apic_timer_interrupt arch/x86/kernel/apic/apic.c:1063 [inline] __sysvec_apic_timer_interrupt+0x105/0x3f0 arch/x86/kernel/apic/apic.c:1080 sysvec_apic_timer_interrupt+0x8e/0xc0 arch/x86/kernel/apic/apic.c:1074 asm_sysvec_apic_timer_interrupt+0x1a/0x20 arch/x86/include/asm/idtentry.h:645 RIP: 0010:__raw_spin_unlock_irqrestore include/linux/spinlock_api_smp.h:152 [inline] RIP: 0010:_raw_spin_unlock_irqrestore+0x31/0x70 kernel/locking/spinlock.c:194 Code: f5 53 48 8b 74 24 10 48 89 fb 48 83 c7 18 e8 c6 d6 1e f7 48 89 df e8 be 55 1f f7 f7 c5 00 02 00 00 75 1f 9c 58 f6 c4 02 75 2f 01 00 00 00 e8 d5 a3 10 f7 65 8b 05 c6 cf b9 75 85 c0 74 12 5b RSP: 0018:ffffc90003edfb10 EFLAGS: 00000246 RAX: 0000000000000006 RBX: ffff8880b98297c0 RCX: 1ffffffff234e1f4 RDX: 0000000000000000 RSI: ffffffff8a8caaa0 RDI: ffffffff8ae97aa0 RBP: 0000000000000293 R08: 0000000000000001 R09: fffffbfff2341fed R10: ffffffff91a0ff6f R11: 0000000000000001 R12: ffff8880b98297c0 R13: 0000000000000001 R14: 00000000ffffb5ed R15: ffff8880b98297c0 __mod_timer+0x8fa/0xea0 kernel/time/timer.c:1134 mod_peer_timer+0x139/0x2c0 drivers/net/wireguard/timers.c:37 wg_timers_any_authenticated_packet_traversal+0x96/0xc0 drivers/net/wireguard/timers.c:215 wg_packet_create_data_done drivers/net/wireguard/send.c:247 [inline] wg_packet_tx_worker+0x13f/0x810 drivers/net/wireguard/send.c:276 process_one_work+0x884/0x15c0 kernel/workqueue.c:2630 process_scheduled_works kernel/workqueue.c:2703 [inline] worker_thread+0x8b9/0x1290 kernel/workqueue.c:2784 kthread+0x33c/0x440 kernel/kthread.c:388 ret_from_fork+0x45/0x80 arch/x86/kernel/process.c:147 ret_from_fork_asm+0x11/0x20 arch/x86/entry/entry_64.S:304 ---------------- Code disassembly (best guess): 0: f5 cmc 1: 53 push %rbx 2: 48 8b 74 24 10 mov 0x10(%rsp),%rsi 7: 48 89 fb mov %rdi,%rbx a: 48 83 c7 18 add $0x18,%rdi e: e8 c6 d6 1e f7 call 0xf71ed6d9 13: 48 89 df mov %rbx,%rdi 16: e8 be 55 1f f7 call 0xf71f55d9 1b: f7 c5 00 02 00 00 test $0x200,%ebp 21: 75 1f jne 0x42 23: 9c pushf 24: 58 pop %rax 25: f6 c4 02 test $0x2,%ah 28: 75 2f jne 0x59 * 2a: bf 01 00 00 00 mov $0x1,%edi <-- trapping instruction 2f: e8 d5 a3 10 f7 call 0xf710a409 34: 65 8b 05 c6 cf b9 75 mov %gs:0x75b9cfc6(%rip),%eax # 0x75b9d001 3b: 85 c0 test %eax,%eax 3d: 74 12 je 0x51 3f: 5b pop %rbx