bisecting fixing commit since 2a987e65025e2b79c6d453b78cb5985ac6e5eb26 building syzkaller on a4a2a50158b25d4af0fd07528f38e6656b903d68 testing commit 2a987e65025e2b79c6d453b78cb5985ac6e5eb26 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 9033bd3ac2aff8b673b26e20430b62fd9e396aac01681810044fe0833e1c0799 run #0: crashed: possible deadlock in sch_direct_xmit run #1: crashed: possible deadlock in sch_direct_xmit run #2: crashed: possible deadlock in sch_direct_xmit run #3: crashed: possible deadlock in sch_direct_xmit run #4: crashed: possible deadlock in sch_direct_xmit run #5: crashed: possible deadlock in sch_direct_xmit run #6: crashed: possible deadlock in sch_direct_xmit run #7: crashed: possible deadlock in sch_direct_xmit run #8: crashed: possible deadlock in sch_direct_xmit run #9: crashed: possible deadlock in sch_direct_xmit run #10: crashed: possible deadlock in sch_direct_xmit run #11: crashed: possible deadlock in sch_direct_xmit run #12: crashed: possible deadlock in sch_direct_xmit run #13: crashed: possible deadlock in sch_direct_xmit run #14: crashed: possible deadlock in sch_direct_xmit run #15: crashed: possible deadlock in sch_direct_xmit run #16: crashed: possible deadlock in sch_direct_xmit run #17: crashed: possible deadlock in sch_direct_xmit run #18: crashed: possible deadlock in sch_direct_xmit run #19: boot failed: can't ssh into the instance testing current HEAD dfd42facf1e4ada021b939b4e19c935dcdd55566 testing commit dfd42facf1e4ada021b939b4e19c935dcdd55566 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: b3dbe0e4de84f09b2840f7023fb29150a6a1c82a6cdd94245295d6b33fed7e7b all runs: crashed: possible deadlock in sch_direct_xmit revisions tested: 2, total time: 25m16.925794699s (build: 13m10.419514355s, test: 11m25.518366383s) the crash still happens on HEAD commit msg: Linux 5.17-rc3 crash: possible deadlock in sch_direct_xmit ============================================ WARNING: possible recursive locking detected 5.17.0-rc3-syzkaller #0 Not tainted -------------------------------------------- syz-executor.1/4035 is trying to acquire lock: ffff88807371c398 (_xmit_ETHER#2){+.-.}-{2:2}, at: spin_lock include/linux/spinlock.h:349 [inline] ffff88807371c398 (_xmit_ETHER#2){+.-.}-{2:2}, at: __netif_tx_lock include/linux/netdevice.h:4123 [inline] ffff88807371c398 (_xmit_ETHER#2){+.-.}-{2:2}, at: sch_direct_xmit+0x24a/0x10a0 net/sched/sch_generic.c:340 but task is already holding lock: ffff888078eb3c98 (_xmit_ETHER#2){+.-.}-{2:2} , at: spin_lock include/linux/spinlock.h:349 [inline] , at: __netif_tx_lock include/linux/netdevice.h:4123 [inline] , at: sch_direct_xmit+0x24a/0x10a0 net/sched/sch_generic.c:340 other info that might help us debug this: Possible unsafe locking scenario: CPU0 ---- lock(_xmit_ETHER#2); lock(_xmit_ETHER#2); *** DEADLOCK *** May be due to missing lock nesting notation 7 locks held by syz-executor.1/4035: #0: ffffffff8ad7a280 (rcu_read_lock_bh){....}-{1:2}, at: lwtunnel_xmit_redirect include/net/lwtunnel.h:95 [inline] #0: ffffffff8ad7a280 (rcu_read_lock_bh){....}-{1:2}, at: ip_finish_output2+0x220/0x1bd0 net/ipv4/ip_output.c:214 #1: ffffffff8ad7a280 (rcu_read_lock_bh){....}-{1:2}, at: __dev_queue_xmit+0x1cc/0x2d50 net/core/dev.c:4040 #2: ffff88806bb0a258 (dev->qdisc_tx_busylock ?: &qdisc_tx_busylock){+...}-{2:2}, at: spin_trylock include/linux/spinlock.h:359 [inline] #2: ffff88806bb0a258 (dev->qdisc_tx_busylock ?: &qdisc_tx_busylock){+...}-{2:2}, at: qdisc_run_begin include/net/sch_generic.h:187 [inline] #2: ffff88806bb0a258 (dev->qdisc_tx_busylock ?: &qdisc_tx_busylock){+...}-{2:2}, at: qdisc_run_begin include/net/sch_generic.h:184 [inline] #2: ffff88806bb0a258 (dev->qdisc_tx_busylock ?: &qdisc_tx_busylock){+...}-{2:2}, at: __dev_xmit_skb net/core/dev.c:3687 [inline] #2: ffff88806bb0a258 (dev->qdisc_tx_busylock ?: &qdisc_tx_busylock){+...}-{2:2}, at: __dev_queue_xmit+0xf14/0x2d50 net/core/dev.c:4081 #3: ffff888078eb3c98 (_xmit_ETHER#2){+.-.}-{2:2}, at: spin_lock include/linux/spinlock.h:349 [inline] #3: ffff888078eb3c98 (_xmit_ETHER#2){+.-.}-{2:2}, at: __netif_tx_lock include/linux/netdevice.h:4123 [inline] #3: ffff888078eb3c98 (_xmit_ETHER#2){+.-.}-{2:2}, at: sch_direct_xmit+0x24a/0x10a0 net/sched/sch_generic.c:340 #4: ffffffff8ad7a280 (rcu_read_lock_bh){....}-{1:2}, at: lwtunnel_xmit_redirect include/net/lwtunnel.h:95 [inline] #4: ffffffff8ad7a280 (rcu_read_lock_bh){....}-{1:2}, at: ip_finish_output2+0x220/0x1bd0 net/ipv4/ip_output.c:214 #5: ffffffff8ad7a280 (rcu_read_lock_bh){....}-{1:2}, at: __dev_queue_xmit+0x1cc/0x2d50 net/core/dev.c:4040 #6: ffff88806ad33258 (dev->qdisc_tx_busylock ?: &qdisc_tx_busylock){+...}-{2:2}, at: spin_trylock include/linux/spinlock.h:359 [inline] #6: ffff88806ad33258 (dev->qdisc_tx_busylock ?: &qdisc_tx_busylock){+...}-{2:2}, at: qdisc_run_begin include/net/sch_generic.h:187 [inline] #6: ffff88806ad33258 (dev->qdisc_tx_busylock ?: &qdisc_tx_busylock){+...}-{2:2}, at: qdisc_run_begin include/net/sch_generic.h:184 [inline] #6: ffff88806ad33258 (dev->qdisc_tx_busylock ?: &qdisc_tx_busylock){+...}-{2:2}, at: __dev_xmit_skb net/core/dev.c:3687 [inline] #6: ffff88806ad33258 (dev->qdisc_tx_busylock ?: &qdisc_tx_busylock){+...}-{2:2}, at: __dev_queue_xmit+0xf14/0x2d50 net/core/dev.c:4081 stack backtrace: CPU: 1 PID: 4035 Comm: syz-executor.1 Not tainted 5.17.0-rc3-syzkaller #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: __dump_stack lib/dump_stack.c:88 [inline] dump_stack_lvl+0x57/0x7d lib/dump_stack.c:106 print_deadlock_bug kernel/locking/lockdep.c:2956 [inline] check_deadlock kernel/locking/lockdep.c:2999 [inline] validate_chain kernel/locking/lockdep.c:3788 [inline] __lock_acquire.cold+0x12c/0x3a9 kernel/locking/lockdep.c:5027 lock_acquire kernel/locking/lockdep.c:5639 [inline] lock_acquire+0x1ab/0x510 kernel/locking/lockdep.c:5604 __raw_spin_lock include/linux/spinlock_api_smp.h:133 [inline] _raw_spin_lock+0x2a/0x40 kernel/locking/spinlock.c:154 spin_lock include/linux/spinlock.h:349 [inline] __netif_tx_lock include/linux/netdevice.h:4123 [inline] sch_direct_xmit+0x24a/0x10a0 net/sched/sch_generic.c:340 __dev_xmit_skb net/core/dev.c:3700 [inline] __dev_queue_xmit+0x111f/0x2d50 net/core/dev.c:4081 neigh_output include/net/neighbour.h:549 [inline] ip_finish_output2+0x5ee/0x1bd0 net/ipv4/ip_output.c:228 iptunnel_xmit+0x5b8/0xa00 net/ipv4/ip_tunnel_core.c:82 ip_tunnel_xmit+0x1340/0x2370 net/ipv4/ip_tunnel.c:810 erspan_xmit+0x106d/0x2c90 net/ipv4/ip_gre.c:713 __netdev_start_xmit include/linux/netdevice.h:4683 [inline] netdev_start_xmit include/linux/netdevice.h:4697 [inline] xmit_one net/core/dev.c:3473 [inline] dev_hard_start_xmit+0x1a5/0x700 net/core/dev.c:3489 sch_direct_xmit+0x12c/0x10a0 net/sched/sch_generic.c:342 __dev_xmit_skb net/core/dev.c:3700 [inline] __dev_queue_xmit+0x111f/0x2d50 net/core/dev.c:4081 neigh_hh_output include/net/neighbour.h:533 [inline] neigh_output include/net/neighbour.h:547 [inline] ip_finish_output2+0xfc9/0x1bd0 net/ipv4/ip_output.c:228 dst_output include/net/dst.h:451 [inline] ip_local_out net/ipv4/ip_output.c:126 [inline] ip_send_skb+0x9f/0x1c0 net/ipv4/ip_output.c:1570 udp_send_skb+0x5d8/0x1460 net/ipv4/udp.c:967 udp_sendmsg+0x16f6/0x2040 net/ipv4/udp.c:1254 udpv6_sendmsg+0x5bb/0x2940 net/ipv6/udp.c:1367 sock_sendmsg_nosec net/socket.c:705 [inline] sock_sendmsg+0xab/0xe0 net/socket.c:725 ____sys_sendmsg+0x392/0x7a0 net/socket.c:2413 ___sys_sendmsg+0xd3/0x150 net/socket.c:2467 __sys_sendmmsg+0x141/0x310 net/socket.c:2553 __do_sys_sendmmsg net/socket.c:2582 [inline] __se_sys_sendmmsg net/socket.c:2579 [inline] __x64_sys_sendmmsg+0x94/0x100 net/socket.c:2579 do_syscall_x64 arch/x86/entry/common.c:50 [inline] do_syscall_64+0x35/0xb0 arch/x86/entry/common.c:80 entry_SYSCALL_64_after_hwframe+0x44/0xae RIP: 0033:0x7f204fe96af9 Code: ff ff c3 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 40 00 48 89 f8 48 89 f7 48 89 d6 48 89 ca 4d 89 c2 4d 89 c8 4c 8b 4c 24 08 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 c7 c1 bc ff ff ff f7 d8 64 89 01 48 RSP: 002b:00007f204f5eb188 EFLAGS: 00000246 ORIG_RAX: 0000000000000133 RAX: ffffffffffffffda RBX: 00007f204ffaa028 RCX: 00007f204fe96af9 RDX: 0000000000000001 RSI: 0000000020004d80 RDI: 0000000000000004 RBP: 00007f204fef0ff7 R08: 0000000000000000 R09: 0000000000000000 R10: 0000000004000000 R11: 0000000000000246 R12: 0000000000000000 R13: 00007ffe31fee06f R14: 00007f204f5eb300 R15: 0000000000022000