bisecting fixing commit since fb893de323e2d39f7a1f6df425703a2edbdf56ea building syzkaller on bc15f7dbbc1c6e2042a7115b3fdacc0ada8c35e7 testing commit fb893de323e2d39f7a1f6df425703a2edbdf56ea with gcc (GCC) 8.1.0 kernel signature: 670a8d859fe1d3549c7cbed15d10252ca78502a62417bab44cd6f7c82234735b all runs: crashed: possible deadlock in __dev_queue_xmit testing current HEAD c2dc4c073fb71b50904493657a7622b481b346e3 testing commit c2dc4c073fb71b50904493657a7622b481b346e3 with gcc (GCC) 8.1.0 kernel signature: e84983a12e50a59847d67f647e9fbf2e36647876f55eb25143e5c245305c7b1a all runs: crashed: possible deadlock in __dev_queue_xmit revisions tested: 2, total time: 16m42.316162781s (build: 10m29.465232264s, test: 5m37.689629703s) the crash still happens on HEAD commit msg: Merge tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost crash: possible deadlock in __dev_queue_xmit ============================================ WARNING: possible recursive locking detected 5.10.0-rc1-syzkaller #0 Not tainted -------------------------------------------- syz-executor.1/8319 is trying to acquire lock: ffff88811d661898 (_xmit_ETHER#2){+.-.}-{2:2}, at: spin_lock include/linux/spinlock.h:354 [inline] ffff88811d661898 (_xmit_ETHER#2){+.-.}-{2:2}, at: __netif_tx_lock include/linux/netdevice.h:4158 [inline] ffff88811d661898 (_xmit_ETHER#2){+.-.}-{2:2}, at: __dev_queue_xmit+0xc15/0xd20 net/core/dev.c:4136 but task is already holding lock: ffff88811bfebc98 (_xmit_ETHER#2){+.-.}-{2:2}, at: spin_lock include/linux/spinlock.h:354 [inline] ffff88811bfebc98 (_xmit_ETHER#2){+.-.}-{2:2}, at: __netif_tx_lock include/linux/netdevice.h:4158 [inline] ffff88811bfebc98 (_xmit_ETHER#2){+.-.}-{2:2}, at: sch_direct_xmit+0x127/0x300 net/sched/sch_generic.c:311 other info that might help us debug this: Possible unsafe locking scenario: CPU0 ---- lock(_xmit_ETHER#2); lock(_xmit_ETHER#2); *** DEADLOCK *** May be due to missing lock nesting notation 10 locks held by syz-executor.1/8319: #0: ffffffff843d8940 (rcu_read_lock){....}-{1:2}, at: l3mdev_l3_out include/net/l3mdev.h:198 [inline] ffffffff843d8940 (rcu_read_lock){....}-{1:2}, at: l3mdev_ip6_out include/net/l3mdev.h:219 [inline] ffffffff843d8940 (rcu_read_lock){....}-{1:2}, at: rawv6_send_hdrinc net/ipv6/raw.c:677 [inline] ffffffff843d8940 (rcu_read_lock){....}-{1:2}, at: rawv6_sendmsg+0x9a5/0x12e0 net/ipv6/raw.c:944 #1: ffffffff843d8900 (rcu_read_lock_bh){....}-{1:2}, at: lwtunnel_xmit_redirect include/net/lwtunnel.h:92 [inline] #1: ffffffff843d8900 (rcu_read_lock_bh){....}-{1:2}, at: ip6_finish_output2+0x7d/0x9f0 net/ipv6/ip6_output.c:103 #2: ffffffff843d8900 (rcu_read_lock_bh){....}-{1:2}, at: __dev_queue_xmit+0x6d/0xd20 net/core/dev.c:4074 #3: ffff88811d795a58 (&sch->seqlock){+...}-{2:2}, at: spin_trylock include/linux/spinlock.h:364 [inline] #3: ffff88811d795a58 (&sch->seqlock){+...}-{2:2}, at: qdisc_run_begin include/net/sch_generic.h:162 [inline] #3: ffff88811d795a58 (&sch->seqlock){+...}-{2:2}, at: qdisc_run include/net/pkt_sched.h:125 [inline] #3: ffff88811d795a58 (&sch->seqlock){+...}-{2:2}, at: __dev_xmit_skb net/core/dev.c:3755 [inline] #3: ffff88811d795a58 (&sch->seqlock){+...}-{2:2}, at: __dev_queue_xmit+0x4c3/0xd20 net/core/dev.c:4108 #4: ffff88811d795948 ( dev->qdisc_running_key ?: &qdisc_running_key){+...}-{0:0}, at: neigh_hh_output include/net/neighbour.h:498 [inline] dev->qdisc_running_key ?: &qdisc_running_key){+...}-{0:0}, at: neigh_output include/net/neighbour.h:507 [inline] dev->qdisc_running_key ?: &qdisc_running_key){+...}-{0:0}, at: ip6_finish_output2+0x35b/0x9f0 net/ipv6/ip6_output.c:117 #5: ffff88811bfebc98 (_xmit_ETHER#2){+.-.}-{2:2}, at: spin_lock include/linux/spinlock.h:354 [inline] #5: ffff88811bfebc98 (_xmit_ETHER#2){+.-.}-{2:2}, at: __netif_tx_lock include/linux/netdevice.h:4158 [inline] #5: ffff88811bfebc98 (_xmit_ETHER#2){+.-.}-{2:2}, at: sch_direct_xmit+0x127/0x300 net/sched/sch_generic.c:311 #6: ffff888114aba220 (k-slock-AF_INET6){+...}-{2:2}, at: spin_trylock include/linux/spinlock.h:364 [inline] #6: ffff888114aba220 (k-slock-AF_INET6){+...}-{2:2}, at: icmpv6_xmit_lock net/ipv6/icmp.c:117 [inline] #6: ffff888114aba220 (k-slock-AF_INET6){+...}-{2:2}, at: icmp6_send+0x3f3/0xb30 net/ipv6/icmp.c:541 #7: ffffffff843d8940 (rcu_read_lock){....}-{1:2}, at: ip6_sk_dst_hoplimit include/net/ipv6.h:827 [inline] #7: ffffffff843d8940 (rcu_read_lock){....}-{1:2}, at: icmp6_send+0x5f6/0xb30 net/ipv6/icmp.c:595 #8: ffffffff843d8900 (rcu_read_lock_bh){....}-{1:2}, at: lwtunnel_xmit_redirect include/net/lwtunnel.h:92 [inline] #8: ffffffff843d8900 (rcu_read_lock_bh){....}-{1:2}, at: ip6_finish_output2+0x7d/0x9f0 net/ipv6/ip6_output.c:103 #9: ffffffff843d8900 (rcu_read_lock_bh){....}-{1:2}, at: __dev_queue_xmit+0x6d/0xd20 net/core/dev.c:4074 stack backtrace: CPU: 0 PID: 8319 Comm: syz-executor.1 Not tainted 5.10.0-rc1-syzkaller #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0xa3/0xc8 lib/dump_stack.c:118 print_deadlock_bug kernel/locking/lockdep.c:2759 [inline] check_deadlock kernel/locking/lockdep.c:2800 [inline] validate_chain kernel/locking/lockdep.c:3591 [inline] __lock_acquire.cold.74+0x12d/0x2a3 kernel/locking/lockdep.c:4837 lock_acquire+0xf9/0x420 kernel/locking/lockdep.c:5442 __raw_spin_lock include/linux/spinlock_api_smp.h:142 [inline] _raw_spin_lock+0x2a/0x40 kernel/locking/spinlock.c:151 spin_lock include/linux/spinlock.h:354 [inline] __netif_tx_lock include/linux/netdevice.h:4158 [inline] __dev_queue_xmit+0xc15/0xd20 net/core/dev.c:4136 neigh_hh_output include/net/neighbour.h:498 [inline] neigh_output include/net/neighbour.h:507 [inline] ip6_finish_output2+0x35b/0x9f0 net/ipv6/ip6_output.c:117 NF_HOOK_COND include/linux/netfilter.h:290 [inline] ip6_output+0x79/0x2b0 net/ipv6/ip6_output.c:176 ip6_send_skb+0x24/0x90 net/ipv6/ip6_output.c:1865 icmp6_send+0x8e8/0xb30 net/ipv6/icmp.c:620 icmpv6_send include/linux/icmpv6.h:24 [inline] ip6_link_failure+0x1e/0x210 net/ipv6/route.c:2669 dst_link_failure include/net/dst.h:426 [inline] ip_tunnel_xmit+0x6ad/0xca0 net/ipv4/ip_tunnel.c:816 erspan_xmit+0x149/0x980 net/ipv4/ip_gre.c:702 __netdev_start_xmit include/linux/netdevice.h:4718 [inline] netdev_start_xmit include/linux/netdevice.h:4732 [inline] xmit_one net/core/dev.c:3564 [inline] dev_hard_start_xmit+0xf9/0x360 net/core/dev.c:3580 sch_direct_xmit+0x14b/0x300 net/sched/sch_generic.c:313 qdisc_restart net/sched/sch_generic.c:376 [inline] __qdisc_run+0x151/0x5c0 net/sched/sch_generic.c:384 qdisc_run include/net/pkt_sched.h:131 [inline] __dev_xmit_skb net/core/dev.c:3755 [inline] __dev_queue_xmit+0x513/0xd20 net/core/dev.c:4108 neigh_hh_output include/net/neighbour.h:498 [inline] neigh_output include/net/neighbour.h:507 [inline] ip6_finish_output2+0x35b/0x9f0 net/ipv6/ip6_output.c:117 NF_HOOK_COND include/linux/netfilter.h:290 [inline] ip6_output+0x79/0x2b0 net/ipv6/ip6_output.c:176 dst_output include/net/dst.h:443 [inline] NF_HOOK include/linux/netfilter.h:301 [inline] rawv6_send_hdrinc net/ipv6/raw.c:687 [inline] rawv6_sendmsg+0xa4d/0x12e0 net/ipv6/raw.c:944 sock_sendmsg_nosec net/socket.c:651 [inline] sock_sendmsg+0x2b/0x40 net/socket.c:671 sock_write_iter+0x95/0xf0 net/socket.c:998 call_write_iter include/linux/fs.h:1887 [inline] new_sync_write+0x18e/0x1a0 fs/read_write.c:518 vfs_write+0x195/0x230 fs/read_write.c:605 ksys_write+0xb9/0xd0 fs/read_write.c:658 do_syscall_64+0x2d/0x70 arch/x86/entry/common.c:46 entry_SYSCALL_64_after_hwframe+0x44/0xa9 RIP: 0033:0x45d189 Code: 5d b4 fb ff c3 66 2e 0f 1f 84 00 00 00 00 00 66 90 48 89 f8 48 89 f7 48 89 d6 48 89 ca 4d 89 c2 4d 89 c8 4c 8b 4c 24 08 0f 05 <48> 3d 01 f0 ff ff 0f 83 2b b4 fb ff c3 66 2e 0f 1f 84 00 00 00 00 RSP: 002b:00007f46b00a0c78 EFLAGS: 00000246 ORIG_RAX: 0000000000000001 RAX: ffffffffffffffda RBX: 0000000000035940 RCX: 000000000045d189 RDX: 0000000000000028 RSI: 0000000020000140 RDI: 0000000000000005 RBP: 000000000118d020 R08: 0000000000000000 R09: 0000000000000000 R10: 0000000000000000 R11: 0000000000000246 R12: 000000000118cfec R13: 00007ffdd9ee300f R14: 00007f46b00a19c0 R15: 000000000118cfec