bisecting fixing commit since c92a9a461dff6140c539c61e457aa97df29517d6 building syzkaller on c9e7aeaef64e4e16a32baac1c66d772afbaf8ed0 testing commit c92a9a461dff6140c539c61e457aa97df29517d6 with gcc (GCC) 8.1.0 all runs: crashed: possible deadlock in sch_direct_xmit testing current HEAD 5c6207539aea8b22490f9569db5aa72ddfd0d486 testing commit 5c6207539aea8b22490f9569db5aa72ddfd0d486 with gcc (GCC) 8.1.0 all runs: crashed: possible deadlock in sch_direct_xmit revisions tested: 2, total time: 29m35.203026657s (build: 7m47.983700117s, test: 20m28.179137406s) the crash still happens on HEAD crash: possible deadlock in sch_direct_xmit ============================================ WARNING: possible recursive locking detected 5.3.0-rc2+ #1 Not tainted -------------------------------------------- syz-executor902/11564 is trying to acquire lock: 0000000032ad3294 (_xmit_ETHER#2){+.-.}, at: spin_lock include/linux/spinlock.h:338 [inline] 0000000032ad3294 (_xmit_ETHER#2){+.-.}, at: __netif_tx_lock include/linux/netdevice.h:3890 [inline] 0000000032ad3294 (_xmit_ETHER#2){+.-.}, at: sch_direct_xmit+0x22a/0x1170 net/sched/sch_generic.c:306 but task is already holding lock: 00000000cc56ef58 (_xmit_ETHER#2){+.-.}, at: spin_lock include/linux/spinlock.h:338 [inline] 00000000cc56ef58 (_xmit_ETHER#2){+.-.}, at: __netif_tx_lock include/linux/netdevice.h:3890 [inline] 00000000cc56ef58 (_xmit_ETHER#2){+.-.}, at: sch_direct_xmit+0x22a/0x1170 net/sched/sch_generic.c:306 other info that might help us debug this: Possible unsafe locking scenario: CPU0 ---- lock(_xmit_ETHER#2); lock(_xmit_ETHER#2); *** DEADLOCK *** May be due to missing lock nesting notation 13 locks held by syz-executor902/11564: #0: 000000006b1c4336 (&tfile->napi_mutex){+.+.}, at: tun_get_user+0x4a3/0x34b0 drivers/net/tun.c:1830 #1: 00000000429bc467 (rcu_read_lock){....}, at: tun_get_user+0x13c3/0x34b0 drivers/net/tun.c:1945 #2: 00000000429bc467 (rcu_read_lock){....}, at: arch_static_branch arch/x86/include/asm/jump_label.h:25 [inline] #2: 00000000429bc467 (rcu_read_lock){....}, at: netif_receive_skb_internal+0x81/0x2b0 net/core/dev.c:5185 #3: 00000000e58c1279 (k-slock-AF_INET){+...}, at: spin_trylock include/linux/spinlock.h:348 [inline] #3: 00000000e58c1279 (k-slock-AF_INET){+...}, at: icmp_xmit_lock net/ipv4/icmp.c:214 [inline] #3: 00000000e58c1279 (k-slock-AF_INET){+...}, at: __icmp_send+0x664/0x1890 net/ipv4/icmp.c:661 #4: 00000000937dc970 (rcu_read_lock_bh){....}, at: lwtunnel_xmit_redirect include/net/lwtunnel.h:92 [inline] #4: 00000000937dc970 (rcu_read_lock_bh){....}, at: ip_finish_output2+0x246/0x2220 net/ipv4/ip_output.c:214 #5: 00000000937dc970 (rcu_read_lock_bh){....}, at: __dev_queue_xmit+0x205/0x2fc0 net/core/dev.c:3804 #6: 00000000534c67ac (&(&sch->seqlock)->rlock){+...}, at: spin_trylock include/linux/spinlock.h:348 [inline] #6: 00000000534c67ac (&(&sch->seqlock)->rlock){+...}, at: qdisc_run_begin include/net/sch_generic.h:159 [inline] #6: 00000000534c67ac (&(&sch->seqlock)->rlock){+...}, at: __dev_xmit_skb net/core/dev.c:3474 [inline] #6: 00000000534c67ac (&(&sch->seqlock)->rlock){+...}, at: __dev_queue_xmit+0x270d/0x2fc0 net/core/dev.c:3838 #7: 000000007164c37a (dev->qdisc_running_key ?: &qdisc_running_key){+...}, at: dev_queue_xmit+0xb/0x10 net/core/dev.c:3902 #8: 00000000cc56ef58 (_xmit_ETHER#2){+.-.}, at: spin_lock include/linux/spinlock.h:338 [inline] #8: 00000000cc56ef58 (_xmit_ETHER#2){+.-.}, at: __netif_tx_lock include/linux/netdevice.h:3890 [inline] #8: 00000000cc56ef58 (_xmit_ETHER#2){+.-.}, at: sch_direct_xmit+0x22a/0x1170 net/sched/sch_generic.c:306 #9: 00000000937dc970 (rcu_read_lock_bh){....}, at: lwtunnel_xmit_redirect include/net/lwtunnel.h:92 [inline] #9: 00000000937dc970 (rcu_read_lock_bh){....}, at: ip_finish_output2+0x246/0x2220 net/ipv4/ip_output.c:214 #10: 00000000937dc970 (rcu_read_lock_bh){....}, at: __dev_queue_xmit+0x205/0x2fc0 net/core/dev.c:3804 #11: 000000008a9e524f (&(&sch->seqlock)->rlock){+...}, at: spin_trylock include/linux/spinlock.h:348 [inline] #11: 000000008a9e524f (&(&sch->seqlock)->rlock){+...}, at: qdisc_run_begin include/net/sch_generic.h:159 [inline] #11: 000000008a9e524f (&(&sch->seqlock)->rlock){+...}, at: __dev_xmit_skb net/core/dev.c:3474 [inline] #11: 000000008a9e524f (&(&sch->seqlock)->rlock){+...}, at: __dev_queue_xmit+0x270d/0x2fc0 net/core/dev.c:3838 #12: 00000000b8e7f723 (dev->qdisc_running_key ?: &qdisc_running_key){+...}, at: dev_queue_xmit+0xb/0x10 net/core/dev.c:3902 stack backtrace: CPU: 1 PID: 11564 Comm: syz-executor902 Not tainted 5.3.0-rc2+ #1 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x115/0x167 lib/dump_stack.c:113 print_deadlock_bug kernel/locking/lockdep.c:2301 [inline] check_deadlock kernel/locking/lockdep.c:2342 [inline] validate_chain kernel/locking/lockdep.c:2881 [inline] __lock_acquire.cold.67+0x13a/0x331 kernel/locking/lockdep.c:3880 lock_acquire+0x194/0x3e0 kernel/locking/lockdep.c:4412 __raw_spin_lock include/linux/spinlock_api_smp.h:142 [inline] _raw_spin_lock+0x2a/0x40 kernel/locking/spinlock.c:151 spin_lock include/linux/spinlock.h:338 [inline] __netif_tx_lock include/linux/netdevice.h:3890 [inline] sch_direct_xmit+0x22a/0x1170 net/sched/sch_generic.c:306 __dev_xmit_skb net/core/dev.c:3477 [inline] __dev_queue_xmit+0x1a29/0x2fc0 net/core/dev.c:3838 dev_queue_xmit+0xb/0x10 net/core/dev.c:3902 neigh_resolve_output+0x475/0x7d0 net/core/neighbour.c:1490 neigh_output include/net/neighbour.h:511 [inline] ip_finish_output2+0x6a2/0x2220 net/ipv4/ip_output.c:228 __ip_finish_output+0x382/0xa90 net/ipv4/ip_output.c:308 ip_finish_output+0x27/0x170 net/ipv4/ip_output.c:318 NF_HOOK_COND include/linux/netfilter.h:294 [inline] ip_mc_output+0x212/0xd40 net/ipv4/ip_output.c:417 dst_output include/net/dst.h:436 [inline] ip_local_out+0x7a/0x140 net/ipv4/ip_output.c:125 iptunnel_xmit+0x534/0x900 net/ipv4/ip_tunnel_core.c:78 ip_tunnel_xmit+0xdab/0x25cb net/ipv4/ip_tunnel.c:818 __gre_xmit+0x4cf/0xa10 net/ipv4/ip_gre.c:444 erspan_xmit+0x816/0x2ef0 net/ipv4/ip_gre.c:679 __netdev_start_xmit include/linux/netdevice.h:4406 [inline] netdev_start_xmit include/linux/netdevice.h:4420 [inline] xmit_one net/core/dev.c:3280 [inline] dev_hard_start_xmit+0x156/0x6e0 net/core/dev.c:3296 sch_direct_xmit+0x2d6/0x1170 net/sched/sch_generic.c:308 __dev_xmit_skb net/core/dev.c:3477 [inline] __dev_queue_xmit+0x1a29/0x2fc0 net/core/dev.c:3838 dev_queue_xmit+0xb/0x10 net/core/dev.c:3902 neigh_resolve_output+0x475/0x7d0 net/core/neighbour.c:1490 neigh_output include/net/neighbour.h:511 [inline] ip_finish_output2+0x6a2/0x2220 net/ipv4/ip_output.c:228 __ip_finish_output+0x382/0xa90 net/ipv4/ip_output.c:308 ip_finish_output+0x27/0x170 net/ipv4/ip_output.c:318 NF_HOOK_COND include/linux/netfilter.h:294 [inline] ip_mc_output+0x212/0xd40 net/ipv4/ip_output.c:417 dst_output include/net/dst.h:436 [inline] ip_local_out+0x7a/0x140 net/ipv4/ip_output.c:125 ip_send_skb+0x36/0xa0 net/ipv4/ip_output.c:1554 ip_push_pending_frames+0x4d/0x70 net/ipv4/ip_output.c:1574 icmp_push_reply+0x2ff/0x550 net/ipv4/icmp.c:389 __icmp_send+0xcbf/0x1890 net/ipv4/icmp.c:732 icmp_send include/net/icmp.h:43 [inline] ip_options_compile+0xad/0xc0 net/ipv4/ip_options.c:486 ip_rcv_options net/ipv4/ip_input.c:278 [inline] ip_rcv_finish_core.isra.15+0x3ae/0x17e0 net/ipv4/ip_input.c:355 ip_rcv_finish+0x4a/0x180 net/ipv4/ip_input.c:411 NF_HOOK include/linux/netfilter.h:305 [inline] ip_rcv+0xcb/0x2f0 net/ipv4/ip_input.c:523 __netif_receive_skb_one_core+0x132/0x190 net/core/dev.c:4999 __netif_receive_skb+0x1f/0x1b0 net/core/dev.c:5113 netif_receive_skb_internal+0xcc/0x2b0 net/core/dev.c:5203 napi_frags_finish net/core/dev.c:5754 [inline] napi_gro_frags+0x8e3/0xae0 net/core/dev.c:5828 tun_get_user+0x22aa/0x34b0 drivers/net/tun.c:1971 tun_chr_write_iter+0xaf/0x150 drivers/net/tun.c:2017 call_write_iter include/linux/fs.h:1870 [inline] do_iter_readv_writev+0x3dd/0x900 fs/read_write.c:693 do_iter_write+0x128/0x540 fs/read_write.c:970 vfs_writev+0x16d/0x2d0 fs/read_write.c:1015 do_writev+0x112/0x2e0 fs/read_write.c:1058 __do_sys_writev fs/read_write.c:1131 [inline] __se_sys_writev fs/read_write.c:1128 [inline] __x64_sys_writev+0x70/0xb0 fs/read_write.c:1128 do_syscall_64+0xd6/0x550 arch/x86/entry/common.c:296 entry_SYSCALL_64_after_hwframe+0x49/0xbe RIP: 0033:0x4450a0 Code: 05 48 3d 01 f0 ff ff 0f 83 9d d0 fb ff c3 66 2e 0f 1f 84 00 00 00 00 00 66 90 83 3d 05 23 29 00 00 75 14 b8 14 00 00 00 0f 05 <48> 3d 01 f0 ff ff 0f 83 74 d0 fb ff c3 48 83 ec 08 e8 fa 2a 00 00 RSP: 002b:00007ffe4c13e248 EFLAGS: 00000246 ORIG_RAX: 0000000000000014 RAX: ffffffffffffffda RBX: 0000000000000046 RCX: 00000000004450a0 RDX: 0000000000000001 RSI: 00007ffe4c13e280 RDI: 0000000000000003 RBP: 00000000004a695f R08: 0000000000000000 R09: 0000000020dcbfaa R10: 0000000000000000 R11: 0000000000000246 R12: 00007ffe4c13e378 R13: 00007ffe4c13e378 R14: 0000000000000000 R15: 0000000000000000