bisecting fixing commit since 64570fbc14f8d7cb3fe3995f20e26bc25ce4b2cc building syzkaller on 838e7e2cd9228583ca33c49a39aea4d863d3e36d testing commit 64570fbc14f8d7cb3fe3995f20e26bc25ce4b2cc compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 73d8aa11e47d71cf9242913b10f4100fd0ed6e1c83357c356b4d35f73c6b965c run #0: crashed: INFO: rcu detected stall in corrupted run #1: crashed: INFO: rcu detected stall in mutex_spin_on_owner run #2: crashed: no output from test machine run #3: crashed: no output from test machine run #4: crashed: no output from test machine run #5: OK run #6: OK run #7: OK run #8: OK run #9: OK run #10: OK run #11: OK run #12: OK run #13: OK run #14: OK run #15: OK run #16: OK run #17: OK run #18: OK run #19: OK reproducer seems to be flaky testing current HEAD a19944809fe9942e6a96292490717904d0690c21 testing commit a19944809fe9942e6a96292490717904d0690c21 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: c40830b3cb5cacf2ff0757870ebb7cf949676ac9358f47bd80538fd0ffd2a9da run #0: crashed: INFO: rcu detected stall in tc_modify_qdisc run #1: crashed: INFO: rcu detected stall in sys_unshare run #2: crashed: INFO: rcu detected stall in corrupted run #3: crashed: INFO: rcu detected stall in sendmsg run #4: crashed: INFO: rcu detected stall in ieee80211_ibss_work run #5: crashed: INFO: rcu detected stall in mutex_spin_on_owner run #6: crashed: INFO: rcu detected stall in batadv_nc_worker run #7: crashed: INFO: rcu detected stall in schedule_timeout run #8: crashed: INFO: rcu detected stall in sys_mprotect run #9: crashed: INFO: rcu detected stall in rtnl_newlink run #10: OK run #11: OK run #12: OK run #13: OK run #14: OK run #15: OK run #16: OK run #17: OK run #18: OK run #19: OK Reproducer flagged being flaky revisions tested: 2, total time: 34m55.332168778s (build: 11m36.371863848s, test: 22m30.374607419s) the crash still happens on HEAD commit msg: Merge tag 'hardening-v5.18-rc3' of git://git.kernel.org/pub/scm/linux/kernel/git/kees/linux crash: INFO: rcu detected stall in rtnl_newlink rcu: INFO: rcu_preempt detected stalls on CPUs/tasks: rcu: 1-...0: (1 ticks this GP) idle=a95/1/0x4000000000000000 softirq=6616/6616 fqs=2100 (detected by 0, t=10503 jiffies, g=5385, q=547) Sending NMI from CPU 0 to CPUs 1: NMI backtrace for cpu 1 CPU: 1 PID: 3929 Comm: syz-executor.5 Not tainted 5.18.0-rc2-syzkaller #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 RIP: 0010:rcu_dynticks_curr_cpu_in_eqs kernel/rcu/tree.c:341 [inline] RIP: 0010:rcu_is_watching+0x9/0xb0 kernel/rcu/tree.c:1138 Code: 90 48 c7 c7 00 f8 ec 88 e8 84 74 34 07 65 8a 05 11 bc a9 7e 0f be c0 c3 66 0f 1f 84 00 00 00 00 00 55 53 65 ff 05 77 7c a8 7e 42 74 34 07 48 c7 c3 10 af 03 00 83 f8 07 89 c5 77 77 48 8d 3c RSP: 0018:ffffc900001e0c50 EFLAGS: 00000006 RAX: 0000000000000001 RBX: 0000000000000001 RCX: ffffffff81543a21 RDX: fffffbfff194ea53 RSI: 0000000000000008 RDI: ffffffff8ca75290 RBP: 1ffff9200003c191 R08: 0000000000000000 R09: ffffffff8ca75297 R10: fffffbfff194ea52 R11: 0000000000000001 R12: 0000000000000001 R13: ffffffff88ed8940 R14: 1ffff9200003c1ad R15: ffff88801a269340 FS: 0000555557478400(0000) GS:ffff8880b9f00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 000055828e1fe5b8 CR3: 0000000072791000 CR4: 0000000000350ee0 Call Trace: rcu_read_lock_held_common kernel/rcu/update.c:108 [inline] rcu_read_lock_sched_held+0x1c/0x70 kernel/rcu/update.c:123 trace_lock_release include/trace/events/lock.h:58 [inline] lock_release+0x522/0x720 kernel/locking/lockdep.c:5652 __raw_spin_unlock_irqrestore include/linux/spinlock_api_smp.h:149 [inline] _raw_spin_unlock_irqrestore+0x16/0x70 kernel/locking/spinlock.c:194 debug_object_deactivate lib/debugobjects.c:757 [inline] debug_object_deactivate+0x264/0x300 lib/debugobjects.c:723 debug_hrtimer_deactivate kernel/time/hrtimer.c:425 [inline] debug_deactivate kernel/time/hrtimer.c:481 [inline] __run_hrtimer kernel/time/hrtimer.c:1653 [inline] __hrtimer_run_queues+0x337/0xb00 kernel/time/hrtimer.c:1749 hrtimer_interrupt+0x2f5/0x780 kernel/time/hrtimer.c:1811 local_apic_timer_interrupt arch/x86/kernel/apic/apic.c:1086 [inline] __sysvec_apic_timer_interrupt+0x146/0x530 arch/x86/kernel/apic/apic.c:1103 sysvec_apic_timer_interrupt+0x8e/0xc0 arch/x86/kernel/apic/apic.c:1097 asm_sysvec_apic_timer_interrupt+0x12/0x20 arch/x86/include/asm/idtentry.h:645 RIP: 0010:console_unlock+0x5ee/0xa50 kernel/printk/printk.c:2779 Code: 58 1e fe ff e8 73 2b 00 00 48 83 3c 24 00 0f 85 69 02 00 00 9c 58 f6 c4 02 0f 85 5c 03 00 00 48 83 3c 24 00 74 01 fb 45 85 e4 <0f> 85 b0 02 00 00 49 c7 c4 a0 6e c2 8a be 04 00 00 00 48 c7 c7 a0 RSP: 0018:ffffc90004a66758 EFLAGS: 00000246 RAX: 0000000000000002 RBX: dffffc0000000000 RCX: 1ffffffff1de43f6 RDX: 0000000000000000 RSI: ffffffff88eb8f00 RDI: ffffffff89429160 RBP: ffffc90004a667b0 R08: 0000000000000001 R09: ffffffff8ef21847 R10: 0000000000000001 R11: 0000000000000001 R12: 0000000000000000 R13: ffffffff8ba0cd88 R14: ffffffff8ba0cd50 R15: 0000000000000000 vprintk_emit+0xa8/0x3c0 kernel/printk/printk.c:2272 dev_vprintk_emit+0x2c9/0x30d drivers/base/core.c:4604 dev_printk_emit+0x9d/0xce drivers/base/core.c:4615 __netdev_printk+0x19c/0x23b net/core/dev.c:11117 netdev_info+0xc8/0xf6 net/core/dev.c:11172 nsim_udp_tunnel_set_port.cold+0xa6/0xe9 drivers/net/netdevsim/udp_tunnels.c:34 udp_tunnel_nic_device_sync_one net/ipv4/udp_tunnel_nic.c:223 [inline] udp_tunnel_nic_device_sync_by_port net/ipv4/udp_tunnel_nic.c:246 [inline] __udp_tunnel_nic_device_sync.part.0+0x6bb/0xc00 net/ipv4/udp_tunnel_nic.c:289 __udp_tunnel_nic_device_sync net/ipv4/udp_tunnel_nic.c:283 [inline] udp_tunnel_nic_device_sync+0xd3/0x130 net/ipv4/udp_tunnel_nic.c:312 udp_tunnel_nic_add_port include/net/udp_tunnel.h:332 [inline] udp_tunnel_nic_add_port include/net/udp_tunnel.h:327 [inline] udp_tunnel_notify_add_rx_port+0x1dd/0x340 net/ipv4/udp_tunnel_core.c:126 geneve_socket_create drivers/net/geneve.c:601 [inline] geneve_sock_add+0x77c/0xc50 drivers/net/geneve.c:676 geneve_open+0xb6/0xe0 drivers/net/geneve.c:718 __dev_open+0x232/0x410 net/core/dev.c:1425 __dev_change_flags+0x44c/0x650 net/core/dev.c:8433 dev_change_flags+0x86/0x150 net/core/dev.c:8504 do_setlink+0x81b/0x2da0 net/core/rtnetlink.c:2731 __rtnl_newlink+0xb05/0x13f0 net/core/rtnetlink.c:3416 rtnl_newlink+0x5a/0x90 net/core/rtnetlink.c:3531 rtnetlink_rcv_msg+0x31d/0x8d0 net/core/rtnetlink.c:5990 netlink_rcv_skb+0x118/0x370 net/netlink/af_netlink.c:2496 netlink_unicast_kernel net/netlink/af_netlink.c:1319 [inline] netlink_unicast+0x433/0x710 net/netlink/af_netlink.c:1345 netlink_sendmsg+0x770/0xc20 net/netlink/af_netlink.c:1921 sock_sendmsg_nosec net/socket.c:705 [inline] sock_sendmsg+0xab/0xe0 net/socket.c:725 __sys_sendto+0x19e/0x270 net/socket.c:2040 __do_sys_sendto net/socket.c:2052 [inline] __se_sys_sendto net/socket.c:2048 [inline] __x64_sys_sendto+0xd8/0x1b0 net/socket.c:2048 do_syscall_x64 arch/x86/entry/common.c:50 [inline] do_syscall_64+0x35/0x80 arch/x86/entry/common.c:80 entry_SYSCALL_64_after_hwframe+0x44/0xae RIP: 0033:0x7ff39943a69c Code: fa fa ff ff 44 8b 4c 24 2c 4c 8b 44 24 20 89 c5 44 8b 54 24 28 48 8b 54 24 18 b8 2c 00 00 00 48 8b 74 24 10 8b 7c 24 08 0f 05 <48> 3d 00 f0 ff ff 77 34 89 ef 48 89 44 24 08 e8 20 fb ff ff 48 8b RSP: 002b:00007ffea68570c0 EFLAGS: 00000293 ORIG_RAX: 000000000000002c RAX: ffffffffffffffda RBX: 00007ff39a4c4320 RCX: 00007ff39943a69c RDX: 000000000000002c RSI: 00007ff39a4c4370 RDI: 0000000000000003 RBP: 0000000000000000 R08: 00007ffea6857114 R09: 000000000000000c R10: 0000000000000000 R11: 0000000000000293 R12: 0000000000000000 R13: 00007ff39a4c4370 R14: 0000000000000003 R15: 0000000000000000 NMI backtrace for cpu 1 CPU: 1 PID: 3929 Comm: syz-executor.5 Not tainted 5.18.0-rc2-syzkaller #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 RIP: 0010:check_preemption_disabled+0x1e/0xe0 lib/smp_processor_id.c:56 Code: bf cc cc cc cc cc cc cc cc cc cc cc 41 54 55 53 48 83 ec 08 65 44 8b 25 10 b3 73 77 65 8b 05 f9 08 74 77 a9 ff ff ff 7f 74 0c <48> 83 c4 08 44 89 e0 5b 5d 41 5c c3 9c 58 f6 c4 02 74 ed 65 48 8b RSP: 0018:ffffc900001e0ca8 EFLAGS: 00000002 RAX: 0000000000010004 RBX: 0000000000000001 RCX: ffffffff81543a21 RDX: fffffbfff194ea53 RSI: ffffffff894290e0 RDI: ffffffff89429120 RBP: 1ffff9200003c1a1 R08: 0000000000000000 R09: ffffffff8ca75297 R10: fffffbfff194ea52 R11: 0000000000000001 R12: 0000000000000001 R13: 0000000140000002 R14: ffff88801a2692e8 R15: ffff888078133080 FS: 0000555557478400(0000) GS:ffff8880b9f00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 000055828e1fe5b8 CR3: 0000000072791000 CR4: 0000000000350ee0 Call Trace: rcu_dynticks_curr_cpu_in_eqs kernel/rcu/tree.c:341 [inline] rcu_is_watching+0xe/0xb0 kernel/rcu/tree.c:1138 rcu_read_lock_held_common kernel/rcu/update.c:108 [inline] rcu_read_lock_sched_held+0x1c/0x70 kernel/rcu/update.c:123 trace_lock_release include/trace/events/lock.h:58 [inline] lock_release+0x522/0x720 kernel/locking/lockdep.c:5652 __raw_spin_unlock include/linux/spinlock_api_smp.h:141 [inline] _raw_spin_unlock+0x12/0x40 kernel/locking/spinlock.c:186 spin_unlock include/linux/spinlock.h:389 [inline] advance_sched+0x37a/0x920 net/sched/sch_taprio.c:763 __run_hrtimer kernel/time/hrtimer.c:1685 [inline] __hrtimer_run_queues+0x4d7/0xb00 kernel/time/hrtimer.c:1749 hrtimer_interrupt+0x2f5/0x780 kernel/time/hrtimer.c:1811 local_apic_timer_interrupt arch/x86/kernel/apic/apic.c:1086 [inline] __sysvec_apic_timer_interrupt+0x146/0x530 arch/x86/kernel/apic/apic.c:1103 sysvec_apic_timer_interrupt+0x8e/0xc0 arch/x86/kernel/apic/apic.c:1097 asm_sysvec_apic_timer_interrupt+0x12/0x20 arch/x86/include/asm/idtentry.h:645 RIP: 0010:console_unlock+0x5ee/0xa50 kernel/printk/printk.c:2779 Code: 58 1e fe ff e8 73 2b 00 00 48 83 3c 24 00 0f 85 69 02 00 00 9c 58 f6 c4 02 0f 85 5c 03 00 00 48 83 3c 24 00 74 01 fb 45 85 e4 <0f> 85 b0 02 00 00 49 c7 c4 a0 6e c2 8a be 04 00 00 00 48 c7 c7 a0 RSP: 0018:ffffc90004a66758 EFLAGS: 00000246 RAX: 0000000000000002 RBX: dffffc0000000000 RCX: 1ffffffff1de43f6 RDX: 0000000000000000 RSI: ffffffff88eb8f00 RDI: ffffffff89429160 RBP: ffffc90004a667b0 R08: 0000000000000001 R09: ffffffff8ef21847 R10: 0000000000000001 R11: 0000000000000001 R12: 0000000000000000 R13: ffffffff8ba0cd88 R14: ffffffff8ba0cd50 R15: 0000000000000000 vprintk_emit+0xa8/0x3c0 kernel/printk/printk.c:2272 dev_vprintk_emit+0x2c9/0x30d drivers/base/core.c:4604 dev_printk_emit+0x9d/0xce drivers/base/core.c:4615 __netdev_printk+0x19c/0x23b net/core/dev.c:11117 netdev_info+0xc8/0xf6 net/core/dev.c:11172 nsim_udp_tunnel_set_port.cold+0xa6/0xe9 drivers/net/netdevsim/udp_tunnels.c:34 udp_tunnel_nic_device_sync_one net/ipv4/udp_tunnel_nic.c:223 [inline] udp_tunnel_nic_device_sync_by_port net/ipv4/udp_tunnel_nic.c:246 [inline] __udp_tunnel_nic_device_sync.part.0+0x6bb/0xc00 net/ipv4/udp_tunnel_nic.c:289 __udp_tunnel_nic_device_sync net/ipv4/udp_tunnel_nic.c:283 [inline] udp_tunnel_nic_device_sync+0xd3/0x130 net/ipv4/udp_tunnel_nic.c:312 udp_tunnel_nic_add_port include/net/udp_tunnel.h:332 [inline] udp_tunnel_nic_add_port include/net/udp_tunnel.h:327 [inline] udp_tunnel_notify_add_rx_port+0x1dd/0x340 net/ipv4/udp_tunnel_core.c:126 geneve_socket_create drivers/net/geneve.c:601 [inline] geneve_sock_add+0x77c/0xc50 drivers/net/geneve.c:676 geneve_open+0xb6/0xe0 drivers/net/geneve.c:718 __dev_open+0x232/0x410 net/core/dev.c:1425 __dev_change_flags+0x44c/0x650 net/core/dev.c:8433 dev_change_flags+0x86/0x150 net/core/dev.c:8504 do_setlink+0x81b/0x2da0 net/core/rtnetlink.c:2731 __rtnl_newlink+0xb05/0x13f0 net/core/rtnetlink.c:3416 rtnl_newlink+0x5a/0x90 net/core/rtnetlink.c:3531 rtnetlink_rcv_msg+0x31d/0x8d0 net/core/rtnetlink.c:5990 netlink_rcv_skb+0x118/0x370 net/netlink/af_netlink.c:2496 netlink_unicast_kernel net/netlink/af_netlink.c:1319 [inline] netlink_unicast+0x433/0x710 net/netlink/af_netlink.c:1345 netlink_sendmsg+0x770/0xc20 net/netlink/af_netlink.c:1921 sock_sendmsg_nosec net/socket.c:705 [inline] sock_sendmsg+0xab/0xe0 net/socket.c:725 __sys_sendto+0x19e/0x270 net/socket.c:2040 __do_sys_sendto net/socket.c:2052 [inline] __se_sys_sendto net/socket.c:2048 [inline] __x64_sys_sendto+0xd8/0x1b0 net/socket.c:2048 do_syscall_x64 arch/x86/entry/common.c:50 [inline] do_syscall_64+0x35/0x80 arch/x86/entry/common.c:80 entry_SYSCALL_64_after_hwframe+0x44/0xae RIP: 0033:0x7ff39943a69c Code: fa fa ff ff 44 8b 4c 24 2c 4c 8b 44 24 20 89 c5 44 8b 54 24 28 48 8b 54 24 18 b8 2c 00 00 00 48 8b 74 24 10 8b 7c 24 08 0f 05 <48> 3d 00 f0 ff ff 77 34 89 ef 48 89 44 24 08 e8 20 fb ff ff 48 8b RSP: 002b:00007ffea68570c0 EFLAGS: 00000293 ORIG_RAX: 000000000000002c RAX: ffffffffffffffda RBX: 00007ff39a4c4320 RCX: 00007ff39943a69c RDX: 000000000000002c RSI: 00007ff39a4c4370 RDI: 0000000000000003 RBP: 0000000000000000 R08: 00007ffea6857114 R09: 000000000000000c R10: 0000000000000000 R11: 0000000000000293 R12: 0000000000000000 R13: 00007ff39a4c4370 R14: 0000000000000003 R15: 0000000000000000 ---------------- Code disassembly (best guess): 0: 90 nop 1: 48 c7 c7 00 f8 ec 88 mov $0xffffffff88ecf800,%rdi 8: e8 84 74 34 07 callq 0x7347491 d: 65 8a 05 11 bc a9 7e mov %gs:0x7ea9bc11(%rip),%al # 0x7ea9bc25 14: 0f be c0 movsbl %al,%eax 17: c3 retq 18: 66 0f 1f 84 00 00 00 nopw 0x0(%rax,%rax,1) 1f: 00 00 21: 55 push %rbp 22: 53 push %rbx 23: 65 ff 05 77 7c a8 7e incl %gs:0x7ea87c77(%rip) # 0x7ea87ca1 * 2a: e8 42 74 34 07 callq 0x7347471 <-- trapping instruction 2f: 48 c7 c3 10 af 03 00 mov $0x3af10,%rbx 36: 83 f8 07 cmp $0x7,%eax 39: 89 c5 mov %eax,%ebp 3b: 77 77 ja 0xb4 3d: 48 rex.W 3e: 8d .byte 0x8d 3f: 3c .byte 0x3c