bisecting cause commit starting from 7ddd09fc4b745fb1d8942f95389583e08412e0cd building syzkaller on bc5869180f69e2ad6c6b823e129e08a8e523d800 testing commit 7ddd09fc4b745fb1d8942f95389583e08412e0cd with gcc (GCC) 8.1.0 kernel signature: a2d52519633101a72e261505a54bb32413bc53f2 run #0: crashed: WARNING in percpu_ref_exit run #1: crashed: WARNING in percpu_ref_exit run #2: crashed: WARNING in percpu_ref_exit run #3: crashed: WARNING in percpu_ref_exit run #4: crashed: WARNING: ODEBUG bug in io_sqe_files_unregister run #5: crashed: WARNING in percpu_ref_exit run #6: crashed: WARNING in percpu_ref_exit run #7: crashed: WARNING in percpu_ref_exit run #8: crashed: WARNING in percpu_ref_exit run #9: crashed: WARNING in percpu_ref_exit testing release v5.4 testing commit 219d54332a09e8d8741c1e1982f5eae56099de85 with gcc (GCC) 8.1.0 kernel signature: e2b40eed2e2a4ea2fab597eac1d481789ca0a3be all runs: OK # git bisect start 7ddd09fc4b745fb1d8942f95389583e08412e0cd 219d54332a09e8d8741c1e1982f5eae56099de85 Bisecting: 9519 revisions left to test after this (roughly 13 steps) [ddebe839c6013ab42f376bdeaaaf66bd0c0d68d6] Merge tag 'clk-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/clk/linux testing commit ddebe839c6013ab42f376bdeaaaf66bd0c0d68d6 with gcc (GCC) 8.1.0 kernel signature: d70242c156483012c53362cd1abef63ab2a819ff all runs: OK # git bisect good ddebe839c6013ab42f376bdeaaaf66bd0c0d68d6 Bisecting: 4770 revisions left to test after this (roughly 12 steps) [9bd7eaa8e6e729307781b9f19cf514a2977c7832] Merge remote-tracking branch 'kbuild/for-next' testing commit 9bd7eaa8e6e729307781b9f19cf514a2977c7832 with gcc (GCC) 8.1.0 kernel signature: 15b1a0bf9208afdc4b3291f393fef6670afa2cba all runs: OK # git bisect good 9bd7eaa8e6e729307781b9f19cf514a2977c7832 Bisecting: 2154 revisions left to test after this (roughly 11 steps) [58d1cf12662cdc07057a90ae6b9ecf7f1409b0f9] Merge remote-tracking branch 'drm/drm-next' testing commit 58d1cf12662cdc07057a90ae6b9ecf7f1409b0f9 with gcc (GCC) 8.1.0 kernel signature: bb067745eb1362a267a0ac9bb98b3e9f64bdeb12 all runs: OK # git bisect good 58d1cf12662cdc07057a90ae6b9ecf7f1409b0f9 Bisecting: 1077 revisions left to test after this (roughly 10 steps) [a6874cba62db51cc00ecea752b6fc6a799498730] Merge remote-tracking branch 'battery/for-next' testing commit a6874cba62db51cc00ecea752b6fc6a799498730 with gcc (GCC) 8.1.0 kernel signature: d5bd6c3ce96cff8c53af203677854e3954b4c839 run #0: crashed: WARNING in percpu_ref_exit run #1: crashed: WARNING in percpu_ref_exit run #2: crashed: WARNING in percpu_ref_exit run #3: crashed: WARNING: ODEBUG bug in io_sqe_files_unregister run #4: crashed: WARNING in percpu_ref_exit run #5: crashed: WARNING in percpu_ref_exit run #6: crashed: WARNING: ODEBUG bug in io_sqe_files_unregister run #7: crashed: WARNING in percpu_ref_exit run #8: crashed: WARNING in percpu_ref_exit run #9: crashed: WARNING in percpu_ref_exit # git bisect bad a6874cba62db51cc00ecea752b6fc6a799498730 Bisecting: 512 revisions left to test after this (roughly 9 steps) [dadf05112e514305c7bf901f9d15ecf3bc54a465] Merge remote-tracking branch 'drm-intel/for-linux-next' testing commit dadf05112e514305c7bf901f9d15ecf3bc54a465 with gcc (GCC) 8.1.0 kernel signature: 2a26d6eb92852adeeaa8e4acdb2d2c14423cb50c all runs: OK # git bisect good dadf05112e514305c7bf901f9d15ecf3bc54a465 Bisecting: 276 revisions left to test after this (roughly 8 steps) [76fc7298494407ab00cffbd9466dd739aead864f] Merge remote-tracking branch 'sound/for-next' testing commit 76fc7298494407ab00cffbd9466dd739aead864f with gcc (GCC) 8.1.0 kernel signature: c5f01ef39cbf056cab2e141dc9158bf944c6f671 all runs: OK # git bisect good 76fc7298494407ab00cffbd9466dd739aead864f Bisecting: 138 revisions left to test after this (roughly 7 steps) [0f501c7cde4086d15c396a95c59631b05dbc0351] ASoC: SOF: move arch_ops under ops testing commit 0f501c7cde4086d15c396a95c59631b05dbc0351 with gcc (GCC) 8.1.0 kernel signature: 3039d7f41d5c29d85d6fcdec946852bed672324b all runs: OK # git bisect good 0f501c7cde4086d15c396a95c59631b05dbc0351 Bisecting: 77 revisions left to test after this (roughly 6 steps) [6b70fabbcbd15005b550c0778d941ef98d34647e] Merge remote-tracking branch 'pcmcia/pcmcia-next' testing commit 6b70fabbcbd15005b550c0778d941ef98d34647e with gcc (GCC) 8.1.0 kernel signature: 4b381396fe21d585c5d76fbfee608cbd121152d3 run #0: crashed: WARNING in percpu_ref_exit run #1: crashed: WARNING: ODEBUG bug in io_sqe_files_unregister run #2: crashed: WARNING: ODEBUG bug in io_sqe_files_unregister run #3: crashed: WARNING in percpu_ref_exit run #4: crashed: WARNING in percpu_ref_exit run #5: crashed: WARNING in percpu_ref_exit run #6: crashed: WARNING: ODEBUG bug in io_sqe_files_unregister run #7: crashed: WARNING in percpu_ref_exit run #8: crashed: WARNING in percpu_ref_exit run #9: crashed: WARNING: ODEBUG bug in io_sqe_files_unregister # git bisect bad 6b70fabbcbd15005b550c0778d941ef98d34647e Bisecting: 30 revisions left to test after this (roughly 5 steps) [7fa07c60d3b014cff1b931ba979c3aec4e9052e6] io_uring: remove two unnecessary function declarations testing commit 7fa07c60d3b014cff1b931ba979c3aec4e9052e6 with gcc (GCC) 8.1.0 kernel signature: 13dabf8caf755cdffdc34e281861f1f5d9821924 run #0: crashed: WARNING in percpu_ref_exit run #1: crashed: WARNING in percpu_ref_exit run #2: crashed: WARNING in percpu_ref_exit run #3: crashed: WARNING in percpu_ref_exit run #4: crashed: WARNING in percpu_ref_exit run #5: crashed: WARNING in percpu_ref_exit run #6: crashed: WARNING: ODEBUG bug in io_sqe_files_unregister run #7: crashed: WARNING in percpu_ref_exit run #8: crashed: WARNING in percpu_ref_exit run #9: crashed: WARNING in percpu_ref_exit # git bisect bad 7fa07c60d3b014cff1b931ba979c3aec4e9052e6 Bisecting: 14 revisions left to test after this (roughly 4 steps) [2c28d602400c7bef103a5ffc0d10608099f695dc] io_uring: add support for fallocate() testing commit 2c28d602400c7bef103a5ffc0d10608099f695dc with gcc (GCC) 8.1.0 kernel signature: 804e82a9e9859ecd9235994a3bbaea7d4a1a116a all runs: OK # git bisect good 2c28d602400c7bef103a5ffc0d10608099f695dc Bisecting: 7 revisions left to test after this (roughly 3 steps) [0325bbcd2dd2d303d6e8f78edeefdab3a39f38ff] io_uring: add support for IORING_OP_CLOSE testing commit 0325bbcd2dd2d303d6e8f78edeefdab3a39f38ff with gcc (GCC) 8.1.0 kernel signature: f9e42dc406bba1b3b474727a2b8efe787f8d008b all runs: OK # git bisect good 0325bbcd2dd2d303d6e8f78edeefdab3a39f38ff Bisecting: 3 revisions left to test after this (roughly 2 steps) [7420542ba233956799f78caf70d5b3e3d818fe12] io-wq: support concurrent non-blocking work testing commit 7420542ba233956799f78caf70d5b3e3d818fe12 with gcc (GCC) 8.1.0 kernel signature: bc40b78384a6b1ec896dd8e68490f0c61999a264 run #0: crashed: WARNING in percpu_ref_exit run #1: crashed: WARNING in percpu_ref_exit run #2: crashed: WARNING in percpu_ref_exit run #3: crashed: WARNING in percpu_ref_exit run #4: crashed: WARNING in percpu_ref_exit run #5: crashed: WARNING: ODEBUG bug in io_sqe_files_unregister run #6: crashed: WARNING in percpu_ref_exit run #7: crashed: WARNING in percpu_ref_exit run #8: crashed: WARNING: ODEBUG bug in io_sqe_files_unregister run #9: crashed: WARNING in percpu_ref_exit # git bisect bad 7420542ba233956799f78caf70d5b3e3d818fe12 Bisecting: 1 revision left to test after this (roughly 1 step) [03448cdee0278f77adca6018d8754f244b050ee7] fs: make two stat prep helpers available testing commit 03448cdee0278f77adca6018d8754f244b050ee7 with gcc (GCC) 8.1.0 kernel signature: 6371308fade2717d7715d2fb3e15bdb2d4189bae all runs: crashed: WARNING in percpu_ref_exit # git bisect bad 03448cdee0278f77adca6018d8754f244b050ee7 Bisecting: 0 revisions left to test after this (roughly 0 steps) [cbb537634780172137459dead490d668d437ef4d] io_uring: avoid ring quiesce for fixed file set unregister and update testing commit cbb537634780172137459dead490d668d437ef4d with gcc (GCC) 8.1.0 kernel signature: 4e30868b008b9439dc0c0bd020fc134583d5af81 all runs: crashed: WARNING in percpu_ref_exit # git bisect bad cbb537634780172137459dead490d668d437ef4d cbb537634780172137459dead490d668d437ef4d is the first bad commit commit cbb537634780172137459dead490d668d437ef4d Author: Jens Axboe Date: Mon Dec 9 11:22:50 2019 -0700 io_uring: avoid ring quiesce for fixed file set unregister and update We currently fully quiesce the ring before an unregister or update of the fixed fileset. This is very expensive, and we can be a bit smarter about this. Add a percpu refcount for the file tables as a whole. Grab a percpu ref when we use a registered file, and put it on completion. This is cheap to do. Upon removal of a file from a set, switch the ref count to atomic mode. When we hit zero ref on the completion side, then we know we can drop the previously registered files. When the old files have been dropped, switch the ref back to percpu mode for normal operation. Since there's a period between doing the update and the kernel being done with it, add a IORING_OP_FILES_UPDATE opcode that can perform the same action. The application knows the update has completed when it gets the CQE for it. Between doing the update and receiving this completion, the application must continue to use the unregistered fd if submitting IO on this particular file. This takes the runtime of test/file-register from liburing from 14s to about 0.7s. Signed-off-by: Jens Axboe fs/io_uring.c | 406 ++++++++++++++++++++++++++++++------------ include/uapi/linux/io_uring.h | 1 + 2 files changed, 298 insertions(+), 109 deletions(-) culprit signature: 4e30868b008b9439dc0c0bd020fc134583d5af81 parent signature: f9e42dc406bba1b3b474727a2b8efe787f8d008b revisions tested: 16, total time: 3h53m39.035427452s (build: 1h35m15.417961845s, test: 2h16m45.165373282s) first bad commit: cbb537634780172137459dead490d668d437ef4d io_uring: avoid ring quiesce for fixed file set unregister and update cc: ["axboe@kernel.dk" "linux-block@vger.kernel.org" "linux-fsdevel@vger.kernel.org" "linux-kernel@vger.kernel.org" "viro@zeniv.linux.org.uk"] crash: WARNING in percpu_ref_exit ------------[ cut here ]------------ WARNING: CPU: 1 PID: 8273 at lib/percpu-refcount.c:111 percpu_ref_exit+0x81/0xa0 lib/percpu-refcount.c:115 Kernel panic - not syncing: panic_on_warn set ... CPU: 1 PID: 8273 Comm: syz-executor.1 Not tainted 5.5.0-rc1-syzkaller #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x12d/0x187 lib/dump_stack.c:118 panic+0x22a/0x4e3 kernel/panic.c:221 __warn.cold.10+0x25/0x2a kernel/panic.c:582 report_bug+0x1b0/0x270 lib/bug.c:195 fixup_bug arch/x86/kernel/traps.c:174 [inline] do_error_trap+0x11b/0x200 arch/x86/kernel/traps.c:267 do_invalid_op+0x36/0x40 arch/x86/kernel/traps.c:286 invalid_op+0x23/0x30 arch/x86/entry/entry_64.S:1027 RIP: 0010:percpu_ref_exit+0x81/0xa0 lib/percpu-refcount.c:111 Code: 65 16 43 fe 4c 89 ea 48 b8 00 00 00 00 00 fc ff df 48 c1 ea 03 80 3c 02 00 75 24 48 c7 43 08 03 00 00 00 5b 41 5c 41 5d 5d c3 <0f> 0b eb ce e8 a6 7b 54 fe eb c0 4c 89 ef e8 9c 7b 54 fe eb 91 4c RSP: 0018:ffffc90003327d20 EFLAGS: 00010282 RAX: dffffc0000000000 RBX: ffff888096385c10 RCX: 0000000000000000 RDX: 1ffff11012c70b85 RSI: 0000000000000006 RDI: ffff888096385c28 RBP: ffffc90003327d38 R08: 0000000000000006 R09: 0000000000000000 R10: 0000000000000000 R11: 0000000000000000 R12: 0000607f5129c028 R13: ffff888096385c18 R14: ffff888096fd6800 R15: ffff888096fd6b90 io_sqe_files_unregister+0x71/0x290 fs/io_uring.c:4404 io_ring_ctx_free fs/io_uring.c:5356 [inline] io_ring_ctx_wait_and_kill+0x395/0x8d0 fs/io_uring.c:5425 io_uring_release+0x3d/0x50 fs/io_uring.c:5433 __fput+0x25a/0x770 fs/file_table.c:280 ____fput+0x9/0x10 fs/file_table.c:313 task_work_run+0x108/0x180 kernel/task_work.c:113 tracehook_notify_resume include/linux/tracehook.h:188 [inline] exit_to_usermode_loop+0x24e/0x2e0 arch/x86/entry/common.c:164 prepare_exit_to_usermode arch/x86/entry/common.c:195 [inline] syscall_return_slowpath arch/x86/entry/common.c:278 [inline] do_syscall_64+0x4ff/0x5f0 arch/x86/entry/common.c:304 entry_SYSCALL_64_after_hwframe+0x49/0xbe RIP: 0033:0x4144b1 Code: 75 14 b8 03 00 00 00 0f 05 48 3d 01 f0 ff ff 0f 83 04 1b 00 00 c3 48 83 ec 08 e8 0a fc ff ff 48 89 04 24 b8 03 00 00 00 0f 05 <48> 8b 3c 24 48 89 c2 e8 53 fc ff ff 48 89 d0 48 83 c4 08 48 3d 01 RSP: 002b:00007fffdc895380 EFLAGS: 00000293 ORIG_RAX: 0000000000000003 RAX: 0000000000000000 RBX: 0000000000000004 RCX: 00000000004144b1 RDX: 0000001b33b20000 RSI: 0000000000000000 RDI: 0000000000000003 RBP: 0000000000000001 R08: ffffffffffffffff R09: ffffffffffffffff R10: 00007fffdc895460 R11: 0000000000000293 R12: 000000000075bf20 R13: 000000000002d1a8 R14: 00000000007606d0 R15: 000000000075bf2c Kernel Offset: disabled Rebooting in 86400 seconds..