bisecting cause commit starting from 088fb7eff3496e0f61fdf68bda89b81a4d0a4434 building syzkaller on 1fa34c1b4ca31728acc7dfc7ec2f221443b8d40f testing commit 088fb7eff3496e0f61fdf68bda89b81a4d0a4434 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 5c735dd759f94cac7a60b3a3afc7db547e433d07764bcaa364045d388da340c3 run #0: crashed: general protection fault in btrfs_stop_all_workers run #1: crashed: general protection fault in btrfs_stop_all_workers run #2: crashed: general protection fault in btrfs_stop_all_workers run #3: crashed: general protection fault in btrfs_stop_all_workers run #4: crashed: general protection fault in btrfs_stop_all_workers run #5: crashed: general protection fault in btrfs_stop_all_workers run #6: crashed: general protection fault in btrfs_stop_all_workers run #7: crashed: general protection fault in btrfs_stop_all_workers run #8: crashed: general protection fault in btrfs_stop_all_workers run #9: crashed: general protection fault in btrfs_stop_all_workers run #10: crashed: general protection fault in btrfs_stop_all_workers run #11: crashed: general protection fault in btrfs_stop_all_workers run #12: crashed: general protection fault in btrfs_stop_all_workers run #13: crashed: general protection fault in btrfs_stop_all_workers run #14: crashed: general protection fault in btrfs_stop_all_workers run #15: crashed: general protection fault in btrfs_stop_all_workers run #16: crashed: general protection fault in btrfs_stop_all_workers run #17: crashed: general protection fault in btrfs_stop_all_workers run #18: crashed: general protection fault in btrfs_stop_all_workers run #19: basic kernel testing failed: failed to copy test binary to VM: timedout after 1m0s ["scp" "-P" "22" "-F" "/dev/null" "-o" "UserKnownHostsFile=/dev/null" "-o" "BatchMode=yes" "-o" "IdentitiesOnly=yes" "-o" "StrictHostKeyChecking=no" "-o" "ConnectTimeout=10" "/tmp/syz-executor3674422506" "root@10.128.10.53:./syz-executor3674422506"] Warning: Permanently added '10.128.10.53' (ECDSA) to the list of known hosts. testing release v5.17 testing commit f443e374ae131c168a065ea1748feac6b2e76613 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: d1c4f560f98c3f62708e8b2f9d5d1458073ad2c2bea7b501d81ffd45587db4d1 all runs: OK # git bisect start 088fb7eff3496e0f61fdf68bda89b81a4d0a4434 f443e374ae131c168a065ea1748feac6b2e76613 Bisecting: 11068 revisions left to test after this (roughly 14 steps) [5627ecb8374a00163d90bc92c33f829ac27895b2] Merge branch 'i2c/for-mergewindow' of git://git.kernel.org/pub/scm/linux/kernel/git/wsa/linux testing commit 5627ecb8374a00163d90bc92c33f829ac27895b2 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 6898a29f06e10187fb22222ae07740fa7f9462daa6aa43b4824954d98fa7fda7 run #0: basic kernel testing failed: BUG: program execution failed: executor NUM: exit status NUM run #1: OK run #2: OK run #3: OK run #4: OK run #5: OK run #6: OK run #7: OK run #8: OK run #9: OK # git bisect good 5627ecb8374a00163d90bc92c33f829ac27895b2 Bisecting: 5520 revisions left to test after this (roughly 13 steps) [e99ab448b039c2c5fcc8f573dcc0fe942bc0de53] Merge branch 'clk-next' of git://git.kernel.org/pub/scm/linux/kernel/git/clk/linux.git testing commit e99ab448b039c2c5fcc8f573dcc0fe942bc0de53 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 121def4b7054e744d4e0baeee5fdb2f36a80ac8a5e7a1a6496d74472fa1dd1a0 all runs: OK # git bisect good e99ab448b039c2c5fcc8f573dcc0fe942bc0de53 Bisecting: 2797 revisions left to test after this (roughly 12 steps) [2c7fa07f0a34941ca04b188050e3509188e39cbd] Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git testing commit 2c7fa07f0a34941ca04b188050e3509188e39cbd compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 88d827e940d6a26bab034c2bb30683170df2fdaeaebd62330f6da50c0b7538a1 all runs: crashed: general protection fault in btrfs_stop_all_workers # git bisect bad 2c7fa07f0a34941ca04b188050e3509188e39cbd Bisecting: 1258 revisions left to test after this (roughly 10 steps) [2d1b9276bf742b5a18aed4c2000ad723c035b7fa] Merge branch 'master' of git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git testing commit 2d1b9276bf742b5a18aed4c2000ad723c035b7fa compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 0310145ca4b481dcd6219a411855fbb23e7f59b5b5a78a0462d500651b7020c0 run #0: basic kernel testing failed: BUG: program execution failed: executor NUM: exit status NUM run #1: crashed: general protection fault in btrfs_stop_all_workers run #2: crashed: general protection fault in btrfs_stop_all_workers run #3: crashed: general protection fault in btrfs_stop_all_workers run #4: crashed: general protection fault in btrfs_stop_all_workers run #5: crashed: general protection fault in btrfs_stop_all_workers run #6: crashed: general protection fault in btrfs_stop_all_workers run #7: crashed: general protection fault in btrfs_stop_all_workers run #8: crashed: general protection fault in btrfs_stop_all_workers run #9: crashed: general protection fault in btrfs_stop_all_workers # git bisect bad 2d1b9276bf742b5a18aed4c2000ad723c035b7fa Bisecting: 734 revisions left to test after this (roughly 10 steps) [70139a2ee2265cf85ea379eb1ef325a4a1852d71] Merge branch 'master' of git://linuxtv.org/mchehab/media-next.git testing commit 70139a2ee2265cf85ea379eb1ef325a4a1852d71 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: ec1870a6d38eee7a3ca92dd5eb0f73a7614517cb524ad612ad3667f0a0e5e4f6 all runs: crashed: general protection fault in btrfs_stop_all_workers # git bisect bad 70139a2ee2265cf85ea379eb1ef325a4a1852d71 Bisecting: 383 revisions left to test after this (roughly 9 steps) [a756543dd501bb93b0f803856a0197738eafbea6] Merge branch 'master' of https://github.com/Paragon-Software-Group/linux-ntfs3.git testing commit a756543dd501bb93b0f803856a0197738eafbea6 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 8d4373a2b53a07c316f07914b28d8ec940cb91d68dbf228747edebe20d085ea8 all runs: crashed: general protection fault in btrfs_stop_all_workers # git bisect bad a756543dd501bb93b0f803856a0197738eafbea6 Bisecting: 185 revisions left to test after this (roughly 8 steps) [4972fdf152faae05b9bc7570f0ba0ae22b93e621] Merge branch 'for-next-next-v5.18-20220422' into for-next-20220422 testing commit 4972fdf152faae05b9bc7570f0ba0ae22b93e621 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 25a023c8b3fcea5479581e72941fc30e6acb844ad444563febcb66e50124525c all runs: crashed: general protection fault in btrfs_stop_all_workers # git bisect bad 4972fdf152faae05b9bc7570f0ba0ae22b93e621 Bisecting: 79 revisions left to test after this (roughly 6 steps) [c6a9fd027ed7addb3cd549f745c17641e6968145] btrfs: check-integrity: simplify bio allocation in btrfsic_read_block testing commit c6a9fd027ed7addb3cd549f745c17641e6968145 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: cfcfbce53d5a67712d98e831ba8020b5b9bea7848bf4bc8b46679be01567fbf0 all runs: OK # git bisect good c6a9fd027ed7addb3cd549f745c17641e6968145 Bisecting: 39 revisions left to test after this (roughly 5 steps) [a9b8e8416ca7fb60cd8ab1e955c59bad8eab1cc4] btrfs: skip compression property for anything other than files and dirs testing commit a9b8e8416ca7fb60cd8ab1e955c59bad8eab1cc4 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: c4223e671781f9ab805eb7a3e4ce430189ec3553fd02ee0ee97c4f8e50f62938 all runs: OK # git bisect good a9b8e8416ca7fb60cd8ab1e955c59bad8eab1cc4 Bisecting: 19 revisions left to test after this (roughly 4 steps) [9d7bd4c387c1b4776832cf8c895ab934efaa2f08] btrfs: use a normal workqueue for rmw_workers testing commit 9d7bd4c387c1b4776832cf8c895ab934efaa2f08 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 006cbeb51b386f701b80a233fd72fe007a096f1223c0161e9840960c1e4b6634 all runs: crashed: general protection fault in btrfs_stop_all_workers # git bisect bad 9d7bd4c387c1b4776832cf8c895ab934efaa2f08 Bisecting: 9 revisions left to test after this (roughly 3 steps) [24c350d76942c12c19c17fecaf23164520009d05] btrfs: raid56: make finish_rmw() subpage compatible testing commit 24c350d76942c12c19c17fecaf23164520009d05 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 601dd6639563d46df14d9fd56c4425489b913c2bc00e5fc0431842535adcfcc3 all runs: OK # git bisect good 24c350d76942c12c19c17fecaf23164520009d05 Bisecting: 4 revisions left to test after this (roughly 2 steps) [6f2c4044cb0f0a473861eb38a07860ff6d292822] btrfs: raid56: make steal_rbio() subpage compatible testing commit 6f2c4044cb0f0a473861eb38a07860ff6d292822 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 8c4c49860893fd87e7505c3e6d39d860e976e8231ae03cf2e406efad23f25568 all runs: OK # git bisect good 6f2c4044cb0f0a473861eb38a07860ff6d292822 Bisecting: 2 revisions left to test after this (roughly 1 step) [446e59dd6c55d120dc09444a1f086e5a81898665] btrfs: raid56: enable subpage support for RAID56 testing commit 446e59dd6c55d120dc09444a1f086e5a81898665 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 5ca6ec9820ef1574ef0d5e8da2021263b60de1a62aab1de86191dd49490796eb all runs: OK # git bisect good 446e59dd6c55d120dc09444a1f086e5a81898665 Bisecting: 0 revisions left to test after this (roughly 1 step) [b98f9472c6b5eece9cdcb3c69bd3c89a0fcbb16b] btrfs: use normal workqueues for scrub testing commit b98f9472c6b5eece9cdcb3c69bd3c89a0fcbb16b compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 4e32846318acb5425ccc820f06c2b63828fa5d687c10c9f5909731db519bf67a all runs: OK # git bisect good b98f9472c6b5eece9cdcb3c69bd3c89a0fcbb16b 9d7bd4c387c1b4776832cf8c895ab934efaa2f08 is the first bad commit commit 9d7bd4c387c1b4776832cf8c895ab934efaa2f08 Author: Christoph Hellwig Date: Mon Apr 18 06:43:11 2022 +0200 btrfs: use a normal workqueue for rmw_workers rmw_workers doesn't need ordered execution or thread disabling threshold (as the thresh parameter is less than DFT_THRESHOLD). Just switch to the normal workqueues that use a lot less resources, especially in the work_struct vs btrfs_work structures. Reviewed-by: Qu Wenruo Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba fs/btrfs/ctree.h | 2 +- fs/btrfs/disk-io.c | 5 ++--- fs/btrfs/raid56.c | 29 ++++++++++++++--------------- 3 files changed, 17 insertions(+), 19 deletions(-) culprit signature: 006cbeb51b386f701b80a233fd72fe007a096f1223c0161e9840960c1e4b6634 parent signature: 4e32846318acb5425ccc820f06c2b63828fa5d687c10c9f5909731db519bf67a revisions tested: 16, total time: 3h47m14.973908752s (build: 1h37m51.959083235s, test: 2h7m47.902561238s) first bad commit: 9d7bd4c387c1b4776832cf8c895ab934efaa2f08 btrfs: use a normal workqueue for rmw_workers recipients (to): ["dsterba@suse.com" "hch@lst.de" "wqu@suse.com"] recipients (cc): [] crash: general protection fault in btrfs_stop_all_workers general protection fault, probably for non-canonical address 0xdffffc0000000023: 0000 [#1] PREEMPT SMP KASAN KASAN: null-ptr-deref in range [0x0000000000000118-0x000000000000011f] CPU: 0 PID: 4546 Comm: syz-executor.4 Not tainted 5.18.0-rc3-syzkaller #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 RIP: 0010:workqueue_sysfs_unregister kernel/workqueue.c:5774 [inline] RIP: 0010:destroy_workqueue+0x29/0x720 kernel/workqueue.c:4408 Code: 90 48 b8 00 00 00 00 00 fc ff df 41 57 41 56 41 55 41 54 49 89 fc 48 81 c7 18 01 00 00 48 89 fa 55 48 c1 ea 03 53 48 83 ec 08 <80> 3c 02 00 0f 85 68 06 00 00 49 8b 84 24 18 01 00 00 48 85 c0 74 RSP: 0018:ffffc90005dcf848 EFLAGS: 00010296 RAX: dffffc0000000000 RBX: ffff888078158000 RCX: 1ffffffff1e3861e RDX: 0000000000000023 RSI: ffffffff88eba320 RDI: 0000000000000118 RBP: ffff888078158000 R08: 0000000000000000 R09: ffffffff8f149a47 R10: 0000000000000001 R11: 0000000000000001 R12: 0000000000000000 R13: ffff888078158fc8 R14: 0000000000001000 R15: 00000000fffffff4 FS: 00007f9cc4e16700(0000) GS:ffff8880b9e00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 00007fc078b9c028 CR3: 00000000681b3000 CR4: 00000000003506f0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 Call Trace: btrfs_stop_all_workers+0x146/0x2ea fs/btrfs/disk-io.c:2286 open_ctree+0x3e68/0x3f1f fs/btrfs/disk-io.c:3959 btrfs_fill_super fs/btrfs/super.c:1430 [inline] btrfs_mount_root.cold+0xe/0x118 fs/btrfs/super.c:1796 legacy_get_tree+0xfa/0x1f0 fs/fs_context.c:610 vfs_get_tree+0x7f/0x2c0 fs/super.c:1497 fc_mount fs/namespace.c:1043 [inline] vfs_kern_mount.part.0+0x70/0x100 fs/namespace.c:1073 btrfs_mount+0x1aa/0x860 fs/btrfs/super.c:1856 legacy_get_tree+0xfa/0x1f0 fs/fs_context.c:610 vfs_get_tree+0x7f/0x2c0 fs/super.c:1497 do_new_mount fs/namespace.c:3040 [inline] path_mount+0x7e8/0x1a40 fs/namespace.c:3370 do_mount fs/namespace.c:3383 [inline] __do_sys_mount fs/namespace.c:3591 [inline] __se_sys_mount fs/namespace.c:3568 [inline] __x64_sys_mount+0x1f5/0x260 fs/namespace.c:3568 do_syscall_x64 arch/x86/entry/common.c:50 [inline] do_syscall_64+0x35/0xb0 arch/x86/entry/common.c:80 entry_SYSCALL_64_after_hwframe+0x44/0xae RIP: 0033:0x7f9cc3c8a61a Code: 48 c7 c2 b8 ff ff ff f7 d8 64 89 02 b8 ff ff ff ff eb d2 e8 b8 04 00 00 0f 1f 84 00 00 00 00 00 49 89 ca b8 a5 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 c7 c1 b8 ff ff ff f7 d8 64 89 01 48 RSP: 002b:00007f9cc4e15f88 EFLAGS: 00000246 ORIG_RAX: 00000000000000a5 RAX: ffffffffffffffda RBX: 0000000020000200 RCX: 00007f9cc3c8a61a RDX: 0000000020000000 RSI: 0000000020000100 RDI: 00007f9cc4e15fe0 RBP: 00007f9cc4e16020 R08: 00007f9cc4e16020 R09: 0000000020000000 R10: 0000000000000000 R11: 0000000000000246 R12: 0000000020000000 R13: 0000000020000100 R14: 00007f9cc4e15fe0 R15: 000000002001ee00 Modules linked in: ---[ end trace 0000000000000000 ]--- RIP: 0010:workqueue_sysfs_unregister kernel/workqueue.c:5774 [inline] RIP: 0010:destroy_workqueue+0x29/0x720 kernel/workqueue.c:4408 Code: 90 48 b8 00 00 00 00 00 fc ff df 41 57 41 56 41 55 41 54 49 89 fc 48 81 c7 18 01 00 00 48 89 fa 55 48 c1 ea 03 53 48 83 ec 08 <80> 3c 02 00 0f 85 68 06 00 00 49 8b 84 24 18 01 00 00 48 85 c0 74 RSP: 0018:ffffc90005dcf848 EFLAGS: 00010296 RAX: dffffc0000000000 RBX: ffff888078158000 RCX: 1ffffffff1e3861e RDX: 0000000000000023 RSI: ffffffff88eba320 RDI: 0000000000000118 RBP: ffff888078158000 R08: 0000000000000000 R09: ffffffff8f149a47 R10: 0000000000000001 R11: 0000000000000001 R12: 0000000000000000 R13: ffff888078158fc8 R14: 0000000000001000 R15: 00000000fffffff4 FS: 00007f9cc4e16700(0000) GS:ffff8880b9f00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 00007f5b5c59a378 CR3: 00000000681b3000 CR4: 00000000003506e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 ---------------- Code disassembly (best guess): 0: 90 nop 1: 48 b8 00 00 00 00 00 movabs $0xdffffc0000000000,%rax 8: fc ff df b: 41 57 push %r15 d: 41 56 push %r14 f: 41 55 push %r13 11: 41 54 push %r12 13: 49 89 fc mov %rdi,%r12 16: 48 81 c7 18 01 00 00 add $0x118,%rdi 1d: 48 89 fa mov %rdi,%rdx 20: 55 push %rbp 21: 48 c1 ea 03 shr $0x3,%rdx 25: 53 push %rbx 26: 48 83 ec 08 sub $0x8,%rsp * 2a: 80 3c 02 00 cmpb $0x0,(%rdx,%rax,1) <-- trapping instruction 2e: 0f 85 68 06 00 00 jne 0x69c 34: 49 8b 84 24 18 01 00 mov 0x118(%r12),%rax 3b: 00 3c: 48 85 c0 test %rax,%rax 3f: 74 .byte 0x74