bisecting cause commit starting from 6b1f86f8e9c7f9de7ca1cb987b2cf25e99b1ae3a building syzkaller on 5ff41e943946a9e71b55566a02c8b1371fc9b8de testing commit 6b1f86f8e9c7f9de7ca1cb987b2cf25e99b1ae3a compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: ab1f4e6fcfb6250b876b9eb580dd0428953288960106b96efa5d205ea4ec9c2e run #0: basic kernel testing failed: BUG: program execution failed: executor NUM: failed to write control pipe: write |NUM: broken pipe run #1: crashed: general protection fault in list_lru_add run #2: crashed: general protection fault in list_lru_add run #3: OK run #4: OK run #5: OK run #6: OK run #7: OK run #8: OK run #9: OK run #10: OK run #11: OK run #12: OK run #13: crashed: general protection fault in list_lru_add run #14: OK run #15: OK run #16: OK run #17: OK run #18: OK run #19: OK reproducer seems to be flaky testing release v5.17 testing commit f443e374ae131c168a065ea1748feac6b2e76613 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 2de8517227f83ff72db0209ee7349635ef249fc6caa7d9c9772b6dbe943a27e7 all runs: OK # git bisect start 6b1f86f8e9c7f9de7ca1cb987b2cf25e99b1ae3a f443e374ae131c168a065ea1748feac6b2e76613 Bisecting: 1085 revisions left to test after this (roughly 10 steps) [69d1dea852b54eecd8ad2ec92a7fd371e9aec4bd] Merge tag 'for-5.18/drivers-2022-03-18' of git://git.kernel.dk/linux-block testing commit 69d1dea852b54eecd8ad2ec92a7fd371e9aec4bd compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 475aad93d9ae482e04cacdf4ff15f1408a8700b8621efd6e92c39fc5e5499c57 run #0: basic kernel testing failed: BUG: program execution failed: executor NUM: failed to write control pipe: write |NUM: broken pipe run #1: OK run #2: OK run #3: OK run #4: OK run #5: OK run #6: OK run #7: OK run #8: OK run #9: OK run #10: OK run #11: OK run #12: OK run #13: OK run #14: OK run #15: OK run #16: OK run #17: OK run #18: OK run #19: OK # git bisect good 69d1dea852b54eecd8ad2ec92a7fd371e9aec4bd Bisecting: 510 revisions left to test after this (roughly 9 steps) [5191290407668028179f2544a11ae9b57f0bcf07] Merge tag 'for-5.18-tag' of git://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux testing commit 5191290407668028179f2544a11ae9b57f0bcf07 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 5c141b048407f2fc5040026d663d5154d7a79455635fe095875e629c3c50c254 run #0: basic kernel testing failed: BUG: program execution failed: executor NUM: exit status NUM run #1: OK run #2: OK run #3: OK run #4: OK run #5: OK run #6: OK run #7: OK run #8: OK run #9: OK run #10: OK run #11: OK run #12: OK run #13: OK run #14: OK run #15: OK run #16: OK run #17: OK run #18: OK run #19: OK # git bisect good 5191290407668028179f2544a11ae9b57f0bcf07 Bisecting: 283 revisions left to test after this (roughly 8 steps) [15423a52cc84e23bc11e4a903cd775adc7c6ab00] mm/damon/sysfs: remove repeat container_of() in damon_sysfs_kdamond_release() testing commit 15423a52cc84e23bc11e4a903cd775adc7c6ab00 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 231c91084c11e9e2c2b385d94f1c56c6cc3e3994f47289ed8fbe95cd250a37fc run #0: crashed: general protection fault in list_lru_add run #1: crashed: general protection fault in list_lru_add run #2: crashed: general protection fault in list_lru_add run #3: OK run #4: crashed: general protection fault in list_lru_add run #5: OK run #6: OK run #7: OK run #8: OK run #9: OK run #10: OK run #11: OK run #12: OK run #13: OK run #14: OK run #15: OK run #16: OK run #17: OK run #18: OK run #19: crashed: general protection fault in list_lru_add # git bisect bad 15423a52cc84e23bc11e4a903cd775adc7c6ab00 Bisecting: 113 revisions left to test after this (roughly 7 steps) [a994402bc4714cefea5770b2d906cef5b0f4dc5c] mm/memory-failure.c: catch unexpected -EFAULT from vma_address() testing commit a994402bc4714cefea5770b2d906cef5b0f4dc5c compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: a9090fd87f49b83e221962d2ad7c92c1f5516604b7072f5fc32300f2e0a27ead run #0: crashed: general protection fault in list_lru_add run #1: crashed: general protection fault in list_lru_add run #2: crashed: general protection fault in list_lru_add run #3: crashed: general protection fault in list_lru_add run #4: crashed: general protection fault in list_lru_add run #5: OK run #6: OK run #7: OK run #8: OK run #9: OK run #10: crashed: general protection fault in list_lru_add run #11: OK run #12: OK run #13: OK run #14: OK run #15: OK run #16: OK run #17: OK run #18: OK run #19: OK # git bisect bad a994402bc4714cefea5770b2d906cef5b0f4dc5c Bisecting: 56 revisions left to test after this (roughly 6 steps) [1f391eb270791359ee79031945dbe3afeaec6ce3] mm: list_lru: rename memcg_drain_all_list_lrus to memcg_reparent_list_lrus testing commit 1f391eb270791359ee79031945dbe3afeaec6ce3 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 1adfbbe1cd777f6fae31e6c3ebed01afb9b4aad3029e6e67d49cc3efbf28c159 run #0: crashed: general protection fault in list_lru_add run #1: OK run #2: OK run #3: OK run #4: crashed: general protection fault in list_lru_add run #5: OK run #6: OK run #7: OK run #8: OK run #9: OK run #10: OK run #11: OK run #12: OK run #13: crashed: general protection fault in list_lru_add run #14: OK run #15: OK run #16: OK run #17: OK run #18: OK run #19: crashed: general protection fault in list_lru_add # git bisect bad 1f391eb270791359ee79031945dbe3afeaec6ce3 Bisecting: 27 revisions left to test after this (roughly 5 steps) [f7cd16a55837f37b4c3835a2c646023e4d0f0e04] tmpfs: support for file creation time testing commit f7cd16a55837f37b4c3835a2c646023e4d0f0e04 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 6c2121f64ffb8924a647cfc2d7838bb957cab8ebb90e17cdee146e9908f47090 all runs: OK # git bisect good f7cd16a55837f37b4c3835a2c646023e4d0f0e04 Bisecting: 13 revisions left to test after this (roughly 4 steps) [2343e88d238f5de973d609d861c505890f94f22e] mm/memcg: disable threshold event handlers on PREEMPT_RT testing commit 2343e88d238f5de973d609d861c505890f94f22e compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 399ed115814b0fde9e411ebba537ee4c046bd851ba21a01ee0449b6feef5410f all runs: OK # git bisect good 2343e88d238f5de973d609d861c505890f94f22e Bisecting: 6 revisions left to test after this (roughly 3 steps) [8b9f3ac5b01db85c6cf74c2c3a71280cc3045c9c] fs: introduce alloc_inode_sb() to allocate filesystems specific inode testing commit 8b9f3ac5b01db85c6cf74c2c3a71280cc3045c9c compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: cc4744a3caf209c9400125d71e9ea79274498c0243c0fbc095e6deded5656d66 all runs: OK # git bisect good 8b9f3ac5b01db85c6cf74c2c3a71280cc3045c9c Bisecting: 3 revisions left to test after this (roughly 2 steps) [f53bf711d4d8e07de2caa3f13f6082c6e24145a4] mm: dcache: use kmem_cache_alloc_lru() to allocate dentry testing commit f53bf711d4d8e07de2caa3f13f6082c6e24145a4 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 4c0ec580aa4eeb0d4ac24c7ad1b9bb218a9ef7cbc68ccd17c7fe80386e9148d1 run #0: basic kernel testing failed: BUG: program execution failed: executor NUM: exit status NUM run #1: OK run #2: OK run #3: OK run #4: OK run #5: OK run #6: OK run #7: OK run #8: OK run #9: OK run #10: OK run #11: OK run #12: OK run #13: OK run #14: OK run #15: OK run #16: OK run #17: OK run #18: OK run #19: OK # git bisect good f53bf711d4d8e07de2caa3f13f6082c6e24145a4 Bisecting: 1 revision left to test after this (roughly 1 step) [da0efe30944476275c902c52fbac812db0541d87] mm: memcontrol: move memcg_online_kmem() to mem_cgroup_css_online() testing commit da0efe30944476275c902c52fbac812db0541d87 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: fcbc61d634bf3980da2c54c33ca14532acab85e0e3002245c0e27e9708e9ff94 all runs: OK # git bisect good da0efe30944476275c902c52fbac812db0541d87 Bisecting: 0 revisions left to test after this (roughly 0 steps) [5abc1e37afa0335c52608d640fd30910b2eeda21] mm: list_lru: allocate list_lru_one only when needed testing commit 5abc1e37afa0335c52608d640fd30910b2eeda21 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 9be0828b92a400a933e2774037de58d3ca2491c5a4fc9d941037b0ad10b1d800 run #0: crashed: general protection fault in list_lru_add run #1: crashed: general protection fault in list_lru_add run #2: crashed: general protection fault in list_lru_add run #3: OK run #4: OK run #5: OK run #6: OK run #7: OK run #8: OK run #9: OK run #10: OK run #11: OK run #12: OK run #13: OK run #14: OK run #15: OK run #16: OK run #17: OK run #18: OK run #19: OK # git bisect bad 5abc1e37afa0335c52608d640fd30910b2eeda21 5abc1e37afa0335c52608d640fd30910b2eeda21 is the first bad commit commit 5abc1e37afa0335c52608d640fd30910b2eeda21 Author: Muchun Song Date: Tue Mar 22 14:41:19 2022 -0700 mm: list_lru: allocate list_lru_one only when needed In our server, we found a suspected memory leak problem. The kmalloc-32 consumes more than 6GB of memory. Other kmem_caches consume less than 2GB memory. After our in-depth analysis, the memory consumption of kmalloc-32 slab cache is the cause of list_lru_one allocation. crash> p memcg_nr_cache_ids memcg_nr_cache_ids = $2 = 24574 memcg_nr_cache_ids is very large and memory consumption of each list_lru can be calculated with the following formula. num_numa_node * memcg_nr_cache_ids * 32 (kmalloc-32) There are 4 numa nodes in our system, so each list_lru consumes ~3MB. crash> list super_blocks | wc -l 952 Every mount will register 2 list lrus, one is for inode, another is for dentry. There are 952 super_blocks. So the total memory is 952 * 2 * 3 MB (~5.6GB). But the number of memory cgroup is less than 500. So I guess more than 12286 containers have been deployed on this machine (I do not know why there are so many containers, it may be a user's bug or the user really want to do that). And memcg_nr_cache_ids has not been reduced to a suitable value. This can waste a lot of memory. Now the infrastructure for dynamic list_lru_one allocation is ready, so remove statically allocated memory code to save memory. Link: https://lkml.kernel.org/r/20220228122126.37293-11-songmuchun@bytedance.com Signed-off-by: Muchun Song Cc: Alex Shi Cc: Anna Schumaker Cc: Chao Yu Cc: Dave Chinner Cc: Fam Zheng Cc: Jaegeuk Kim Cc: Johannes Weiner Cc: Kari Argillander Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Qi Zheng Cc: Roman Gushchin Cc: Shakeel Butt Cc: Theodore Ts'o Cc: Trond Myklebust Cc: Vladimir Davydov Cc: Vlastimil Babka Cc: Wei Yang Cc: Xiongchun Duan Cc: Yang Shi Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds include/linux/list_lru.h | 7 +-- mm/list_lru.c | 121 ++++++++++++++++++++++++++--------------------- mm/memcontrol.c | 6 ++- 3 files changed, 77 insertions(+), 57 deletions(-) culprit signature: 9be0828b92a400a933e2774037de58d3ca2491c5a4fc9d941037b0ad10b1d800 parent signature: fcbc61d634bf3980da2c54c33ca14532acab85e0e3002245c0e27e9708e9ff94 Reproducer flagged being flaky revisions tested: 13, total time: 2h34m41.253354069s (build: 1h20m51.068081793s, test: 1h12m38.542147728s) first bad commit: 5abc1e37afa0335c52608d640fd30910b2eeda21 mm: list_lru: allocate list_lru_one only when needed recipients (to): ["akpm@linux-foundation.org" "songmuchun@bytedance.com" "torvalds@linux-foundation.org"] recipients (cc): [] crash: general protection fault in list_lru_add general protection fault, probably for non-canonical address 0xdffffc0000000001: 0000 [#1] PREEMPT SMP KASAN KASAN: null-ptr-deref in range [0x0000000000000008-0x000000000000000f] CPU: 1 PID: 2966 Comm: udevd Not tainted 5.17.0-syzkaller #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 RIP: 0010:list_add_tail include/linux/list.h:102 [inline] RIP: 0010:list_lru_add+0x1b6/0x4a0 mm/list_lru.c:138 Code: 0f 84 ec 01 00 00 8b 44 24 08 4d 89 e9 85 c0 0f 89 93 01 00 00 4d 8d 45 08 48 b8 00 00 00 00 00 fc ff df 4c 89 c2 48 c1 ea 03 <80> 3c 02 00 0f 85 70 02 00 00 4d 8b 7d 08 4c 89 ca 48 89 df 4c 89 RSP: 0018:ffffc90001b37940 EFLAGS: 00010202 RAX: dffffc0000000000 RBX: ffff88823bcee0f8 RCX: 0000000000000001 RDX: 0000000000000001 RSI: ffffffff88eba2c0 RDI: ffff888021ed8090 RBP: ffff88801c36c7f0 R08: 0000000000000008 R09: 0000000000000000 R10: fffff940011de770 R11: 0000000000000001 R12: 0000000000000001 R13: 0000000000000000 R14: ffff88807e28d680 R15: ffff888021ed8080 FS: 00007fa26194b840(0000) GS:ffff8880b9f00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 00000000200c1038 CR3: 000000007b83e000 CR4: 00000000003506e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 Call Trace: d_lru_add fs/dcache.c:431 [inline] retain_dentry fs/dcache.c:685 [inline] dput+0x543/0xa00 fs/dcache.c:908 handle_mounts fs/namei.c:1552 [inline] step_into+0xca2/0x20f0 fs/namei.c:1850 open_last_lookups fs/namei.c:3423 [inline] path_openat+0x3d4/0x2390 fs/namei.c:3606 do_filp_open+0x199/0x3d0 fs/namei.c:3636 do_sys_openat2+0x11e/0x400 fs/open.c:1214 do_sys_open fs/open.c:1230 [inline] __do_sys_openat fs/open.c:1246 [inline] __se_sys_openat fs/open.c:1241 [inline] __x64_sys_openat+0x11b/0x1d0 fs/open.c:1241 do_syscall_x64 arch/x86/entry/common.c:50 [inline] do_syscall_64+0x35/0xb0 arch/x86/entry/common.c:80 entry_SYSCALL_64_after_hwframe+0x44/0xae RIP: 0033:0x7fa261aa2697 Code: 25 00 00 41 00 3d 00 00 41 00 74 37 64 8b 04 25 18 00 00 00 85 c0 75 5b 44 89 e2 48 89 ee bf 9c ff ff ff b8 01 01 00 00 0f 05 <48> 3d 00 f0 ff ff 0f 87 85 00 00 00 48 83 c4 68 5d 41 5c c3 0f 1f RSP: 002b:00007ffd66e35720 EFLAGS: 00000246 ORIG_RAX: 0000000000000101 RAX: ffffffffffffffda RBX: 0000560344daeba0 RCX: 00007fa261aa2697 RDX: 0000000000080000 RSI: 00007ffd66e35858 RDI: 00000000ffffff9c RBP: 00007ffd66e35858 R08: 0000000000000008 R09: 0000000000000001 R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000080000 R13: 0000560344daeba0 R14: 0000000000000001 R15: 0000000000000000 Modules linked in: ---[ end trace 0000000000000000 ]--- RIP: 0010:list_add_tail include/linux/list.h:102 [inline] RIP: 0010:list_lru_add+0x1b6/0x4a0 mm/list_lru.c:138 Code: 0f 84 ec 01 00 00 8b 44 24 08 4d 89 e9 85 c0 0f 89 93 01 00 00 4d 8d 45 08 48 b8 00 00 00 00 00 fc ff df 4c 89 c2 48 c1 ea 03 <80> 3c 02 00 0f 85 70 02 00 00 4d 8b 7d 08 4c 89 ca 48 89 df 4c 89 RSP: 0018:ffffc90001b37940 EFLAGS: 00010202 RAX: dffffc0000000000 RBX: ffff88823bcee0f8 RCX: 0000000000000001 RDX: 0000000000000001 RSI: ffffffff88eba2c0 RDI: ffff888021ed8090 RBP: ffff88801c36c7f0 R08: 0000000000000008 R09: 0000000000000000 R10: fffff940011de770 R11: 0000000000000001 R12: 0000000000000001 R13: 0000000000000000 R14: ffff88807e28d680 R15: ffff888021ed8080 FS: 00007fa26194b840(0000) GS:ffff8880b9f00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 00000000200c1038 CR3: 000000007b83e000 CR4: 00000000003506e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 ---------------- Code disassembly (best guess): 0: 0f 84 ec 01 00 00 je 0x1f2 6: 8b 44 24 08 mov 0x8(%rsp),%eax a: 4d 89 e9 mov %r13,%r9 d: 85 c0 test %eax,%eax f: 0f 89 93 01 00 00 jns 0x1a8 15: 4d 8d 45 08 lea 0x8(%r13),%r8 19: 48 b8 00 00 00 00 00 movabs $0xdffffc0000000000,%rax 20: fc ff df 23: 4c 89 c2 mov %r8,%rdx 26: 48 c1 ea 03 shr $0x3,%rdx * 2a: 80 3c 02 00 cmpb $0x0,(%rdx,%rax,1) <-- trapping instruction 2e: 0f 85 70 02 00 00 jne 0x2a4 34: 4d 8b 7d 08 mov 0x8(%r13),%r15 38: 4c 89 ca mov %r9,%rdx 3b: 48 89 df mov %rbx,%rdi 3e: 4c rex.WR 3f: 89 .byte 0x89