bisecting cause commit starting from aa5334b1f96801cd09775217a72ff252ef614d7a building syzkaller on 7268fa62257981feeebc89e55b5ce45294beff8c testing commit aa5334b1f96801cd09775217a72ff252ef614d7a compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: e7c03b18998a0c12ed5586e78908028e8de8504fa0208246957de83d26f7dab5 all runs: crashed: WARNING in inet_csk_get_port testing release v5.17 testing commit f443e374ae131c168a065ea1748feac6b2e76613 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: c8b40f922e699d2b82278dc1a2019f8e54aaf2723a317b904bc87dc7cb862306 all runs: OK # git bisect start aa5334b1f96801cd09775217a72ff252ef614d7a f443e374ae131c168a065ea1748feac6b2e76613 Bisecting: 8696 revisions left to test after this (roughly 13 steps) [b14ffae378aa1db993e62b01392e70d1e585fb23] Merge tag 'drm-next-2022-03-24' of git://anongit.freedesktop.org/drm/drm testing commit b14ffae378aa1db993e62b01392e70d1e585fb23 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 6ff81b93bc47056ea5078c01a112a063d2cf911eb7ded4d5515a4d26670e2515 all runs: OK # git bisect good b14ffae378aa1db993e62b01392e70d1e585fb23 Bisecting: 4347 revisions left to test after this (roughly 12 steps) [2975dbdc3989cd66a4cb5a7c5510de2de8ee4d14] Merge tag 'net-5.18-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net testing commit 2975dbdc3989cd66a4cb5a7c5510de2de8ee4d14 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 5869e077bb86d949adcd31d0615d8c258b46b7abf9bc11f3a7a602b3e64e4ac4 run #0: basic kernel testing failed: BUG: program execution failed: executor NUM: exit status NUM run #1: OK run #2: OK run #3: OK run #4: OK run #5: OK run #6: OK run #7: OK run #8: OK run #9: OK # git bisect good 2975dbdc3989cd66a4cb5a7c5510de2de8ee4d14 Bisecting: 2161 revisions left to test after this (roughly 11 steps) [68533eb1fb197a413fd8612ebb88e111ade3beac] Merge tag 'net-5.18-rc6' of git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net testing commit 68533eb1fb197a413fd8612ebb88e111ade3beac compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 3aec4f1012edea8ae166e3a2fab6304202048d95fd3a2c5b1c9d94cc69baf79d all runs: OK # git bisect good 68533eb1fb197a413fd8612ebb88e111ade3beac Bisecting: 1080 revisions left to test after this (roughly 10 steps) [6cd2126ac602305f52e43666400b135b20b46b07] net/mlx5: Cleanup XFRM attributes struct testing commit 6cd2126ac602305f52e43666400b135b20b46b07 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 16996b596942d4676e903872f142eb23dc6fb95965bda391480c3ac0e0db871e all runs: OK # git bisect good 6cd2126ac602305f52e43666400b135b20b46b07 Bisecting: 540 revisions left to test after this (roughly 9 steps) [7c4e983c4f3cf94fcd879730c6caa877e0768a4d] net: allow gso_max_size to exceed 65536 testing commit 7c4e983c4f3cf94fcd879730c6caa877e0768a4d compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: f2ead5948394498503a6a9988d51c3d986f8e043745a1ad65e7c8cad64c4a996 all runs: OK # git bisect good 7c4e983c4f3cf94fcd879730c6caa877e0768a4d Bisecting: 325 revisions left to test after this (roughly 8 steps) [d904c8cc0302393640bc29ee62193f88ddc53126] Merge tag 'net-5.18-rc8' of git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net testing commit d904c8cc0302393640bc29ee62193f88ddc53126 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: a63268c76f277600bdea8850613f6b4c9c6222acee4d02545dca53aa55dcce84 run #0: basic kernel testing failed: BUG: program execution failed: executor NUM: exit status NUM run #1: OK run #2: OK run #3: OK run #4: OK run #5: OK run #6: OK run #7: OK run #8: OK run #9: OK # git bisect good d904c8cc0302393640bc29ee62193f88ddc53126 Bisecting: 188 revisions left to test after this (roughly 7 steps) [78488a64aea94a3336ee97f345c1496e9bc5ebdf] iwlwifi: mei: fix potential NULL-ptr deref testing commit 78488a64aea94a3336ee97f345c1496e9bc5ebdf compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: ae798cd0b221a6bbeed04066dd41d11f1fe473eb52f132e55dfa3da71027f6a2 all runs: OK # git bisect good 78488a64aea94a3336ee97f345c1496e9bc5ebdf Bisecting: 93 revisions left to test after this (roughly 7 steps) [6431ce6cd3bb2ccf55acab0ccaae33208e9a0ab3] Merge tag 'mlx5-updates-2022-05-17' of git://git.kernel.org/pub/scm/linux/kernel/git/saeed/linux testing commit 6431ce6cd3bb2ccf55acab0ccaae33208e9a0ab3 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 53feb4c0c5f4c7e8dbbe4fa5e8cf4963888d09cdccbfeca77061e3fb5a556fac all runs: OK # git bisect good 6431ce6cd3bb2ccf55acab0ccaae33208e9a0ab3 Bisecting: 46 revisions left to test after this (roughly 6 steps) [1ec619ee4a052fb9ac48b57554ac2722a0bfe73c] net: mtk_eth_soc: provide mtk_sgmii_config() testing commit 1ec619ee4a052fb9ac48b57554ac2722a0bfe73c compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 4e0132122c29c17aa8557a4db542979a884d294b31ea88bb7bc3026d43247d81 all runs: OK # git bisect good 1ec619ee4a052fb9ac48b57554ac2722a0bfe73c Bisecting: 21 revisions left to test after this (roughly 5 steps) [805cb5aadc2a88c453cfe620b28e12ff2fac27a6] Merge git://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf-next testing commit 805cb5aadc2a88c453cfe620b28e12ff2fac27a6 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: ec76a606daba49a8e767762bc2ba717fe6ce50579b7e3efa965c782a0c853736 all runs: OK # git bisect good 805cb5aadc2a88c453cfe620b28e12ff2fac27a6 Bisecting: 10 revisions left to test after this (roughly 4 steps) [c09b0cd2cc6c3f91988a20d45fa45c889f72c56c] net: avoid strange behavior with skb_defer_max == 1 testing commit c09b0cd2cc6c3f91988a20d45fa45c889f72c56c compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 1c9e9be041d0a544d4a84c9a4d4b70232376ce00743957a4fca9128596d06cc3 all runs: OK # git bisect good c09b0cd2cc6c3f91988a20d45fa45c889f72c56c Bisecting: 5 revisions left to test after this (roughly 3 steps) [c2e10f53455c898050738d6a5f8c237f27aec225] net: vxlan: Fix kernel coding style testing commit c2e10f53455c898050738d6a5f8c237f27aec225 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: e6ea1ec0e148dfbde75148c249dfd47f8a4e8de7a471701227559062fb7aa581 all runs: OK # git bisect good c2e10f53455c898050738d6a5f8c237f27aec225 Bisecting: 2 revisions left to test after this (roughly 2 steps) [eac67d83bf2553f98cfddd42cfbcfd6f9ccfc287] wwan: iosm: use a flexible array rather than allocate short objects testing commit eac67d83bf2553f98cfddd42cfbcfd6f9ccfc287 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: f3737f9af630fbee598f552ab3e3a9e484e84ccb6cf6310720bf10aaec98e484 all runs: OK # git bisect good eac67d83bf2553f98cfddd42cfbcfd6f9ccfc287 Bisecting: 0 revisions left to test after this (roughly 1 step) [538aaf9b2383701094a47797b4554c6a21c83eed] selftests: Add test for timing a bind request to a port with a populated bhash entry testing commit 538aaf9b2383701094a47797b4554c6a21c83eed compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 9ac9faeb55a480708ae5ff126d3e076384f66a51a5c91d1309ac29214cd63a87 run #0: basic kernel testing failed: BUG: program execution failed: executor NUM: exit status NUM run #1: crashed: WARNING in inet_csk_get_port run #2: crashed: WARNING in inet_csk_get_port run #3: crashed: WARNING in inet_csk_get_port run #4: crashed: WARNING in inet_csk_get_port run #5: crashed: WARNING in inet_csk_get_port run #6: crashed: WARNING in inet_csk_get_port run #7: crashed: WARNING in inet_csk_get_port run #8: crashed: WARNING in inet_csk_get_port run #9: crashed: WARNING in inet_csk_get_port # git bisect bad 538aaf9b2383701094a47797b4554c6a21c83eed Bisecting: 0 revisions left to test after this (roughly 0 steps) [d5a42de8bdbe25081f07b801d8b35f4d75a791f4] net: Add a second bind table hashed by port and address testing commit d5a42de8bdbe25081f07b801d8b35f4d75a791f4 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.2 kernel signature: 343fb67dcba897ae38839775b9789cba2964a5ec960a615d7cd38382bc8cef5b all runs: crashed: WARNING in inet_csk_get_port # git bisect bad d5a42de8bdbe25081f07b801d8b35f4d75a791f4 d5a42de8bdbe25081f07b801d8b35f4d75a791f4 is the first bad commit commit d5a42de8bdbe25081f07b801d8b35f4d75a791f4 Author: Joanne Koong Date: Thu May 19 17:18:33 2022 -0700 net: Add a second bind table hashed by port and address We currently have one tcp bind table (bhash) which hashes by port number only. In the socket bind path, we check for bind conflicts by traversing the specified port's inet_bind2_bucket while holding the bucket's spinlock (see inet_csk_get_port() and inet_csk_bind_conflict()). In instances where there are tons of sockets hashed to the same port at different addresses, checking for a bind conflict is time-intensive and can cause softirq cpu lockups, as well as stops new tcp connections since __inet_inherit_port() also contests for the spinlock. This patch proposes adding a second bind table, bhash2, that hashes by port and ip address. Searching the bhash2 table leads to significantly faster conflict resolution and less time holding the spinlock. Signed-off-by: Joanne Koong Reviewed-by: Eric Dumazet Acked-by: Kuniyuki Iwashima Signed-off-by: Jakub Kicinski include/net/inet_connection_sock.h | 3 + include/net/inet_hashtables.h | 68 +++++++++- include/net/sock.h | 14 +++ net/dccp/proto.c | 33 ++++- net/ipv4/inet_connection_sock.c | 247 +++++++++++++++++++++++++++---------- net/ipv4/inet_hashtables.c | 193 +++++++++++++++++++++++++++-- net/ipv4/tcp.c | 14 ++- 7 files changed, 489 insertions(+), 83 deletions(-) culprit signature: 343fb67dcba897ae38839775b9789cba2964a5ec960a615d7cd38382bc8cef5b parent signature: f3737f9af630fbee598f552ab3e3a9e484e84ccb6cf6310720bf10aaec98e484 revisions tested: 17, total time: 3h57m58.60807374s (build: 1h46m13.785268013s, test: 2h10m2.852473537s) first bad commit: d5a42de8bdbe25081f07b801d8b35f4d75a791f4 net: Add a second bind table hashed by port and address recipients (to): ["edumazet@google.com" "joannelkoong@gmail.com" "kuba@kernel.org" "kuniyu@amazon.co.jp"] recipients (cc): [] crash: WARNING in inet_csk_get_port nf_conntrack: default automatic helper assignment has been turned off for security reasons and CT-based firewall rule not found. Use the iptables CT target to attach helpers instead. ------------[ cut here ]------------ WARNING: CPU: 1 PID: 4083 at net/ipv4/inet_connection_sock.c:525 inet_csk_get_port+0xec1/0x17e0 net/ipv4/inet_connection_sock.c:524 Modules linked in: CPU: 1 PID: 4083 Comm: syz-executor.0 Not tainted 5.18.0-rc7-syzkaller #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 RIP: 0010:inet_csk_get_port+0xec1/0x17e0 net/ipv4/inet_connection_sock.c:525 Code: c1 ea 03 80 3c 02 00 0f 85 9b 05 00 00 48 8b 44 24 18 4c 89 d6 48 8b 78 18 e8 4b e6 fe ff e9 c3 fa ff ff 0f 0b e9 61 fc ff ff <0f> 0b e9 90 fc ff ff e8 d3 2d 79 01 85 c0 0f 84 8c fd ff ff 80 3d RSP: 0018:ffffc90002cffbf8 EFLAGS: 00010283 RAX: ffff888023617200 RBX: 0000000000000000 RCX: 1ffffffff1e75b16 RDX: 1ffff110297db59d RSI: ffffffff890ba2a0 RDI: ffff88814bedace8 RBP: ffff88814bedace0 R08: 0000000000000001 R09: ffffffff8f37794f R10: ffff888023618100 R11: 0000000000000001 R12: 0000000000000001 R13: 0000000000000000 R14: ffff88814beda640 R15: ffffc90001676870 FS: 00007fb0f366d700(0000) GS:ffff8880b9f00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 0000000020001540 CR3: 00000000722d7000 CR4: 00000000003506e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 Call Trace: inet_csk_listen_start+0x127/0x380 net/ipv4/inet_connection_sock.c:1178 inet_listen+0x1c4/0x500 net/ipv4/af_inet.c:228 __sys_listen+0x14d/0x200 net/socket.c:1778 __do_sys_listen net/socket.c:1787 [inline] __se_sys_listen net/socket.c:1785 [inline] __x64_sys_listen+0x4b/0x70 net/socket.c:1785 do_syscall_x64 arch/x86/entry/common.c:50 [inline] do_syscall_64+0x35/0xb0 arch/x86/entry/common.c:80 entry_SYSCALL_64_after_hwframe+0x44/0xae RIP: 0033:0x7fb0f24890e9 Code: ff ff c3 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 40 00 48 89 f8 48 89 f7 48 89 d6 48 89 ca 4d 89 c2 4d 89 c8 4c 8b 4c 24 08 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 c7 c1 b8 ff ff ff f7 d8 64 89 01 48 RSP: 002b:00007fb0f366d168 EFLAGS: 00000246 ORIG_RAX: 0000000000000032 RAX: ffffffffffffffda RBX: 00007fb0f259bf60 RCX: 00007fb0f24890e9 RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000004 RBP: 00007fb0f24e308d R08: 0000000000000000 R09: 0000000000000000 R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000000 R13: 00007fff0f6963ef R14: 00007fb0f366d300 R15: 0000000000022000