======================================================
WARNING: possible circular locking dependency detected
6.1.0-syzkaller-04356-g91e2286160ed #0 Not tainted
------------------------------------------------------
kworker/u4:5/1024 is trying to acquire lock:
ffff88807c6ad3b0 (&rs->rs_recv_lock){...-}-{2:2}, at: rds_wake_sk_sleep+0x23/0xe0 net/rds/af_rds.c:109
but task is already holding lock:
ffff888071d02100 (&rm->m_rs_lock){..-.}-{2:2}, at: rds_send_remove_from_sock+0x340/0x9e0 net/rds/send.c:628
which lock already depends on the new lock.
the existing dependency chain (in reverse order) is:
-> #1 (&rm->m_rs_lock){..-.}-{2:2}:
__raw_spin_lock_irqsave include/linux/spinlock_api_smp.h:110 [inline]
_raw_spin_lock_irqsave+0x3d/0x60 kernel/locking/spinlock.c:162
rds_message_purge net/rds/message.c:138 [inline]
rds_message_put+0x1dd/0xc20 net/rds/message.c:180
rds_inc_put net/rds/recv.c:82 [inline]
rds_inc_put+0x13e/0x1a0 net/rds/recv.c:76
rds_clear_recv_queue+0x14b/0x350 net/rds/recv.c:767
rds_release+0xd8/0x3c0 net/rds/af_rds.c:73
__sock_release+0xcd/0x280 net/socket.c:650
sock_close+0x1c/0x20 net/socket.c:1365
__fput+0x27c/0xa90 fs/file_table.c:320
task_work_run+0x16f/0x270 kernel/task_work.c:179
resume_user_mode_work include/linux/resume_user_mode.h:49 [inline]
exit_to_user_mode_loop kernel/entry/common.c:171 [inline]
exit_to_user_mode_prepare+0x23c/0x250 kernel/entry/common.c:203
__syscall_exit_to_user_mode_work kernel/entry/common.c:285 [inline]
syscall_exit_to_user_mode+0x1d/0x50 kernel/entry/common.c:296
do_syscall_64+0x46/0xb0 arch/x86/entry/common.c:86
entry_SYSCALL_64_after_hwframe+0x63/0xcd
-> #0 (&rs->rs_recv_lock){...-}-{2:2}:
check_prev_add kernel/locking/lockdep.c:3097 [inline]
check_prevs_add kernel/locking/lockdep.c:3216 [inline]
validate_chain kernel/locking/lockdep.c:3831 [inline]
__lock_acquire+0x2a43/0x56d0 kernel/locking/lockdep.c:5055
lock_acquire kernel/locking/lockdep.c:5668 [inline]
lock_acquire+0x1e3/0x630 kernel/locking/lockdep.c:5633
__raw_read_lock_irqsave include/linux/rwlock_api_smp.h:160 [inline]
_raw_read_lock_irqsave+0x49/0x90 kernel/locking/spinlock.c:236
rds_wake_sk_sleep+0x23/0xe0 net/rds/af_rds.c:109
rds_send_remove_from_sock+0xb9/0x9e0 net/rds/send.c:634
rds_send_path_drop_acked+0x2f3/0x3d0 net/rds/send.c:710
rds_tcp_write_space+0x1b5/0x690 net/rds/tcp_send.c:198
tcp_new_space net/ipv4/tcp_input.c:5483 [inline]
tcp_check_space+0x11b/0x810 net/ipv4/tcp_input.c:5502
tcp_data_snd_check net/ipv4/tcp_input.c:5511 [inline]
tcp_rcv_established+0x93e/0x2230 net/ipv4/tcp_input.c:6019
tcp_v4_do_rcv+0x670/0x9b0 net/ipv4/tcp_ipv4.c:1721
tcp_v4_rcv+0x2eab/0x3280 net/ipv4/tcp_ipv4.c:2142
ip_protocol_deliver_rcu+0x9f/0x460 net/ipv4/ip_input.c:205
ip_local_deliver_finish+0x2ec/0x4c0 net/ipv4/ip_input.c:233
NF_HOOK include/linux/netfilter.h:302 [inline]
NF_HOOK include/linux/netfilter.h:296 [inline]
ip_local_deliver+0x1ae/0x200 net/ipv4/ip_input.c:254
dst_input include/net/dst.h:454 [inline]
ip_rcv_finish+0x1cf/0x2f0 net/ipv4/ip_input.c:449
NF_HOOK include/linux/netfilter.h:302 [inline]
NF_HOOK include/linux/netfilter.h:296 [inline]
ip_rcv+0xae/0xd0 net/ipv4/ip_input.c:569
__netif_receive_skb_one_core+0x114/0x180 net/core/dev.c:5482
__netif_receive_skb+0x1f/0x1c0 net/core/dev.c:5596
process_backlog+0x3e4/0x810 net/core/dev.c:5924
__napi_poll+0xb8/0x770 net/core/dev.c:6485
napi_poll net/core/dev.c:6552 [inline]
net_rx_action+0xa00/0xde0 net/core/dev.c:6663
__do_softirq+0x1fb/0xadc kernel/softirq.c:571
do_softirq.part.0+0xde/0x130 kernel/softirq.c:472
do_softirq kernel/softirq.c:464 [inline]
__local_bh_enable_ip+0x106/0x130 kernel/softirq.c:396
local_bh_enable include/linux/bottom_half.h:33 [inline]
rcu_read_unlock_bh include/linux/rcupdate.h:834 [inline]
ip_finish_output2+0x7dc/0x2180 net/ipv4/ip_output.c:229
__ip_finish_output net/ipv4/ip_output.c:306 [inline]
__ip_finish_output+0x396/0x650 net/ipv4/ip_output.c:288
ip_finish_output+0x31/0x280 net/ipv4/ip_output.c:316
NF_HOOK_COND include/linux/netfilter.h:291 [inline]
ip_output+0x1a3/0x320 net/ipv4/ip_output.c:430
dst_output include/net/dst.h:444 [inline]
ip_local_out net/ipv4/ip_output.c:126 [inline]
__ip_queue_xmit+0x8e2/0x1bf0 net/ipv4/ip_output.c:532
__tcp_transmit_skb+0x1967/0x3800 net/ipv4/tcp_output.c:1399
__tcp_send_ack.part.0+0x3a7/0x760 net/ipv4/tcp_output.c:3983
__tcp_send_ack net/ipv4/tcp_output.c:3989 [inline]
tcp_send_ack+0x81/0xa0 net/ipv4/tcp_output.c:3989
__tcp_ack_snd_check+0x156/0x9c0 net/ipv4/tcp_input.c:5536
tcp_ack_snd_check net/ipv4/tcp_input.c:5582 [inline]
tcp_rcv_established+0x992/0x2230 net/ipv4/tcp_input.c:6020
tcp_v4_do_rcv+0x670/0x9b0 net/ipv4/tcp_ipv4.c:1721
sk_backlog_rcv include/net/sock.h:1113 [inline]
__release_sock+0x133/0x3b0 net/core/sock.c:2928
release_sock+0x58/0x1b0 net/core/sock.c:3485
rds_tcp_tune+0x50c/0x770 net/rds/tcp.c:522
rds_tcp_accept_one+0x2fd/0xd10 net/rds/tcp_listen.c:136
rds_tcp_accept_worker+0x59/0x80 net/rds/tcp.c:532
process_one_work+0x9bf/0x1710 kernel/workqueue.c:2289
worker_thread+0x669/0x1090 kernel/workqueue.c:2436
kthread+0x2e8/0x3a0 kernel/kthread.c:376
ret_from_fork+0x1f/0x30 arch/x86/entry/entry_64.S:308
other info that might help us debug this:
Possible unsafe locking scenario:
CPU0 CPU1
---- ----
lock(&rm->m_rs_lock);
lock(&rs->rs_recv_lock);
lock(&rm->m_rs_lock);
lock(&rs->rs_recv_lock);
*** DEADLOCK ***
9 locks held by kworker/u4:5/1024:
#0: ffff88814a15d938 ((wq_completion)krdsd){+.+.}-{0:0}, at: arch_atomic64_set arch/x86/include/asm/atomic64_64.h:34 [inline]
#0: ffff88814a15d938 ((wq_completion)krdsd){+.+.}-{0:0}, at: arch_atomic_long_set include/linux/atomic/atomic-long.h:41 [inline]
#0: ffff88814a15d938 ((wq_completion)krdsd){+.+.}-{0:0}, at: atomic_long_set include/linux/atomic/atomic-instrumented.h:1280 [inline]
#0: ffff88814a15d938 ((wq_completion)krdsd){+.+.}-{0:0}, at: set_work_data kernel/workqueue.c:636 [inline]
#0: ffff88814a15d938 ((wq_completion)krdsd){+.+.}-{0:0}, at: set_work_pool_and_clear_pending kernel/workqueue.c:663 [inline]
#0: ffff88814a15d938 ((wq_completion)krdsd){+.+.}-{0:0}, at: process_one_work+0x86d/0x1710 kernel/workqueue.c:2260
#1: ffffc90005017da8 ((work_completion)(&rtn->rds_tcp_accept_w)){+.+.}-{0:0}, at: process_one_work+0x8a1/0x1710 kernel/workqueue.c:2264
#2: ffff88807a50f3f0 (k-sk_lock-AF_INET6){+.+.}-{0:0}, at: lock_sock include/net/sock.h:1725 [inline]
#2: ffff88807a50f3f0 (k-sk_lock-AF_INET6){+.+.}-{0:0}, at: rds_tcp_tune+0xbb/0x770 net/rds/tcp.c:497
#3: ffffffff8c7917c0 (rcu_read_lock){....}-{1:2}, at: read_pnet include/net/net_namespace.h:383 [inline]
#3: ffffffff8c7917c0 (rcu_read_lock){....}-{1:2}, at: sock_net include/net/sock.h:649 [inline]
#3: ffffffff8c7917c0 (rcu_read_lock){....}-{1:2}, at: __ip_queue_xmit+0x41/0x1bf0 net/ipv4/ip_output.c:457
#4: ffffffff8c7917c0 (rcu_read_lock){....}-{1:2}, at: __skb_unlink include/linux/skbuff.h:2327 [inline]
#4: ffffffff8c7917c0 (rcu_read_lock){....}-{1:2}, at: __skb_dequeue include/linux/skbuff.h:2342 [inline]
#4: ffffffff8c7917c0 (rcu_read_lock){....}-{1:2}, at: process_backlog+0x38f/0x810 net/core/dev.c:5922
#5: ffffffff8c7917c0 (rcu_read_lock){....}-{1:2}, at: __skb_pull include/linux/skbuff.h:2612 [inline]
#5: ffffffff8c7917c0 (rcu_read_lock){....}-{1:2}, at: ip_local_deliver_finish+0x20a/0x4c0 net/ipv4/ip_input.c:230
#6: ffff88802660ca30 (k-slock-AF_INET/1){+.-.}-{2:2}, at: tcp_v4_rcv+0x2c27/0x3280 net/ipv4/tcp_ipv4.c:2138
#7: ffff88802660cd38 (k-clock-AF_INET){++.-}-{2:2}, at: rds_tcp_write_space+0x29/0x690 net/rds/tcp_send.c:184
#8: ffff888071d02100 (&rm->m_rs_lock){..-.}-{2:2}, at: rds_send_remove_from_sock+0x340/0x9e0 net/rds/send.c:628
stack backtrace:
CPU: 0 PID: 1024 Comm: kworker/u4:5 Not tainted 6.1.0-syzkaller-04356-g91e2286160ed #0
Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 10/26/2022
Workqueue: krdsd rds_tcp_accept_worker
Call Trace:
__dump_stack lib/dump_stack.c:88 [inline]
dump_stack_lvl+0xd1/0x138 lib/dump_stack.c:106
check_noncircular+0x25f/0x2e0 kernel/locking/lockdep.c:2177
check_prev_add kernel/locking/lockdep.c:3097 [inline]
check_prevs_add kernel/locking/lockdep.c:3216 [inline]
validate_chain kernel/locking/lockdep.c:3831 [inline]
__lock_acquire+0x2a43/0x56d0 kernel/locking/lockdep.c:5055
lock_acquire kernel/locking/lockdep.c:5668 [inline]
lock_acquire+0x1e3/0x630 kernel/locking/lockdep.c:5633
__raw_read_lock_irqsave include/linux/rwlock_api_smp.h:160 [inline]
_raw_read_lock_irqsave+0x49/0x90 kernel/locking/spinlock.c:236
rds_wake_sk_sleep+0x23/0xe0 net/rds/af_rds.c:109
rds_send_remove_from_sock+0xb9/0x9e0 net/rds/send.c:634
rds_send_path_drop_acked+0x2f3/0x3d0 net/rds/send.c:710
rds_tcp_write_space+0x1b5/0x690 net/rds/tcp_send.c:198
tcp_new_space net/ipv4/tcp_input.c:5483 [inline]
tcp_check_space+0x11b/0x810 net/ipv4/tcp_input.c:5502
tcp_data_snd_check net/ipv4/tcp_input.c:5511 [inline]
tcp_rcv_established+0x93e/0x2230 net/ipv4/tcp_input.c:6019
tcp_v4_do_rcv+0x670/0x9b0 net/ipv4/tcp_ipv4.c:1721
tcp_v4_rcv+0x2eab/0x3280 net/ipv4/tcp_ipv4.c:2142
ip_protocol_deliver_rcu+0x9f/0x460 net/ipv4/ip_input.c:205
ip_local_deliver_finish+0x2ec/0x4c0 net/ipv4/ip_input.c:233
NF_HOOK include/linux/netfilter.h:302 [inline]
NF_HOOK include/linux/netfilter.h:296 [inline]
ip_local_deliver+0x1ae/0x200 net/ipv4/ip_input.c:254
dst_input include/net/dst.h:454 [inline]
ip_rcv_finish+0x1cf/0x2f0 net/ipv4/ip_input.c:449
NF_HOOK include/linux/netfilter.h:302 [inline]
NF_HOOK include/linux/netfilter.h:296 [inline]
ip_rcv+0xae/0xd0 net/ipv4/ip_input.c:569
__netif_receive_skb_one_core+0x114/0x180 net/core/dev.c:5482
__netif_receive_skb+0x1f/0x1c0 net/core/dev.c:5596
process_backlog+0x3e4/0x810 net/core/dev.c:5924
__napi_poll+0xb8/0x770 net/core/dev.c:6485
napi_poll net/core/dev.c:6552 [inline]
net_rx_action+0xa00/0xde0 net/core/dev.c:6663
__do_softirq+0x1fb/0xadc kernel/softirq.c:571
do_softirq.part.0+0xde/0x130 kernel/softirq.c:472
do_softirq kernel/softirq.c:464 [inline]
__local_bh_enable_ip+0x106/0x130 kernel/softirq.c:396
local_bh_enable include/linux/bottom_half.h:33 [inline]
rcu_read_unlock_bh include/linux/rcupdate.h:834 [inline]
ip_finish_output2+0x7dc/0x2180 net/ipv4/ip_output.c:229
__ip_finish_output net/ipv4/ip_output.c:306 [inline]
__ip_finish_output+0x396/0x650 net/ipv4/ip_output.c:288
ip_finish_output+0x31/0x280 net/ipv4/ip_output.c:316
NF_HOOK_COND include/linux/netfilter.h:291 [inline]
ip_output+0x1a3/0x320 net/ipv4/ip_output.c:430
dst_output include/net/dst.h:444 [inline]
ip_local_out net/ipv4/ip_output.c:126 [inline]
__ip_queue_xmit+0x8e2/0x1bf0 net/ipv4/ip_output.c:532
__tcp_transmit_skb+0x1967/0x3800 net/ipv4/tcp_output.c:1399
__tcp_send_ack.part.0+0x3a7/0x760 net/ipv4/tcp_output.c:3983
__tcp_send_ack net/ipv4/tcp_output.c:3989 [inline]
tcp_send_ack+0x81/0xa0 net/ipv4/tcp_output.c:3989
__tcp_ack_snd_check+0x156/0x9c0 net/ipv4/tcp_input.c:5536
tcp_ack_snd_check net/ipv4/tcp_input.c:5582 [inline]
tcp_rcv_established+0x992/0x2230 net/ipv4/tcp_input.c:6020
tcp_v4_do_rcv+0x670/0x9b0 net/ipv4/tcp_ipv4.c:1721
sk_backlog_rcv include/net/sock.h:1113 [inline]
__release_sock+0x133/0x3b0 net/core/sock.c:2928
release_sock+0x58/0x1b0 net/core/sock.c:3485
rds_tcp_tune+0x50c/0x770 net/rds/tcp.c:522
rds_tcp_accept_one+0x2fd/0xd10 net/rds/tcp_listen.c:136
rds_tcp_accept_worker+0x59/0x80 net/rds/tcp.c:532
process_one_work+0x9bf/0x1710 kernel/workqueue.c:2289
worker_thread+0x669/0x1090 kernel/workqueue.c:2436
kthread+0x2e8/0x3a0 kernel/kthread.c:376
ret_from_fork+0x1f/0x30 arch/x86/entry/entry_64.S:308