TCP: request_sock_TCPv6: Possible SYN flooding on port 20002. Sending cookies. Check SNMP counters. ======================================================== WARNING: possible irq lock inversion dependency detected 5.2.0-rc5-next-20190621 #20 Not tainted -------------------------------------------------------- modprobe/10164 just changed the state of lock: 00000000129985f3 (&(&ctx->ctx_lock)->rlock){..-.}, at: spin_lock_irq include/linux/spinlock.h:363 [inline] 00000000129985f3 (&(&ctx->ctx_lock)->rlock){..-.}, at: free_ioctx_users+0x2d/0x490 fs/aio.c:620 but this lock took another, SOFTIRQ-unsafe lock in the past: (&ctx->fault_pending_wqh){+.+.} and interrupts could create inverse lock ordering between them. other info that might help us debug this: Chain exists of: &(&ctx->ctx_lock)->rlock --> &ctx->fd_wqh --> &ctx->fault_pending_wqh Possible interrupt unsafe locking scenario: CPU0 CPU1 ---- ---- lock(&ctx->fault_pending_wqh); local_irq_disable(); lock(&(&ctx->ctx_lock)->rlock); lock(&ctx->fd_wqh); lock(&(&ctx->ctx_lock)->rlock); *** DEADLOCK *** 6 locks held by modprobe/10164: #0: 0000000095a959e5 (&mm->mmap_sem#2){++++}, at: do_user_addr_fault arch/x86/mm/fault.c:1379 [inline] #0: 0000000095a959e5 (&mm->mmap_sem#2){++++}, at: __do_page_fault+0x337/0xdc0 arch/x86/mm/fault.c:1504 #1: 0000000093181468 (rcu_read_lock){....}, at: filemap_map_pages+0x105/0x13e0 mm/filemap.c:2600 #2: 000000008a03469f (&(ptlock_ptr(page))->rlock#2){+.+.}, at: spin_lock include/linux/spinlock.h:338 [inline] #2: 000000008a03469f (&(ptlock_ptr(page))->rlock#2){+.+.}, at: pte_alloc_one_map mm/memory.c:3198 [inline] #2: 000000008a03469f (&(ptlock_ptr(page))->rlock#2){+.+.}, at: alloc_set_pte+0xa97/0x15e0 mm/memory.c:3316 #3: 0000000093181468 (rcu_read_lock){....}, at: rcu_read_lock+0x0/0x70 mm/memcontrol.c:3215 #4: 0000000059ad06ae (rcu_callback){....}, at: __rcu_reclaim kernel/rcu/rcu.h:210 [inline] #4: 0000000059ad06ae (rcu_callback){....}, at: rcu_do_batch kernel/rcu/tree.c:2092 [inline] #4: 0000000059ad06ae (rcu_callback){....}, at: invoke_rcu_callbacks kernel/rcu/tree.c:2310 [inline] #4: 0000000059ad06ae (rcu_callback){....}, at: rcu_core+0xb21/0x15a0 kernel/rcu/tree.c:2291 #5: 00000000358b1f9d (rcu_read_lock_sched){....}, at: percpu_ref_call_confirm_rcu lib/percpu-refcount.c:126 [inline] #5: 00000000358b1f9d (rcu_read_lock_sched){....}, at: percpu_ref_switch_to_atomic_rcu+0x20e/0x570 lib/percpu-refcount.c:165 the shortest dependencies between 2nd lock and 1st lock: -> (&ctx->fault_pending_wqh){+.+.} { HARDIRQ-ON-W at: lock_acquire+0x190/0x410 kernel/locking/lockdep.c:4418 __raw_spin_lock include/linux/spinlock_api_smp.h:142 [inline] _raw_spin_lock+0x2f/0x40 kernel/locking/spinlock.c:151 spin_lock include/linux/spinlock.h:338 [inline] userfaultfd_release+0x4ca/0x710 fs/userfaultfd.c:923 __fput+0x2ff/0x890 fs/file_table.c:280 ____fput+0x16/0x20 fs/file_table.c:313 task_work_run+0x145/0x1c0 kernel/task_work.c:113 tracehook_notify_resume include/linux/tracehook.h:188 [inline] exit_to_usermode_loop+0x280/0x2d0 arch/x86/entry/common.c:168 prepare_exit_to_usermode arch/x86/entry/common.c:199 [inline] syscall_return_slowpath arch/x86/entry/common.c:279 [inline] do_syscall_64+0x5a9/0x6a0 arch/x86/entry/common.c:304 entry_SYSCALL_64_after_hwframe+0x49/0xbe SOFTIRQ-ON-W at: lock_acquire+0x190/0x410 kernel/locking/lockdep.c:4418 __raw_spin_lock include/linux/spinlock_api_smp.h:142 [inline] _raw_spin_lock+0x2f/0x40 kernel/locking/spinlock.c:151 spin_lock include/linux/spinlock.h:338 [inline] userfaultfd_release+0x4ca/0x710 fs/userfaultfd.c:923 __fput+0x2ff/0x890 fs/file_table.c:280 ____fput+0x16/0x20 fs/file_table.c:313 task_work_run+0x145/0x1c0 kernel/task_work.c:113 tracehook_notify_resume include/linux/tracehook.h:188 [inline] exit_to_usermode_loop+0x280/0x2d0 arch/x86/entry/common.c:168 prepare_exit_to_usermode arch/x86/entry/common.c:199 [inline] syscall_return_slowpath arch/x86/entry/common.c:279 [inline] do_syscall_64+0x5a9/0x6a0 arch/x86/entry/common.c:304 entry_SYSCALL_64_after_hwframe+0x49/0xbe INITIAL USE at: lock_acquire+0x190/0x410 kernel/locking/lockdep.c:4418 __raw_spin_lock include/linux/spinlock_api_smp.h:142 [inline] _raw_spin_lock+0x2f/0x40 kernel/locking/spinlock.c:151 spin_lock include/linux/spinlock.h:338 [inline] userfaultfd_ctx_read fs/userfaultfd.c:1047 [inline] userfaultfd_read+0x54d/0x1950 fs/userfaultfd.c:1205 __vfs_read+0x8a/0x110 fs/read_write.c:425 vfs_read+0x1f0/0x440 fs/read_write.c:461 ksys_read+0x14f/0x290 fs/read_write.c:587 __do_sys_read fs/read_write.c:597 [inline] __se_sys_read fs/read_write.c:595 [inline] __x64_sys_read+0x73/0xb0 fs/read_write.c:595 do_syscall_64+0xfd/0x6a0 arch/x86/entry/common.c:301 entry_SYSCALL_64_after_hwframe+0x49/0xbe } ... key at: [] __key.46310+0x0/0x40 ... acquired at: __raw_spin_lock include/linux/spinlock_api_smp.h:142 [inline] _raw_spin_lock+0x2f/0x40 kernel/locking/spinlock.c:151 spin_lock include/linux/spinlock.h:338 [inline] userfaultfd_ctx_read fs/userfaultfd.c:1047 [inline] userfaultfd_read+0x54d/0x1950 fs/userfaultfd.c:1205 __vfs_read+0x8a/0x110 fs/read_write.c:425 vfs_read+0x1f0/0x440 fs/read_write.c:461 ksys_read+0x14f/0x290 fs/read_write.c:587 __do_sys_read fs/read_write.c:597 [inline] __se_sys_read fs/read_write.c:595 [inline] __x64_sys_read+0x73/0xb0 fs/read_write.c:595 do_syscall_64+0xfd/0x6a0 arch/x86/entry/common.c:301 entry_SYSCALL_64_after_hwframe+0x49/0xbe -> (&ctx->fd_wqh){....} { INITIAL USE at: lock_acquire+0x190/0x410 kernel/locking/lockdep.c:4418 __raw_spin_lock_irqsave include/linux/spinlock_api_smp.h:110 [inline] _raw_spin_lock_irqsave+0x95/0xcd kernel/locking/spinlock.c:159 __wake_up_common_lock+0xc7/0x190 kernel/sched/wait.c:121 __wake_up+0xe/0x10 kernel/sched/wait.c:146 userfaultfd_event_wait_completion+0x2ca/0xa70 fs/userfaultfd.c:620 dup_fctx fs/userfaultfd.c:722 [inline] dup_userfaultfd_complete+0x241/0x3d0 fs/userfaultfd.c:730 dup_mmap kernel/fork.c:606 [inline] dup_mm+0xf17/0x1430 kernel/fork.c:1347 copy_mm kernel/fork.c:1403 [inline] copy_process+0x28b7/0x69f0 kernel/fork.c:2042 _do_fork+0x146/0xfa0 kernel/fork.c:2379 __do_sys_clone kernel/fork.c:2521 [inline] __se_sys_clone kernel/fork.c:2501 [inline] __x64_sys_clone+0x186/0x250 kernel/fork.c:2501 do_syscall_64+0xfd/0x6a0 arch/x86/entry/common.c:301 entry_SYSCALL_64_after_hwframe+0x49/0xbe } ... key at: [] __key.46313+0x0/0x40 ... acquired at: __raw_spin_lock include/linux/spinlock_api_smp.h:142 [inline] _raw_spin_lock+0x2f/0x40 kernel/locking/spinlock.c:151 spin_lock include/linux/spinlock.h:338 [inline] aio_poll fs/aio.c:1749 [inline] __io_submit_one fs/aio.c:1823 [inline] io_submit_one+0xefa/0x2ef0 fs/aio.c:1860 __do_sys_io_submit fs/aio.c:1919 [inline] __se_sys_io_submit fs/aio.c:1889 [inline] __x64_sys_io_submit+0x1bd/0x570 fs/aio.c:1889 do_syscall_64+0xfd/0x6a0 arch/x86/entry/common.c:301 entry_SYSCALL_64_after_hwframe+0x49/0xbe -> (&(&ctx->ctx_lock)->rlock){..-.} { IN-SOFTIRQ-W at: lock_acquire+0x190/0x410 kernel/locking/lockdep.c:4418 __raw_spin_lock_irq include/linux/spinlock_api_smp.h:128 [inline] _raw_spin_lock_irq+0x60/0x80 kernel/locking/spinlock.c:167 spin_lock_irq include/linux/spinlock.h:363 [inline] free_ioctx_users+0x2d/0x490 fs/aio.c:620 percpu_ref_put_many include/linux/percpu-refcount.h:293 [inline] percpu_ref_put include/linux/percpu-refcount.h:309 [inline] percpu_ref_call_confirm_rcu lib/percpu-refcount.c:130 [inline] percpu_ref_switch_to_atomic_rcu+0x4c0/0x570 lib/percpu-refcount.c:165 __rcu_reclaim kernel/rcu/rcu.h:222 [inline] rcu_do_batch kernel/rcu/tree.c:2092 [inline] invoke_rcu_callbacks kernel/rcu/tree.c:2310 [inline] rcu_core+0xa3a/0x15a0 kernel/rcu/tree.c:2291 __do_softirq+0x260/0x9c8 kernel/softirq.c:292 invoke_softirq kernel/softirq.c:373 [inline] irq_exit+0x19b/0x1e0 kernel/softirq.c:413 exiting_irq arch/x86/include/asm/apic.h:536 [inline] smp_apic_timer_interrupt+0x16d/0x5b0 arch/x86/kernel/apic/apic.c:1068 apic_timer_interrupt+0xf/0x20 arch/x86/entry/entry_64.S:806 __sanitizer_cov_trace_const_cmp4+0xd/0x20 kernel/kcov.c:188 __this_cpu_preempt_check+0x31/0x2a0 lib/smp_processor_id.c:16 __mod_lruvec_state+0x132/0x310 mm/memcontrol.c:763 __mod_lruvec_page_state include/linux/memcontrol.h:710 [inline] page_add_file_rmap+0x561/0xd60 mm/rmap.c:1208 alloc_set_pte+0x381/0x15e0 mm/memory.c:3337 filemap_map_pages+0x1138/0x13e0 mm/filemap.c:2648 do_fault_around mm/memory.c:3480 [inline] do_read_fault mm/memory.c:3514 [inline] do_fault mm/memory.c:3648 [inline] handle_pte_fault mm/memory.c:3879 [inline] __handle_mm_fault+0x3249/0x3f20 mm/memory.c:4003 handle_mm_fault+0x3b7/0xaa0 mm/memory.c:4040 do_user_addr_fault arch/x86/mm/fault.c:1438 [inline] __do_page_fault+0x531/0xdc0 arch/x86/mm/fault.c:1504 do_page_fault+0x71/0x5e1 arch/x86/mm/fault.c:1535 page_fault+0x1e/0x30 arch/x86/entry/entry_64.S:1161 INITIAL USE at: lock_acquire+0x190/0x410 kernel/locking/lockdep.c:4418 __raw_spin_lock_irq include/linux/spinlock_api_smp.h:128 [inline] _raw_spin_lock_irq+0x60/0x80 kernel/locking/spinlock.c:167 spin_lock_irq include/linux/spinlock.h:363 [inline] aio_poll fs/aio.c:1747 [inline] __io_submit_one fs/aio.c:1823 [inline] io_submit_one+0xeb5/0x2ef0 fs/aio.c:1860 __do_sys_io_submit fs/aio.c:1919 [inline] __se_sys_io_submit fs/aio.c:1889 [inline] __x64_sys_io_submit+0x1bd/0x570 fs/aio.c:1889 do_syscall_64+0xfd/0x6a0 arch/x86/entry/common.c:301 entry_SYSCALL_64_after_hwframe+0x49/0xbe } ... key at: [] __key.53644+0x0/0x40 ... acquired at: mark_lock_irq kernel/locking/lockdep.c:3239 [inline] mark_lock+0x4fa/0x11e0 kernel/locking/lockdep.c:3617 mark_usage kernel/locking/lockdep.c:3488 [inline] __lock_acquire+0x1e8c/0x4680 kernel/locking/lockdep.c:3839 lock_acquire+0x190/0x410 kernel/locking/lockdep.c:4418 __raw_spin_lock_irq include/linux/spinlock_api_smp.h:128 [inline] _raw_spin_lock_irq+0x60/0x80 kernel/locking/spinlock.c:167 spin_lock_irq include/linux/spinlock.h:363 [inline] free_ioctx_users+0x2d/0x490 fs/aio.c:620 percpu_ref_put_many include/linux/percpu-refcount.h:293 [inline] percpu_ref_put include/linux/percpu-refcount.h:309 [inline] percpu_ref_call_confirm_rcu lib/percpu-refcount.c:130 [inline] percpu_ref_switch_to_atomic_rcu+0x4c0/0x570 lib/percpu-refcount.c:165 __rcu_reclaim kernel/rcu/rcu.h:222 [inline] rcu_do_batch kernel/rcu/tree.c:2092 [inline] invoke_rcu_callbacks kernel/rcu/tree.c:2310 [inline] rcu_core+0xa3a/0x15a0 kernel/rcu/tree.c:2291 __do_softirq+0x260/0x9c8 kernel/softirq.c:292 invoke_softirq kernel/softirq.c:373 [inline] irq_exit+0x19b/0x1e0 kernel/softirq.c:413 exiting_irq arch/x86/include/asm/apic.h:536 [inline] smp_apic_timer_interrupt+0x16d/0x5b0 arch/x86/kernel/apic/apic.c:1068 apic_timer_interrupt+0xf/0x20 arch/x86/entry/entry_64.S:806 __sanitizer_cov_trace_const_cmp4+0xd/0x20 kernel/kcov.c:188 __this_cpu_preempt_check+0x31/0x2a0 lib/smp_processor_id.c:16 __mod_lruvec_state+0x132/0x310 mm/memcontrol.c:763 __mod_lruvec_page_state include/linux/memcontrol.h:710 [inline] page_add_file_rmap+0x561/0xd60 mm/rmap.c:1208 alloc_set_pte+0x381/0x15e0 mm/memory.c:3337 filemap_map_pages+0x1138/0x13e0 mm/filemap.c:2648 do_fault_around mm/memory.c:3480 [inline] do_read_fault mm/memory.c:3514 [inline] do_fault mm/memory.c:3648 [inline] handle_pte_fault mm/memory.c:3879 [inline] __handle_mm_fault+0x3249/0x3f20 mm/memory.c:4003 handle_mm_fault+0x3b7/0xaa0 mm/memory.c:4040 do_user_addr_fault arch/x86/mm/fault.c:1438 [inline] __do_page_fault+0x531/0xdc0 arch/x86/mm/fault.c:1504 do_page_fault+0x71/0x5e1 arch/x86/mm/fault.c:1535 page_fault+0x1e/0x30 arch/x86/entry/entry_64.S:1161 stack backtrace: CPU: 1 PID: 10164 Comm: modprobe Not tainted 5.2.0-rc5-next-20190621 #20 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x172/0x1f0 lib/dump_stack.c:113 print_irq_inversion_bug.part.0+0x2c5/0x2d2 kernel/locking/lockdep.c:3101 print_irq_inversion_bug kernel/locking/lockdep.c:3102 [inline] check_usage_forwards.cold+0x20/0x29 kernel/locking/lockdep.c:3126 mark_lock_irq kernel/locking/lockdep.c:3239 [inline] mark_lock+0x4fa/0x11e0 kernel/locking/lockdep.c:3617 mark_usage kernel/locking/lockdep.c:3488 [inline] __lock_acquire+0x1e8c/0x4680 kernel/locking/lockdep.c:3839 lock_acquire+0x190/0x410 kernel/locking/lockdep.c:4418 __raw_spin_lock_irq include/linux/spinlock_api_smp.h:128 [inline] _raw_spin_lock_irq+0x60/0x80 kernel/locking/spinlock.c:167 spin_lock_irq include/linux/spinlock.h:363 [inline] free_ioctx_users+0x2d/0x490 fs/aio.c:620 percpu_ref_put_many include/linux/percpu-refcount.h:293 [inline] percpu_ref_put include/linux/percpu-refcount.h:309 [inline] percpu_ref_call_confirm_rcu lib/percpu-refcount.c:130 [inline] percpu_ref_switch_to_atomic_rcu+0x4c0/0x570 lib/percpu-refcount.c:165 __rcu_reclaim kernel/rcu/rcu.h:222 [inline] rcu_do_batch kernel/rcu/tree.c:2092 [inline] invoke_rcu_callbacks kernel/rcu/tree.c:2310 [inline] rcu_core+0xa3a/0x15a0 kernel/rcu/tree.c:2291 __do_softirq+0x260/0x9c8 kernel/softirq.c:292 invoke_softirq kernel/softirq.c:373 [inline] irq_exit+0x19b/0x1e0 kernel/softirq.c:413 exiting_irq arch/x86/include/asm/apic.h:536 [inline] smp_apic_timer_interrupt+0x16d/0x5b0 arch/x86/kernel/apic/apic.c:1068 apic_timer_interrupt+0xf/0x20 arch/x86/entry/entry_64.S:806 RIP: 0010:__sanitizer_cov_trace_const_cmp4+0xd/0x20 kernel/kcov.c:189 Code: d6 0f b7 f7 bf 03 00 00 00 48 89 e5 48 8b 4d 08 e8 d8 fe ff ff 5d c3 66 0f 1f 44 00 00 55 89 f2 89 fe bf 05 00 00 00 48 89 e5 <48> 8b 4d 08 e8 ba fe ff ff 5d c3 0f 1f 84 00 00 00 00 00 55 48 89 RSP: 0000:ffff8880576e7a38 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13 RAX: ffff888059de6100 RBX: 0000000000000001 RCX: 000000000000000b RDX: 0000000000000001 RSI: 0000000000000000 RDI: 0000000000000005 RBP: ffff8880576e7a38 R08: ffff888059de6100 R09: fffff9400005a157 R10: fffff9400005a156 R11: ffffea00002d0ab3 R12: ffffffff879585c0 R13: 0000000000000001 R14: 0000000000000001 R15: 000000000000000b __this_cpu_preempt_check+0x31/0x2a0 lib/smp_processor_id.c:16 __mod_lruvec_state+0x132/0x310 mm/memcontrol.c:763 __mod_lruvec_page_state include/linux/memcontrol.h:710 [inline] page_add_file_rmap+0x561/0xd60 mm/rmap.c:1208 alloc_set_pte+0x381/0x15e0 mm/memory.c:3337 filemap_map_pages+0x1138/0x13e0 mm/filemap.c:2648 do_fault_around mm/memory.c:3480 [inline] do_read_fault mm/memory.c:3514 [inline] do_fault mm/memory.c:3648 [inline] handle_pte_fault mm/memory.c:3879 [inline] __handle_mm_fault+0x3249/0x3f20 mm/memory.c:4003 handle_mm_fault+0x3b7/0xaa0 mm/memory.c:4040 do_user_addr_fault arch/x86/mm/fault.c:1438 [inline] __do_page_fault+0x531/0xdc0 arch/x86/mm/fault.c:1504 do_page_fault+0x71/0x5e1 arch/x86/mm/fault.c:1535 page_fault+0x1e/0x30 arch/x86/entry/entry_64.S:1161 RIP: 0033:0x7f87be364f84 Code: 10 4d 89 4b 18 5b 5d c3 66 66 66 66 66 2e 0f 1f 84 00 00 00 00 00 48 83 ec 08 48 8b 87 98 02 00 00 48 85 c0 74 5f 48 8b 40 08 <8b> 08 89 8f ec 02 00 00 8b 50 08 44 8b 40 04 8d 72 ff 85 d6 75 72 RSP: 002b:00007ffcd8e417a0 EFLAGS: 00010202 RAX: 00007f87be146638 RBX: 00007f87be57a4c8 RCX: 00007f87be3721d7 RDX: 00007f87be35ae40 RSI: 0000000000000030 RDI: 00007f87be57a4c8 RBP: 00007ffcd8e41910 R08: 0000000070000029 R09: 000000006ffffdff R10: 000000006ffffeff R11: 0000000000000246 R12: 00007ffcd8e419f8 R13: 000000006fffff48 R14: 00007ffcd8e416f0 R15: 00007ffcd8e41690 TCP: request_sock_TCPv6: Possible SYN flooding on port 20002. Sending cookies. Check SNMP counters. TCP: request_sock_TCPv6: Possible SYN flooding on port 20002. Sending cookies. Check SNMP counters. TCP: request_sock_TCPv6: Possible SYN flooding on port 20002. Sending cookies. Check SNMP counters.