witness: lock order reversal: 1st 0xffff8000fffea6a0 fdlock (&newfdp->fd_fd.fd_lock) 2nd 0xfffffd806ff95d70 inode (&ip->i_lock) lock order [1] fdlock (&newfdp->fd_fd.fd_lock) -> [2] inode (&ip->i_lock) lock order data 0xffffffff8338a33b -> 0xffffffff8337ffd2 is missing lock order [2] inode (&ip->i_lock) -> [3] sbufrcv (&so->so_rcv.sb_lock) #0 rw_do_enter_write+0xba sys/kern/kern_rwlock.c:234 #1 sblock+0xb6 sys/kern/uipc_socket2.c:536 #2 soreceive+0x27d sys/kern/uipc_socket.c:878 #3 fifo_read+0x117 sys/miscfs/fifofs/fifo_vnops.c:264 #4 VOP_READ+0x101 sys/kern/vfs_vops.c:227 #5 vn_rdwr+0x15b sys/kern/vfs_vnops.c:-1 #6 vndsetcred+0xa1 sys/dev/vnd.c:685 #7 vndioctl+0xdfc sys/dev/vnd.c:486 #8 VOP_IOCTL+0xac sys/kern/vfs_vops.c:264 #9 vn_ioctl+0xf8 sys/kern/vfs_vnops.c:531 #10 sys_ioctl+0x674 sys/kern/sys_generic.c:-1 #11 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #11 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765 #12 Xsyscall+0x128 lock order [3] sbufrcv (&so->so_rcv.sb_lock) -> [1] fdlock (&newfdp->fd_fd.fd_lock) #0 rw_do_enter_write+0xba sys/kern/kern_rwlock.c:234 #1 unp_externalize+0x3cf sys/kern/uipc_usrreq.c:1074 #2 soreceive+0xc24 sys/kern/uipc_socket.c:1017 #3 recvit+0x40b sys/kern/uipc_syscalls.c:1078 #4 sys_recvmsg+0x1bf sys/kern/uipc_syscalls.c:878 #5 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #5 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765 #6 Xsyscall+0x128 Stopped at db_enter+0x25: addq $0x8,%rsp ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic the kernel did not panic ddb{0}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438 witness_checkorder(fffffd806ff95d70,9,0) at witness_checkorder+0x10b9 sys/kern/subr_witness.c:-1 rw_do_enter_write(fffffd806ff95d58,1) at rw_do_enter_write+0xba sys/kern/kern_rwlock.c:234 rrw_enter(fffffd806ff95d58,1) at rrw_enter+0xc6 sys/kern/kern_rwlock.c:621 VOP_LOCK(fffffd80700a3370,2001) at VOP_LOCK+0xa3 sys/kern/vfs_vops.c:527 vn_lock(fffffd80700a3370,2001) at vn_lock+0xa4 sys/kern/vfs_vnops.c:570 vn_closefile(fffffd806c921c38,ffff80002a270548) at vn_closefile+0x111 vn_close sys/kern/vfs_vnops.c:292 [inline] vn_closefile(fffffd806c921c38,ffff80002a270548) at vn_closefile+0x111 sys/kern/vfs_vnops.c:615 fdrop(fffffd806c921c38,ffff80002a270548) at fdrop+0x121 sys/kern/kern_descrip.c:1280 knote_remove(ffff80002a270548,fffffd806f804e00,fffffd806f804e90,3,0) at knote_remove+0x449 sys/kern/kern_event.c:2168 knote_fdclose(ffff80002a270548,3) at knote_fdclose+0xf9 sys/kern/kern_event.c:2203 fdrelease(ffff80002a270548,3) at fdrelease+0xf9 sys/kern/kern_descrip.c:760 syscall(ffff80003c40e000) at syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] syscall(ffff80003c40e000) at syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x9250a5de50, count: -13 ddb{0}> show registers rdi 0 rsi 0x80000 acpi_pdirpa+0x6be71 rbp 0xffff80003c40db10 rbx 0 rdx 0xffff8000014d1780 rcx 0xffff80002a270548 rax 0x7ffff acpi_pdirpa+0x6be70 r8 0xffff80003c40d9f0 r9 0x8080808080808080 r10 0x7486688d3bcfa273 r11 0xe5f821d3ded11f98 r12 0xfffffd8003f40d00 r13 0xfffffd800472cd78 r14 0x3 r15 0xffffffff834145c3 substchar+0x7bc8 rip 0xffffffff82bb41e5 db_enter+0x25 cs 0x8 rflags 0x246 rsp 0xffff80003c40db00 ss 0x10 db_enter+0x25: addq $0x8,%rsp ddb{0}> show proc PROC (syz-executor) tid=295956 pid=35363 tcnt=4 stat=onproc flags process=0 proc=4000000 runpri=32, usrpri=50, slppri=32, nice=20 wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0 forw=0xffffffffffffffff, list=0xffff80002a2707e0,0xffff80002a270d20 process=0xffff8000fffef038 user=0xffff80003c409000, vmspace=0xfffffd806c94f9a8 estcpu=36, cpticks=0, pctcpu=0.0, user=0, sys=0, intr=0 ddb{0}> ps PID TID PPID UID S FLAGS WAIT COMMAND 35363 300006 59321 0 3 0x80 nanoslp syz-executor *35363 295956 59321 0 7 0x4000000 syz-executor 35363 339912 59321 0 3 0x4000080 fsleep syz-executor 35363 222124 59321 0 3 0x4000080 fsleep syz-executor 8513 414497 74049 0 3 0x80 nanoslp syz-executor 8513 438417 74049 0 3 0x4000080 wsevent_read syz-executor 8513 6575 74049 0 3 0x4000080 fsleep syz-executor 12328 44690 17889 0 3 0x80 nanoslp syz-executor 12328 215687 17889 0 3 0x4000080 lockf syz-executor 12328 123570 17889 0 3 0x4000080 fsleep syz-executor 59321 381015 63114 0 3 0x82 nanoslp syz-executor 45495 64712 0 0 3 0x14200 acct acct 58776 388183 8364 0 3 0x100082 sbwait ndp 8364 453837 28172 0 3 0x10008a sigsusp sh 74049 166324 63114 0 3 0x82 nanoslp syz-executor 33504 106066 63114 0 3 0x82 nanoslp syz-executor 28172 28297 63114 0 3 0x82 wait syz-executor 25389 484600 63114 0 3 0x82 wait syz-executor 17889 258539 63114 0 3 0x82 nanoslp syz-executor 93607 422903 63114 0 3 0x82 nanoslp syz-executor 7518 52895 63114 0 3 0x82 nanoslp syz-executor 63114 365292 84198 0 3 0x82 kqread syz-executor 84198 373089 97560 0 3 0x10008a sigsusp ksh 97560 143752 84884 0 3 0x98 kqread sshd-session 84884 289909 99657 0 3 0x92 kqread sshd-session 15753 181067 1 0 3 0x100083 ttyin getty 99657 55382 1 0 3 0x88 kqread sshd 65314 92220 32295 74 3 0x1100092 bpf pflogd 32295 463045 1 0 3 0x80 sbwait pflogd 8626 340697 82693 73 3 0x1100090 kqread syslogd 82693 508989 1 0 3 0x100082 sbwait syslogd 10007 66329 1 0 3 0x100080 kqread resolvd 56397 456127 5909 77 3 0x100092 kqread dhcpleased 41509 64124 5909 77 3 0x100092 kqread dhcpleased 5909 407236 1 0 3 0x80 kqread dhcpleased 40052 261342 0 0 3 0x14200 bored smr 48431 478391 0 0 3 0x14200 pgzero zerothread 50411 326877 0 0 3 0x14200 aiodoned aiodoned 85624 222837 0 0 3 0x14200 syncer update 16967 293353 0 0 3 0x14200 cleaner cleaner 42631 29089 0 0 3 0x14200 reaper reaper 85320 390501 0 0 3 0x14200 pgdaemon pagedaemon 34911 137956 0 0 3 0x14200 bored viomb 68345 405942 0 0 3 0x40014200 acpi0 acpi0 51618 308294 0 0 7 0x40014200 idle1 87365 203418 0 0 3 0x14200 bored softnet1 9352 30812 0 0 3 0x14200 bored softnet0 3878 59316 0 0 3 0x14200 bored systqmp 42362 387851 0 0 3 0x14200 bored systq 47879 214943 0 0 3 0x14200 tmoslp softclockmp 34648 454883 0 0 3 0x40014200 tmoslp softclock 16125 103025 0 0 3 0x40014200 idle0 1 222998 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{0}> show all locks Process 35363 (syz-executor) thread 0xffff80002a270548 (295956) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff83a0a408) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 vn_closefile+0x41 sys/kern/vfs_vnops.c:608 #2 fdrop+0x121 sys/kern/kern_descrip.c:1280 #3 knote_remove+0x449 sys/kern/kern_event.c:2168 #4 knote_fdclose+0xf9 sys/kern/kern_event.c:2203 #5 fdrelease+0xf9 sys/kern/kern_descrip.c:760 #6 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #6 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765 #7 Xsyscall+0x128 exclusive rwlock fdlock r = 0 (0xffff8000fffea6a0) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 sys_close+0xd0 sys/kern/kern_descrip.c:778 #3 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #3 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765 #4 Xsyscall+0x128 ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10199 11102K 11538K 166960K 11747 0 pcb 17 15K 17K 166960K 80 0 rtable 229 8K 9K 166960K 424 0 pf 34 17K 22K 166960K 78 0 ifaddr 43 7K 7K 166960K 62 0 ifgroup 55 2K 2K 166960K 91 0 sysctl 4 1K 9K 166960K 8 0 counters 68 36K 37K 166960K 98 0 ioctlops 0 0K 4K 166960K 1509 0 iov 0 0K 16K 166960K 16 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1374 86K 87K 166960K 1616 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 1K 5K 166960K 5 0 VM map 2 1K 1K 166960K 2 0 sem 11 0K 0K 166960K 21 0 dirhash 12 2K 2K 166960K 12 0 ACPI 1692 195K 286K 166960K 12470 0 file desc 19 69K 110K 166960K 421 0 sigio 0 0K 0K 166960K 5 0 proc 71 115K 196K 166960K 579 0 subproc 72 4K 4K 166960K 81 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 44 0 in_multi 99 7K 7K 166960K 128 0 ether_multi 1 0K 0K 166960K 6 0 mrt 0 0K 0K 166960K 5 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 85 387K 387K 166960K 85 0 exec 0 0K 1K 166960K 435 0 fusefs mount 1 32K 32K 166960K 1 0 tdb 3 0K 0K 166960K 3 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 270 162K 178K 166960K 5712 0 UVM aobj 8 4K 4K 166960K 8 0 pinsyscall 45 90K 106K 166960K 1614 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 0K 166960K 20 0 NDP 12 0K 2K 166960K 41 0 temp 49 8669K 8739K 166960K 24638 0 kqueue 14 22K 30K 166960K 84 0 SYN cache 2 16K 16K 166960K 2 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 26 0 0 1 0 1 1 0 8 0 rtpcb 120 93 0 89 2 1 1 2 0 8 0 rtentry 176 124 0 24 5 0 5 5 0 8 0 unpcb 144 293 0 269 4 0 4 4 0 8 3 syncache 336 5 0 5 2 1 1 1 0 8 1 tcpcb 736 83 0 78 1 0 1 1 0 8 0 arp 136 20 0 2 1 0 1 1 0 8 0 inpcb 328 464 0 456 11 2 9 10 0 8 8 nd6 152 27 0 3 1 0 1 1 0 8 0 pkpcb 40 1 0 1 1 0 1 1 0 8 1 kcovpl 48 9 0 1 1 0 1 1 0 8 0 ppxss 1192 9 0 9 1 0 1 1 0 8 1 pffrag 232 4 0 1 1 0 1 1 0 482 0 pffrnode 88 3 0 1 1 0 1 1 0 8 0 pffrent 40 5 0 2 1 0 1 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfrktable 1344 3 0 3 1 0 1 1 0 8 1 pfanchor 1288 1 0 1 1 0 1 1 0 8 1 pfstitem 24 31 0 0 1 0 1 1 0 8 0 pfstkey 128 31 0 0 1 0 1 1 0 8 0 pfstate 448 31 0 0 4 0 4 4 0 8 0 pfrule 1344 22 0 17 2 1 1 2 0 8 0 rttmr 136 1 0 1 1 0 1 1 0 8 1 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 563 0 116 31 0 31 31 0 8 1 art_table 40 564 0 116 5 0 5 5 0 8 0 art_node 32 124 0 34 1 0 1 1 0 8 0 sysvmsgpl 40 6 0 2 1 0 1 1 0 8 0 semupl 112 3 0 3 1 0 1 1 0 8 1 semapl 112 17 0 8 1 0 1 1 0 8 0 shmpl 112 4 0 0 1 0 1 1 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 2046 0 536 95 0 95 95 0 8 0 ffsino 296 2046 0 536 117 0 117 117 0 8 0 nchpl 144 2610 0 907 64 0 64 64 0 8 0 rtmask 32 2 0 2 1 0 1 1 0 8 1 vnodes 216 2292 0 0 128 0 128 128 0 8 0 namei 1024 8356 0 8356 1 0 1 1 0 8 1 percpumem 16 64 0 15 1 0 1 1 0 8 0 vcpupl 3968 1 0 0 1 0 1 1 0 8 0 vmpool 840 1 0 0 1 0 1 1 0 8 0 pfiaddrpl 120 1 0 1 1 0 1 1 0 8 1 kstatmem 264 48 0 22 3 0 3 3 0 8 1 scsiplug 72 4 0 4 1 1 0 1 0 8 0 scxspl 216 11184 0 11184 10 2 8 8 1 8 8 plimitpl 152 103 0 86 1 0 1 1 0 8 0 sigapl 424 735 0 685 7 0 7 7 0 8 0 knotepl 120 747 0 0 23 0 23 23 0 8 0 kqueuepl 224 174 0 164 5 3 2 5 0 8 1 pipepl 344 139 0 112 3 0 3 3 0 8 0 fdescpl 528 718 0 685 3 0 3 3 0 8 0 filepl 160 3598 0 3359 15 1 14 15 0 8 3 lockfpl 104 124 0 120 1 0 1 1 0 8 0 lockfspl 48 56 0 53 1 0 1 1 0 8 0 sessionpl 144 25 0 16 1 0 1 1 0 8 0 pgrppl 48 44 0 26 1 0 1 1 0 8 0 ucredpl 104 362 0 347 1 0 1 1 0 8 0 zombiepl 144 689 0 685 1 0 1 1 0 8 0 processpl 1232 735 0 685 5 0 5 5 0 8 0 procpl 664 1204 0 1147 6 0 6 6 0 8 0 sosppl 176 1 0 1 1 0 1 1 0 8 1 sockpl 752 863 0 827 14 2 12 14 0 8 8 mcl64k 65536 3 0 0 1 0 1 1 0 8 0 mcl16k 16384 1 0 0 1 0 1 1 0 8 0 mcl12k 12288 1 0 0 1 0 1 1 0 8 0 mcl9k 9216 1 0 0 1 0 1 1 0 8 0 mcl8k 8192 2 0 0 1 0 1 1 0 8 0 mcl4k 4096 119 0 0 15 0 15 15 0 8 0 mcl2k 2048 50 0 0 7 0 7 7 0 8 0 mtagpl 96 53 0 0 2 0 2 2 0 8 0 mbufpl 256 368 0 0 23 0 23 23 0 8 0 bufpl 280 4216 0 132 292 0 292 292 0 8 0 anonpl 32 8523 0 0 69 0 69 69 0 246 0 amapchunkpl 152 29885 0 29342 36 0 36 36 0 158 12 amappl16 200 2613 0 2584 16 2 14 15 0 8 8 amappl15 192 4 0 4 1 1 0 1 0 8 0 amappl14 184 9 0 8 1 0 1 1 0 8 0 amappl13 176 446 0 444 1 0 1 1 0 8 0 amappl12 168 1101 0 1056 3 0 3 3 0 8 0 amappl11 160 3 0 3 1 1 0 1 0 8 0 amappl10 152 50 0 36 1 0 1 1 0 8 0 amappl9 144 251 0 251 1 1 0 1 0 8 0 amappl8 136 27 0 25 1 0 1 1 0 8 0 amappl7 128 91 0 90 1 0 1 1 0 8 0 amappl6 120 297 0 282 1 0 1 1 0 8 0 amappl5 112 75 0 65 1 0 1 1 0 8 0 amappl4 104 439 0 408 1 0 1 1 0 8 0 amappl3 96 2911 0 2808 4 1 3 3 0 8 0 amappl2 88 842 0 765 2 0 2 2 0 8 0 amappl1 80 10908 0 10283 15 0 15 15 0 8 0 amappl 88 4884 0 4700 5 0 5 5 0 92 0 uvmvnodes 80 107 0 0 3 0 3 3 0 8 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 7 0 0 1 0 1 1 0 8 0 uaddrrnd 24 718 0 685 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 718 0 685 1 0 1 1 0 8 0 vmmpekpl 168 7675 0 7636 3 0 3 3 0 8 0 vmmpepl 168 53518 0 51486 104 0 104 104 0 357 14 vmsppl 488 717 0 685 5 0 5 5 0 8 0 rwobjpl 80 17869 0 16770 29 0 29 29 0 8 3 pdppl 4096 1445 0 1371 102 26 76 85 0 8 2 pvpl 32 15522 0 0 126 0 126 126 0 265 0 pmappl 256 718 0 685 3 0 3 3 0 8 0 extentpl 40 45 0 27 1 0 1 1 0 8 0 phpool 112 294 0 29 8 0 8 8 0 8 0 ddb{0}> machine ddbcpu 0 Invalid cpu 0 ddb{0}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438 witness_checkorder(fffffd806ff95d70,9,0) at witness_checkorder+0x10b9 sys/kern/subr_witness.c:-1 rw_do_enter_write(fffffd806ff95d58,1) at rw_do_enter_write+0xba sys/kern/kern_rwlock.c:234 rrw_enter(fffffd806ff95d58,1) at rrw_enter+0xc6 sys/kern/kern_rwlock.c:621 VOP_LOCK(fffffd80700a3370,2001) at VOP_LOCK+0xa3 sys/kern/vfs_vops.c:527 vn_lock(fffffd80700a3370,2001) at vn_lock+0xa4 sys/kern/vfs_vnops.c:570 vn_closefile(fffffd806c921c38,ffff80002a270548) at vn_closefile+0x111 vn_close sys/kern/vfs_vnops.c:292 [inline] vn_closefile(fffffd806c921c38,ffff80002a270548) at vn_closefile+0x111 sys/kern/vfs_vnops.c:615 fdrop(fffffd806c921c38,ffff80002a270548) at fdrop+0x121 sys/kern/kern_descrip.c:1280 knote_remove(ffff80002a270548,fffffd806f804e00,fffffd806f804e90,3,0) at knote_remove+0x449 sys/kern/kern_event.c:2168 knote_fdclose(ffff80002a270548,3) at knote_fdclose+0xf9 sys/kern/kern_event.c:2203 fdrelease(ffff80002a270548,3) at fdrelease+0xf9 sys/kern/kern_descrip.c:760 syscall(ffff80003c40e000) at syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] syscall(ffff80003c40e000) at syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x9250a5de50, count: -13 ddb{0}> machine ddbcpu 1 Stopped at x86_ipi_db+0x27: addq $0x8,%rsp ddb{1}> trace x86_ipi_db(ffff8000299edff0) at x86_ipi_db+0x27 sys/arch/amd64/amd64/db_interface.c:394 x86_ipi_handler() at x86_ipi_handler+0xd9 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 acpicpu_idle() at acpicpu_idle+0x457 sys/dev/acpi/acpicpu_x86.c:1224 sched_idle(ffff8000299edff0) at sched_idle+0x391 sys/kern/kern_sched.c:191 end trace frame: 0x0, count: -5