witness: lock order reversal: 1st 0xfffffd8078268638 fdlock (&newfdp->fd_fd.fd_lock) 2nd 0xfffffd8078bda700 inode (&ip->i_lock) lock order data w2 -> w1 missing lock order data w1 -> w2 missing Stopped at db_enter+0x18: addq $0x8,%rsp ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic the kernel did not panic ddb{1}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 witness_checkorder(fffffd8078bda700,9,0) at witness_checkorder+0x108b rw_enter(fffffd8078bda6f0,1) at rw_enter+0xd4 rrw_enter(fffffd8078bda6f0,1) at rrw_enter+0x88 sys/kern/kern_rwlock.c:461 VOP_LOCK(fffffd80698d7750,2001) at VOP_LOCK+0x87 sys/kern/vfs_vops.c:614 vn_lock(fffffd80698d7750,2001) at vn_lock+0x84 sys/kern/vfs_vnops.c:579 vget(fffffd80698d7750,2001) at vget+0x1f7 sys/kern/vfs_subr.c:676 ktrwriteraw(ffff800021272a80,fffffd80698d7750,fffffd807f7d84e0,ffff8000244e1150,ffff8000244e1130) at ktrwriteraw+0x138 sys/kern/kern_ktrace.c:659 ktrstruct(ffff800021272a80,ffffffff823b92f4,ffff8000244e1238,8) at ktrstruct+0x169 ktrwrite2 sys/kern/kern_ktrace.c:627 [inline] ktrstruct(ffff800021272a80,ffffffff823b92f4,ffff8000244e1238,8) at ktrstruct+0x169 sys/kern/kern_ktrace.c:311 sys_socketpair(ffff800021272a80,ffff8000244e12a8,ffff8000244e12f0) at sys_socketpair+0x3ed sys/kern/uipc_syscalls.c:470 syscall(ffff8000244e1370) at syscall+0x5bf mi_syscall sys/sys/syscall_mi.h:102 [inline] syscall(ffff8000244e1370) at syscall+0x5bf sys/arch/amd64/amd64/trap.c:590 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xbee9fc74420, count: -12 ddb{1}> show registers rdi 0xffff8000234c0000 rsi 0xc1c1 __ALIGN_SIZE+0xb1c1 rbp 0xffff8000244e0d60 rbx 0x3 rdx 0xffff8000234c0000 rcx 0xc1c0 __ALIGN_SIZE+0xb1c0 rax 0xffffffff81d2dab7 db_enter+0x17 r8 0xffffffff81fdeae1 witness_checkorder+0x1061 r9 0x5 r10 0xccd9fa041cb2d945 r11 0x1dee2d2314a96f7a r12 0 r13 0xfffffd8078bda700 r14 0 r15 0xfffffd8002cf56c0 rip 0xffffffff81d2dab8 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff8000244e0d50 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb{1}> show proc PROC (syz-executor.1) pid=387157 stat=onproc flags process=10 proc=4000001 pri=32, usrpri=86, nice=20 forw=0xffffffffffffffff, list=0xffff800021273a40,0xffffffff827e0ab0 process=0xffff800021224010 user=0xffff8000244dc000, vmspace=0xfffffd806a39e180 estcpu=36, cpticks=1, pctcpu=0.0 user=0, sys=0, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 21171 497147 40018 60928 2 0x10 syz-executor.1 *21171 387157 40018 60928 7 0x4000011 syz-executor.1 33139 203220 0 0 3 0x14200 bored sosplice 40018 63660 19754 0 3 0x82 nanoslp syz-executor.1 52987 287009 19754 0 3 0x82 nanoslp syz-executor.0 19754 418030 19898 0 3 0x82 thrsleep syz-fuzzer 19754 34424 19898 0 3 0x4000082 thrsleep syz-fuzzer 19754 206370 19898 0 3 0x4000082 thrsleep syz-fuzzer 19754 324178 19898 0 3 0x4000082 thrsleep syz-fuzzer 19754 471170 19898 0 3 0x4000082 thrsleep syz-fuzzer 19754 78596 19898 0 3 0x4000082 thrsleep syz-fuzzer 19754 126292 19898 0 3 0x4000082 thrsleep syz-fuzzer 19754 128063 19898 0 3 0x4000082 kqread syz-fuzzer 19898 317140 27407 0 3 0x10008a sigsusp ksh 27407 494354 5972 0 3 0x92 select sshd 38126 140567 1 0 3 0x100083 ttyin getty 5972 92467 1 0 3 0x80 select sshd 43804 130674 76069 74 3 0x100092 bpf pflogd 76069 462744 1 0 3 0x80 netio pflogd 8101 277801 54302 73 3 0x100090 kqread syslogd 54302 169308 1 0 3 0x100082 netio syslogd 21659 443709 1 77 3 0x100090 poll dhclient 64366 434928 1 0 3 0x80 poll dhclient 39308 202006 0 0 3 0x14200 bored smr 57204 176542 0 0 2 0x14200 zerothread 94442 225598 0 0 3 0x14200 aiodoned aiodoned 86935 190080 0 0 3 0x14200 syncer update 70692 434988 0 0 3 0x14200 cleaner cleaner 62334 279789 0 0 7 0x14200 reaper 5618 4446 0 0 3 0x14200 pgdaemon pagedaemon 25348 139173 0 0 3 0x14200 bored crynlk 15186 438508 0 0 3 0x14200 bored crypto 99077 267143 0 0 3 0x14200 bored viomb 23051 176205 0 0 3 0x40014200 acpi0 acpi0 35509 347577 0 0 3 0x40014200 idle1 52984 31213 0 0 3 0x14200 bored softnet 70285 372051 0 0 3 0x14200 bored systqmp 26636 476343 0 0 3 0x14200 bored systq 2384 125138 0 0 3 0x40014200 bored softclock 99769 107073 0 0 3 0x40014200 idle0 1 100259 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{1}> show all locks Process 21171 (syz-executor.1) thread 0xffff800021272a80 (387157) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff827f3e18) #0 witness_lock+0x4b0 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4b0 sys/kern/subr_witness.c:1182 #1 ktrstruct+0xee #2 sys_socketpair+0x3ed sys/kern/uipc_syscalls.c:470 #3 syscall+0x5bf mi_syscall sys/sys/syscall_mi.h:102 [inline] #3 syscall+0x5bf sys/arch/amd64/amd64/trap.c:590 #4 Xsyscall+0x128 exclusive rwlock fdlock r = 0 (0xfffffd8078268638) #0 witness_lock+0x4b0 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4b0 sys/kern/subr_witness.c:1182 #1 sys_socketpair+0x219 #2 syscall+0x5bf mi_syscall sys/sys/syscall_mi.h:102 [inline] #2 syscall+0x5bf sys/arch/amd64/amd64/trap.c:590 #3 Xsyscall+0x128 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10150 6495K 7266K 78643K 13613 0 pcb 13 8K 8K 78643K 224 0 rtable 110 3K 4K 78643K 385 0 ifaddr 53 12K 13K 78643K 139 0 counters 44 34K 34K 78643K 82 0 ioctlops 0 0K 4K 78643K 1494 0 iov 0 0K 24K 78643K 29 0 mount 1 1K 1K 78643K 1 0 log 0 0K 0K 78643K 1 0 vnodes 1228 77K 77K 78643K 2184 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 9K 78643K 24 0 VM map 2 1K 1K 78643K 2 0 sem 12 0K 0K 78643K 311 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1697 195K 286K 78643K 12598 0 file desc 5 13K 25K 78643K 2339 0 sigio 0 0K 0K 78643K 48 0 proc 61 63K 95K 78643K 522 0 subproc 32 2K 2K 78643K 34 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 164 0 in_multi 33 2K 2K 78643K 108 0 ether_multi 1 0K 0K 78643K 34 0 mrt 0 0K 0K 78643K 20 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 49 228K 228K 78643K 49 0 exec 0 0K 2K 78643K 387 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 7 26K 26K 78643K 7 0 UVM amap 244 192K 192K 78643K 29240 0 UVM aobj 93 3K 3K 78643K 110 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 0K 78643K 200 0 NDP 8 0K 0K 78643K 31 0 temp 114 3982K 4067K 78643K 8052 0 kqueue 3 4K 8K 78643K 98 0 SYN cache 2 16K 16K 78643K 2 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 20 0 0 1 0 1 1 0 8 0 rtpcb 120 53 0 51 1 0 1 1 0 8 0 rtentry 112 56 0 11 2 0 2 2 0 8 0 unpcb 120 408 0 394 1 0 1 1 0 8 0 syncache 296 4 0 4 1 1 0 1 0 8 0 tcpqe 32 187 0 187 1 1 0 1 0 8 0 tcpcb 736 355 0 351 8 7 1 4 0 8 0 arp 120 7 0 0 1 0 1 1 0 8 0 inpcb 304 985 0 979 1 0 1 1 0 8 0 rttmr 72 10 0 10 1 1 0 1 0 8 0 nd6 48 6 0 0 1 0 1 1 0 8 0 pkpcb 40 18 0 18 2 2 0 1 0 8 0 kcovpl 48 2 0 0 1 0 1 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfstitem 24 44 0 30 1 0 1 1 0 8 0 pfstkey 112 44 0 30 1 0 1 1 0 8 0 pfstate 320 44 0 30 4 1 3 3 0 8 0 pfrule 1360 21 0 16 2 1 1 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 188 0 0 12 0 12 12 0 8 0 art_table 32 189 0 0 2 0 2 2 0 8 0 art_node 16 55 0 14 1 0 1 1 0 8 0 sysvmsgpl 40 13 0 13 1 1 0 1 0 8 0 semapl 112 309 0 299 1 0 1 1 0 8 0 shmpl 112 107 0 17 3 0 3 3 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 4638 0 3235 89 1 88 89 0 8 0 ffsino 272 4638 0 3235 95 0 95 95 0 8 0 nchpl 144 7574 0 5963 60 0 60 60 0 8 0 uvmvnodes 72 5266 0 0 96 0 96 96 0 8 0 vnodes 224 5266 0 0 310 0 310 310 0 8 0 namei 1024 18238 0 18238 1 0 1 1 0 8 1 percpumem 16 52 0 19 1 0 1 1 0 8 0 vcpupl 1984 10 0 0 2 0 2 2 0 8 0 vmpool 560 10 0 0 1 0 1 1 0 8 0 scsiplug 72 4 0 4 1 1 0 1 0 8 0 scxspl 216 21475 0 21475 9 8 1 8 0 8 1 plimitpl 152 92 0 84 1 0 1 1 0 8 0 sigapl 424 2555 0 2522 4 0 4 4 0 8 0 futexpl 56 16965 0 16965 1 0 1 1 0 8 1 knotepl 112 159 0 139 1 0 1 1 0 8 0 kqueuepl 168 1001 0 999 1 0 1 1 0 8 0 pipepl 336 110 0 99 3 2 1 2 0 8 0 fdescpl 496 2538 0 2522 3 0 3 3 0 8 0 filepl 152 8302 0 8196 5 0 5 5 0 8 0 lockfpl 104 253 0 252 1 0 1 1 0 8 0 lockfspl 48 100 0 99 1 0 1 1 0 8 0 sessionpl 144 18 0 7 1 0 1 1 0 8 0 pgrppl 48 18 0 7 1 0 1 1 0 8 0 ucredpl 96 935 0 924 1 0 1 1 0 8 0 zombiepl 144 2522 0 2521 1 0 1 1 0 8 0 processpl 1080 2555 0 2521 3 0 3 3 0 8 0 procpl 672 5258 0 5215 4 0 4 4 0 8 0 sosppl 168 53 0 53 4 4 0 1 0 8 0 sockpl 480 1464 0 1442 6 3 3 4 0 8 0 mcl64k 65536 7 0 0 1 0 1 1 0 8 0 mcl16k 16384 13 0 0 2 0 2 2 0 8 0 mcl12k 12288 18 0 0 2 0 2 2 0 8 0 mcl9k 9216 4 0 0 1 0 1 1 0 8 0 mcl8k 8192 6 0 0 1 0 1 1 0 8 0 mcl4k 4096 13 0 0 2 0 2 2 0 8 0 mcl2k 2048 195 0 0 18 0 18 18 0 8 0 mtagpl 96 187 0 0 5 0 5 5 0 8 0 mbufpl 256 581 0 0 33 0 33 33 0 8 0 bufpl 280 6075 0 173 422 0 422 422 0 8 0 anonpl 24 620862 0 613541 46 1 45 45 0 186 0 amapchunkpl 152 68648 0 68215 22 5 17 18 0 158 0 amappl16 200 4294 0 4138 10 1 9 9 0 8 0 amappl15 192 63 0 57 1 0 1 1 0 8 0 amappl14 184 4 0 1 1 0 1 1 0 8 0 amappl13 176 1187 0 1183 1 0 1 1 0 8 0 amappl12 168 37 0 27 1 0 1 1 0 8 0 amappl11 160 52 0 39 1 0 1 1 0 8 0 amappl10 152 28 0 22 1 0 1 1 0 8 0 amappl9 144 1440 0 1437 1 0 1 1 0 8 0 amappl8 136 347 0 297 2 0 2 2 0 8 0 amappl7 128 1212 0 1201 1 0 1 1 0 8 0 amappl6 120 104 0 91 1 0 1 1 0 8 0 amappl5 112 1288 0 1276 1 0 1 1 0 8 0 amappl4 104 549 0 521 1 0 1 1 0 8 0 amappl3 96 2447 0 2439 1 0 1 1 0 8 0 amappl2 88 465 0 407 2 0 2 2 0 8 0 amappl1 80 44838 0 44403 13 3 10 12 0 8 0 amappl 88 28790 0 28637 4 0 4 4 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 64 109 0 17 2 0 2 2 0 8 0 uaddrrnd 24 2548 0 2522 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 2548 0 2522 1 0 1 1 0 8 0 vmmpekpl 168 15814 0 15781 2 0 2 2 0 8 0 vmmpepl 168 282476 0 280950 73 5 68 69 0 357 0 vmsppl 368 2547 0 2521 3 0 3 3 0 8 0 rwobjpl 56 49419 0 48520 14 1 13 13 0 8 0 pdppl 4096 5103 0 5052 68 17 51 52 0 8 0 pvpl 32 1348321 0 1338012 141 51 90 129 0 265 2 pmappl 232 2547 0 2521 3 1 2 2 0 8 0 extentpl 40 58 0 40 1 0 1 1 0 8 0 phpool 112 345 0 30 10 0 10 10 0 8 0 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+0x1a: addq $0x8,%rsp ddb{0}> trace x86_ipi_db(ffffffff8273aff0) at x86_ipi_db+0x1a sys/arch/amd64/amd64/db_interface.c:352 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 __mp_lock(ffffffff827f3c10) at __mp_lock+0x122 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff827f3c10) at __mp_lock+0x122 sys/kern/kern_lock.c:147 softintr_dispatch(0) at softintr_dispatch+0x4e sys/arch/amd64/amd64/softintr.c:90 Xsoftclock() at Xsoftclock+0x1f __mp_lock(ffffffff827f3c10) at __mp_lock+0x122 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff827f3c10) at __mp_lock+0x122 sys/kern/kern_lock.c:147 uvm_unmap_detach(ffff8000211b1ab0,1) at uvm_unmap_detach+0xfc sys/uvm/uvm_map.c:1585 uvm_map_teardown(fffffd806a39e2f0) at uvm_map_teardown+0x262 sys/uvm/uvm_map.c:2761 uvmspace_free(fffffd806a39e2f0) at uvmspace_free+0xa6 sys/uvm/uvm_map.c:3655 reaper(ffff8000211b3ce8) at reaper+0x189 sys/kern/kern_exit.c:462 end trace frame: 0x0, count: -11 ddb{0}> machine ddbcpu 1 Stopped at db_enter+0x18: addq $0x8,%rsp ddb{1}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 witness_checkorder(fffffd8078bda700,9,0) at witness_checkorder+0x108b rw_enter(fffffd8078bda6f0,1) at rw_enter+0xd4 rrw_enter(fffffd8078bda6f0,1) at rrw_enter+0x88 sys/kern/kern_rwlock.c:461 VOP_LOCK(fffffd80698d7750,2001) at VOP_LOCK+0x87 sys/kern/vfs_vops.c:614 vn_lock(fffffd80698d7750,2001) at vn_lock+0x84 sys/kern/vfs_vnops.c:579 vget(fffffd80698d7750,2001) at vget+0x1f7 sys/kern/vfs_subr.c:676 ktrwriteraw(ffff800021272a80,fffffd80698d7750,fffffd807f7d84e0,ffff8000244e1150,ffff8000244e1130) at ktrwriteraw+0x138 sys/kern/kern_ktrace.c:659 ktrstruct(ffff800021272a80,ffffffff823b92f4,ffff8000244e1238,8) at ktrstruct+0x169 ktrwrite2 sys/kern/kern_ktrace.c:627 [inline] ktrstruct(ffff800021272a80,ffffffff823b92f4,ffff8000244e1238,8) at ktrstruct+0x169 sys/kern/kern_ktrace.c:311 sys_socketpair(ffff800021272a80,ffff8000244e12a8,ffff8000244e12f0) at sys_socketpair+0x3ed sys/kern/uipc_syscalls.c:470 syscall(ffff8000244e1370) at syscall+0x5bf mi_syscall sys/sys/syscall_mi.h:102 [inline] syscall(ffff8000244e1370) at syscall+0x5bf sys/arch/amd64/amd64/trap.c:590 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xbee9fc74420, count: -12