witness: lock order reversal: 1st 0xffff80000142d2a0 sbufrcv (&so->so_rcv.sb_lock) 2nd 0xfffffd8068b8c580 inode (&ip->i_lock) lock order [1] sbufrcv (&so->so_rcv.sb_lock) -> [2] inode (&ip->i_lock) lock order data 0xffffffff832f9abd -> 0xffffffff8334590b is missing lock order [2] inode (&ip->i_lock) -> [1] sbufrcv (&so->so_rcv.sb_lock) #0 rw_do_enter_write+0xba sys/kern/kern_rwlock.c:234 #1 sblock+0xb6 sys/kern/uipc_socket2.c:536 #2 soreceive+0x27d sys/kern/uipc_socket.c:870 #3 fifo_read+0x117 sys/miscfs/fifofs/fifo_vnops.c:264 #4 VOP_READ+0x101 sys/kern/vfs_vops.c:227 #5 vn_rdwr+0x15b sys/kern/vfs_vnops.c:-1 #6 vndsetcred+0xa1 sys/dev/vnd.c:685 #7 vndioctl+0xdfb sys/dev/vnd.c:486 #8 VOP_IOCTL+0xac sys/kern/vfs_vops.c:264 #9 vn_ioctl+0xf8 sys/kern/vfs_vnops.c:531 #10 sys_ioctl+0x674 sys/kern/sys_generic.c:-1 #11 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #11 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:746 #12 Xsyscall+0x128 Stopped at db_enter+0x25: addq $0x8,%rsp ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic the kernel did not panic ddb{1}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438 witness_checkorder(fffffd8068b8c580,9,0) at witness_checkorder+0x10b9 sys/kern/subr_witness.c:-1 rw_do_enter_write(fffffd8068b8c568,1) at rw_do_enter_write+0xba sys/kern/kern_rwlock.c:234 rrw_enter(fffffd8068b8c568,1) at rrw_enter+0xc6 sys/kern/kern_rwlock.c:621 VOP_LOCK(fffffd806e11a8c0,2001) at VOP_LOCK+0xa3 sys/kern/vfs_vops.c:527 vn_lock(fffffd806e11a8c0,2001) at vn_lock+0xa4 sys/kern/vfs_vnops.c:570 vfs_getcwd_common(fffffd806e11a8c0,fffffd80621ff018,0,0,200,0,44fe00c032bcbf6) at vfs_getcwd_common+0xd1 sys/kern/vfs_getcwd.c:287 vn_isunder(fffffd806e11a8c0,fffffd80621ff018,ffff80002a314038) at vn_isunder+0x56 sys/kern/vfs_vnops.c:694 unp_externalize(fffffd8062ed8200,f0,0) at unp_externalize+0x26f sys/kern/uipc_usrreq.c:1090 soreceive(ffff80000142d1b0,ffff80003c459478,ffff80003c459428,0,ffff80003c459460,ffff80003c4595ec,5c1ec78f915974c8) at soreceive+0xc24 sys/kern/uipc_socket.c:1009 recvit(ffff80002a314038,5,ffff80003c4595c0,0,ffff80003c459670) at recvit+0x40b sys/kern/uipc_syscalls.c:1078 sys_recvmsg(ffff80002a314038,ffff80003c459720,ffff80003c459670) at sys_recvmsg+0x1bf sys/kern/uipc_syscalls.c:878 syscall(ffff80003c459720) at syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] syscall(ffff80003c459720) at syscall+0xbd4 sys/arch/amd64/amd64/trap.c:746 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xdcdd7b28ce0, count: -14 ddb{1}> show registers rdi 0 rsi 0x80000 acpi_pdirpa+0x6be71 rbp 0xffff80003c458f10 rbx 0 rdx 0xffff80000147dd80 rcx 0xffff80002a314038 rax 0x7ffff acpi_pdirpa+0x6be70 r8 0xffff80003c458df0 r9 0x8080808080808080 r10 0xf518831d7555a87d r11 0xa58ba22fd046cee5 r12 0xfffffd8003eff640 r13 0xfffffd80046e6580 r14 0x3 r15 0xffffffff833e2b64 substchar+0xf1da rip 0xffffffff831577a5 db_enter+0x25 cs 0x8 rflags 0x246 rsp 0xffff80003c458f00 ss 0 db_enter+0x25: addq $0x8,%rsp ddb{1}> show proc PROC (syz-executor) tid=122745 pid=13333 tcnt=3 stat=onproc flags process=1000000 proc=4000000 runpri=24, usrpri=50, slppri=24, nice=20 wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0 forw=0xffffffffffffffff, list=0xffff80002a3154f8,0xffff8000fffe2d30 process=0xffff80003c4be6b0 user=0xffff80003c454000, vmspace=0xfffffd806a1ba9a8 estcpu=36, cpticks=0, pctcpu=0.0, user=0, sys=0, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 68059 64213 17591 0 2 0 syz-executor 68059 119759 17591 0 3 0x4000000 smrbar syz-executor 91457 212859 66196 0 7 0 syz-executor 91457 486544 66196 0 3 0x4000080 fsleep syz-executor 13333 298586 32468 0 2 0x1000000 syz-executor *13333 122745 32468 0 7 0x5000000 syz-executor 13333 416941 32468 0 3 0x5000080 fsleep syz-executor 77144 412995 16874 0 2 0 syz-executor 77144 74462 16874 0 3 0x4000080 msgwait syz-executor 77144 7044 16874 0 3 0x4000080 ttyretype syz-executor 66039 376489 18411 0 3 0x80 nanoslp syz-executor 66039 263124 18411 0 3 0x4000080 kqsel syz-executor 66039 200107 18411 0 3 0x4000080 fsleep syz-executor 47360 52437 57907 0 3 0x80 nanoslp syz-executor 47360 176677 57907 0 3 0x4000080 fsleep syz-executor 47360 360003 57907 0 3 0x4000080 fsleep syz-executor 47360 314346 57907 0 3 0x4000080 ttyout syz-executor 82832 33276 0 0 3 0x14200 acct acct 70349 215510 0 0 3 0x14280 nfsidl nfsio 73547 211034 0 0 3 0x14280 nfsidl nfsio 96902 382075 0 0 3 0x14280 nfsidl nfsio 76709 490719 0 0 3 0x14280 nfsidl nfsio 90224 449116 0 0 3 0x14280 nfsidl nfsio 62180 206127 0 0 3 0x14280 nfsidl nfsio 84268 83942 0 0 3 0x14280 nfsidl nfsio 5585 473059 0 0 3 0x14280 nfsidl nfsio 86449 197277 0 0 3 0x14280 nfsidl nfsio 69166 144821 0 0 3 0x14280 nfsidl nfsio 12541 63693 0 0 3 0x14280 nfsidl nfsio 56876 400120 0 0 3 0x14280 nfsidl nfsio 43860 42539 0 0 3 0x14280 nfsidl nfsio 12118 31291 0 0 3 0x14280 nfsidl nfsio 77609 259868 0 0 3 0x14280 nfsidl nfsio 16935 363022 0 0 3 0x14280 nfsidl nfsio 94436 432670 0 0 3 0x14280 nfsidl nfsio 58387 15621 0 0 3 0x14280 nfsidl nfsio 77095 38611 0 0 3 0x14280 nfsidl nfsio 77715 164453 0 0 3 0x14280 nfsidl nfsio 53752 410450 0 0 3 0x14200 bored sosplice 46706 185268 35837 0 3 0x2 biowait syz-executor 57907 130522 35837 0 3 0x82 nanoslp syz-executor 66196 106325 35837 0 3 0x82 nanoslp syz-executor 32468 218492 35837 0 3 0x82 nanoslp syz-executor 72914 286956 35837 0 3 0x2 biowait syz-executor 16874 191621 35837 0 3 0x82 nanoslp syz-executor 18411 451919 35837 0 3 0x82 nanoslp syz-executor 17591 339391 35837 0 3 0x82 nanoslp syz-executor 35837 236827 56827 0 3 0x82 kqread syz-executor 56827 494933 37602 0 3 0x10008a sigsusp ksh 37602 361193 46575 0 3 0x98 kqread sshd-session 46575 178519 1370 0 3 0x92 kqread sshd-session 33783 197143 1 0 3 0x100083 ttyin getty 1370 104876 1 0 3 0x88 kqread sshd 30077 160299 22367 74 3 0x1100092 bpf pflogd 22367 68812 1 0 3 0x80 sbwait pflogd 97293 358357 82461 73 3 0x1100090 kqread syslogd 82461 443019 1 0 3 0x100082 sbwait syslogd 19033 485155 1 0 3 0x100080 kqread resolvd 6051 291765 38837 77 3 0x100092 kqread dhcpleased 59346 447774 38837 77 3 0x100092 kqread dhcpleased 38837 241630 1 0 3 0x80 kqread dhcpleased 93553 447359 0 0 3 0x14200 bored smr 78616 54067 0 0 2 0x14200 zerothread 41370 441319 0 0 3 0x14200 aiodoned aiodoned 17942 427652 0 0 3 0x14200 syncer update 1922 497588 0 0 3 0x14200 cleaner cleaner 18106 323060 0 0 3 0x14200 reaper reaper 77547 499244 0 0 3 0x14200 pgdaemon pagedaemon 2429 207269 0 0 3 0x14200 bored viomb 65983 468874 0 0 3 0x40014200 acpi0 acpi0 23365 66674 0 0 3 0x40014200 idle1 1990 496194 0 0 3 0x14200 bored softnet1 61429 27935 0 0 2 0x14200 softnet0 53339 114464 0 0 3 0x14200 smrbar systqmp 66311 78833 0 0 3 0x14200 bored systq 35354 367400 0 0 3 0x14200 tmoslp softclockmp 57352 149667 0 0 3 0x40014200 tmoslp softclock 78670 392328 0 0 3 0x40014200 idle0 1 302654 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{1}> show all locks Process 68059 (syz-executor) thread 0xffff8000fffe2558 (119759) exclusive rwlock clonelk r = 0 (0xffffffff837bee88) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 if_clone_destroy+0x67 sys/net/if.c:-1 #3 ifioctl+0x59d sys/net/if.c:2092 #4 sys_ioctl+0x674 sys/kern/sys_generic.c:-1 #5 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #5 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:746 #6 Xsyscall+0x128 Process 91457 (syz-executor) thread 0xffff8000fffe2d20 (212859) exclusive rwlock uobjlk r = 0 (0xfffffd806b990088) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 uvm_fault+0x1e5 sys/uvm/uvm_fault.c:655 #3 upageflttrap+0xa9 sys/arch/amd64/amd64/trap.c:192 #4 usertrap+0x3c6 sys/arch/amd64/amd64/trap.c:603 #5 recall_trap+0x8 shared rwlock vmmaplk r = 0 (0xfffffd806ef754d8) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_read+0x3e8 sys/kern/kern_rwlock.c:413 #2 uvmfault_lookup+0x122 sys/uvm/uvm_fault.c:1902 #3 uvm_fault_check+0x4f sys/uvm/uvm_fault.c:699 #4 uvm_fault+0x106 sys/uvm/uvm_fault.c:633 #5 upageflttrap+0xa9 sys/arch/amd64/amd64/trap.c:192 #6 usertrap+0x3c6 sys/arch/amd64/amd64/trap.c:603 #7 recall_trap+0x8 Process 13333 (syz-executor) thread 0xffff80002a314038 (122745) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff83915fc0) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 unp_externalize+0x155 sys/kern/uipc_usrreq.c:-1 #2 soreceive+0xc24 sys/kern/uipc_socket.c:1009 #3 recvit+0x40b sys/kern/uipc_syscalls.c:1078 #4 sys_recvmsg+0x1bf sys/kern/uipc_syscalls.c:878 #5 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #5 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:746 #6 Xsyscall+0x128 exclusive rwlock sbufrcv r = 0 (0xffff80000142d2a0) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 sblock+0xb6 sys/kern/uipc_socket2.c:536 #3 soreceive+0x27d sys/kern/uipc_socket.c:870 #4 recvit+0x40b sys/kern/uipc_syscalls.c:1078 #5 sys_recvmsg+0x1bf sys/kern/uipc_syscalls.c:878 #6 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #6 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:746 #7 Xsyscall+0x128 Process 46706 (syz-executor) thread 0xffff80002a2702b0 (185268) exclusive rrwlock inode r = 0 (0xfffffd806e31d348) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 rrw_enter+0xc6 sys/kern/kern_rwlock.c:621 #3 VOP_LOCK+0xa3 sys/kern/vfs_vops.c:527 #4 ufs_ihashins+0x4f ufs_ihash sys/ufs/ufs/ufs_ihash.c:-1 [inline] #4 ufs_ihashins+0x4f sys/ufs/ufs/ufs_ihash.c:159 #5 ffs_vget+0x187 sys/ufs/ffs/ffs_vfsops.c:1232 #6 ffs_inode_alloc+0x279 sys/ufs/ffs/ffs_alloc.c:393 #7 ufs_mkdir+0xfc sys/ufs/ufs/ufs_vnops.c:1112 #8 VOP_MKDIR+0x101 sys/kern/vfs_vops.c:394 #9 domkdirat+0x179 sys/kern/vfs_syscalls.c:3113 #10 syscall+0xb17 mi_syscall sys/sys/syscall_mi.h:176 [inline] #10 syscall+0xb17 sys/arch/amd64/amd64/trap.c:746 #11 Xsyscall+0x128 exclusive rrwlock inode r = 0 (0xfffffd806be0a328) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 rrw_enter+0xc6 sys/kern/kern_rwlock.c:621 #3 VOP_LOCK+0xa3 sys/kern/vfs_vops.c:527 #4 vn_lock+0xa4 sys/kern/vfs_vnops.c:570 #5 vfs_lookup+0x11c sys/kern/vfs_lookup.c:-1 #6 namei+0x7ca sys/kern/vfs_lookup.c:250 #7 domkdirat+0x8b sys/kern/vfs_syscalls.c:3098 #8 syscall+0xb17 mi_syscall sys/sys/syscall_mi.h:176 [inline] #8 syscall+0xb17 sys/arch/amd64/amd64/trap.c:746 #9 Xsyscall+0x128 Process 72914 (syz-executor) thread 0xffff80002a2722a8 (286956) exclusive rrwlock inode r = 0 (0xfffffd8068b8cd98) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 rrw_enter+0xc6 sys/kern/kern_rwlock.c:621 #3 VOP_LOCK+0xa3 sys/kern/vfs_vops.c:527 #4 vn_lock+0xa4 sys/kern/vfs_vnops.c:570 #5 vget+0x2a2 sys/kern/vfs_subr.c:693 #6 ufs_ihashget+0x185 sys/ufs/ufs/ufs_ihash.c:98 #7 ffs_vget+0x8c sys/ufs/ffs/ffs_vfsops.c:1203 #8 ufs_lookup+0x1a36 sys/ufs/ufs/ufs_lookup.c:478 #9 VOP_LOOKUP+0x6e sys/kern/vfs_vops.c:85 #10 vfs_lookup+0x93a sys/kern/vfs_lookup.c:566 #11 namei+0x7ca sys/kern/vfs_lookup.c:250 #12 dounlinkat+0xc1 sys/kern/vfs_syscalls.c:1868 #13 syscall+0xb17 mi_syscall sys/sys/syscall_mi.h:176 [inline] #13 syscall+0xb17 sys/arch/amd64/amd64/trap.c:746 #14 Xsyscall+0x128 exclusive rrwlock inode r = 0 (0xfffffd806d1e1448) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 rrw_enter+0xc6 sys/kern/kern_rwlock.c:621 #3 VOP_LOCK+0xa3 sys/kern/vfs_vops.c:527 #4 vn_lock+0xa4 sys/kern/vfs_vnops.c:570 #5 vfs_lookup+0x11c sys/kern/vfs_lookup.c:-1 #6 namei+0x7ca sys/kern/vfs_lookup.c:250 #7 dounlinkat+0xc1 sys/kern/vfs_syscalls.c:1868 #8 syscall+0xb17 mi_syscall sys/sys/syscall_mi.h:176 [inline] #8 syscall+0xb17 sys/arch/amd64/amd64/trap.c:746 #9 Xsyscall+0x128 Process 53339 (systqmp) thread 0xffff8000ffffe298 (114464) shared rwlock systqmp r = 0 (0xffffffff837992e8) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 taskq_thread+0x12a sys/kern/kern_task.c:442 #2 proc_trampoline+0x10 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10207 11089K 12483K 166960K 12765 0 pcb 19 14K 16K 166960K 163 0 rtable 188 9K 9K 166960K 401 0 pf 37 18K 19K 166960K 116 0 ifaddr 39 6K 8K 166960K 86 0 ifgroup 55 2K 2K 166960K 133 0 sysctl 4 1K 9K 166960K 11 0 counters 68 36K 38K 166960K 156 0 ioctlops 0 0K 4K 166960K 1589 0 iov 0 0K 24K 166960K 28 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1489 94K 95K 166960K 2031 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 1K 9K 166960K 11 0 VM map 2 1K 1K 166960K 2 0 sem 11 1K 1K 166960K 15 0 dirhash 12 2K 2K 166960K 24 0 ACPI 1692 195K 286K 166960K 12470 0 file desc 16 57K 240K 166960K 630 0 sigio 0 0K 0K 166960K 7 0 proc 72 115K 180K 166960K 564 0 subproc 72 4K 4K 166960K 72 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 48 0 in_multi 84 6K 7K 166960K 116 0 ether_multi 1 0K 0K 166960K 3 0 mrt 1 0K 0K 166960K 9 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 85 387K 387K 166960K 85 0 exec 0 0K 1K 166960K 554 0 fusefs mount 1 32K 32K 166960K 1 0 pfkey data 0 0K 0K 166960K 2 0 tdb 3 0K 0K 166960K 3 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 234 167K 181K 166960K 7097 0 UVM aobj 20 4K 4K 166960K 23 0 pinsyscall 41 82K 104K 166960K 1682 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 0K 166960K 29 0 NDP 12 0K 1K 166960K 60 0 temp 48 8645K 8738K 166960K 29048 0 kqueue 14 22K 28K 166960K 110 0 SYN cache 2 16K 16K 166960K 2 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 26 0 0 1 0 1 1 0 8 0 rtpcb 120 65 0 62 1 0 1 1 0 8 0 rtentry 176 110 0 38 5 0 5 5 0 8 0 unpcb 144 475 0 456 6 4 2 6 0 8 1 syncache 336 8 0 8 1 1 0 1 0 8 0 tcpcb 736 127 0 121 2 1 1 2 0 8 0 arp 136 13 0 3 1 0 1 1 0 8 0 inpcb 328 509 0 498 8 0 8 8 0 8 5 nd6 152 17 0 4 1 0 1 1 0 8 0 pkpcb 40 2 0 2 2 1 1 1 0 8 1 kcovpl 48 8 0 0 1 0 1 1 0 8 0 ppxss 1192 38 0 38 1 0 1 1 0 8 1 pppxif 1504 2 0 2 1 0 1 1 0 8 1 pfstscr 40 1 0 1 1 0 1 1 0 8 1 pffrag 232 6 0 2 1 0 1 1 0 482 0 pffrnode 88 5 0 2 1 0 1 1 0 8 0 pffrent 40 12 0 8 1 0 1 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pftag 88 1 0 0 1 0 1 1 0 8 0 pfqueue 320 1 0 1 1 0 1 1 0 8 1 pfstitem 24 26 0 1 1 0 1 1 0 8 0 pfstkey 128 28 0 3 1 0 1 1 0 8 0 pfstate 384 27 0 2 3 0 3 3 0 8 0 pfrule 1344 23 0 17 2 1 1 2 0 8 0 rttmr 136 2 0 2 1 1 0 1 0 8 0 art_heap8 4096 2 0 0 2 0 2 2 0 8 0 art_heap4 256 512 0 187 30 2 28 30 0 8 3 art_table 40 514 0 187 5 0 5 5 0 8 0 art_node 32 110 0 46 1 0 1 1 0 8 0 sysvmsgpl 40 7 0 6 1 0 1 1 0 8 0 semupl 112 1 0 1 1 1 0 1 0 8 0 semapl 112 10 0 1 1 0 1 1 0 8 0 shmpl 112 20 0 3 1 0 1 1 0 8 0 dirhash 1024 25 0 8 3 0 3 3 0 8 0 dino2pl 256 2495 0 996 95 0 95 95 0 8 0 ffsino 296 2495 0 996 117 0 117 117 0 8 0 nchpl 144 3322 0 1636 63 0 63 63 0 8 0 rtmask 32 7 0 7 1 0 1 1 0 8 1 uvmvnodes 80 2880 0 0 59 0 59 59 0 8 0 vnodes 216 2880 0 0 160 0 160 160 0 8 0 namei 1024 11173 0 11172 2 1 1 2 0 8 0 percpumem 16 93 0 44 1 0 1 1 0 8 0 kstatmem 264 90 0 62 3 0 3 3 0 8 1 acpiwqpl 32 2 0 2 1 0 1 1 1 8 1 scsiplug 72 1 0 1 1 1 0 1 0 8 0 scxspl 216 11433 0 11431 10 2 8 8 1 8 7 plimitpl 152 123 0 105 1 0 1 1 0 8 0 sigapl 424 922 0 854 9 1 8 8 0 8 0 knotepl 120 451 0 0 14 0 14 14 0 8 0 kqueuepl 224 239 0 227 3 0 3 3 0 8 2 pipepl 344 155 0 128 4 1 3 4 0 8 0 fdescpl 528 884 0 854 3 0 3 3 0 8 0 filepl 160 4929 0 4705 17 1 16 16 0 8 6 lockfpl 104 573 0 570 3 1 2 3 0 8 1 lockfspl 48 271 0 268 2 1 1 2 0 8 0 sessionpl 144 22 0 13 1 0 1 1 0 8 0 pgrppl 48 34 0 17 1 0 1 1 0 8 0 ucredpl 104 557 0 544 1 0 1 1 0 8 0 zombiepl 144 854 0 854 1 0 1 1 0 8 1 processpl 1232 922 0 854 7 1 6 6 0 8 0 procpl 664 1711 0 1632 8 0 8 8 0 8 0 sockpl 752 1067 0 1034 19 8 11 17 0 8 7 mcl64k 65536 5 0 0 1 0 1 1 0 8 0 mcl16k 16384 2 0 0 1 0 1 1 0 8 0 mcl12k 12288 2 0 0 1 0 1 1 0 8 0 mcl8k 8192 2 0 0 1 0 1 1 0 8 0 mcl4k 4096 118 0 0 15 0 15 15 0 8 0 mcl2k 2048 75 0 0 10 0 10 10 0 8 0 mtagpl 96 4 0 0 1 0 1 1 0 8 0 mbufpl 256 385 0 0 24 0 24 24 0 8 0 bufpl 280 4181 0 125 290 0 290 290 0 8 0 anonpl 32 7962 0 0 65 0 65 65 0 246 0 amapchunkpl 152 23315 0 22793 30 3 27 27 0 158 3 amappl16 200 2538 0 2445 13 3 10 10 0 8 1 amappl15 192 4 0 4 2 2 0 1 0 8 0 amappl14 184 116 0 104 1 0 1 1 0 8 0 amappl13 176 1 0 1 1 1 0 1 0 8 0 amappl12 168 1522 0 1492 4 1 3 3 0 8 0 amappl11 160 69 0 54 1 0 1 1 0 8 0 amappl10 152 7 0 7 1 1 0 1 0 8 0 amappl9 144 257 0 256 2 1 1 1 0 8 0 amappl8 136 34 0 31 1 0 1 1 0 8 0 amappl7 128 104 0 92 1 0 1 1 0 8 0 amappl6 120 174 0 171 1 0 1 1 0 8 0 amappl5 112 121 0 112 1 0 1 1 0 8 0 amappl4 104 297 0 277 1 0 1 1 0 8 0 amappl3 96 3925 0 3839 3 0 3 3 0 8 0 amappl2 88 1120 0 1043 2 0 2 2 0 8 0 amappl1 80 10150 0 9543 17 4 13 16 0 8 0 amappl 88 6340 0 6180 5 0 5 5 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 8 0 8 2 1 1 1 0 8 1 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 22 0 3 1 0 1 1 0 8 0 uaddrrnd 24 884 0 854 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 884 0 854 1 0 1 1 0 8 0 vmmpekpl 168 8550 0 8515 2 0 2 2 0 8 0 vmmpepl 168 60437 0 58438 97 0 97 97 0 357 4 vmsppl 488 883 0 854 5 1 4 5 0 8 0 rwobjpl 80 20616 0 16726 82 1 81 81 0 8 0 pdppl 4096 1776 0 1708 100 28 72 86 0 8 4 pvpl 32 15683 0 0 129 2 127 127 0 265 0 pmappl 256 883 0 854 3 0 3 3 0 8 0 extentpl 40 45 0 27 1 0 1 1 0 8 0 phpool 112 293 0 35 8 0 8 8 0 8 0 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+0x27: addq $0x8,%rsp ddb{0}> trace x86_ipi_db(ffffffff837efff0) at x86_ipi_db+0x27 sys/arch/amd64/amd64/db_interface.c:394 x86_ipi_handler() at x86_ipi_handler+0xd9 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 __sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc+0x2c kd_curproc sys/dev/kcov.c:584 [inline] __sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc+0x2c sys/dev/kcov.c:153 __mp_lock(ffffffff83915db8) at __mp_lock+0x1a3 __mp_lock_spin sys/kern/kern_lock.c:134 [inline] __mp_lock(ffffffff83915db8) at __mp_lock+0x1a3 sys/kern/kern_lock.c:165 intr_handler(ffff80002ffdbbe0,ffff800000069c00) at intr_handler+0xe9 sys/arch/amd64/amd64/intr.c:559 Xintr_ioapic_edge16_untramp() at Xintr_ioapic_edge16_untramp+0x18f __mp_lock(ffffffff83915db8) at __mp_lock+0x192 __mp_lock_spin sys/kern/kern_lock.c:134 [inline] __mp_lock(ffffffff83915db8) at __mp_lock+0x192 sys/kern/kern_lock.c:165 intr_handler(ffff80002ffdbd40,ffff800000079a80) at intr_handler+0xe9 sys/arch/amd64/amd64/intr.c:559 Xintr_ioapic_edge23_untramp() at Xintr_ioapic_edge23_untramp+0x18f __mp_lock(ffffffff83915db8) at __mp_lock+0x192 __mp_lock_spin sys/kern/kern_lock.c:134 [inline] __mp_lock(ffffffff83915db8) at __mp_lock+0x192 sys/kern/kern_lock.c:165 uvm_fault(fffffd806ef753d8,1a6615ee000,0,1) at uvm_fault+0x1ea sys/uvm/uvm_fault.c:656 upageflttrap(ffff80002ffdc0d0,1a6615ee000) at upageflttrap+0xa9 sys/arch/amd64/amd64/trap.c:192 usertrap(ffff80002ffdc0d0) at usertrap+0x3c6 sys/arch/amd64/amd64/trap.c:603 recall_trap() at recall_trap+0x8 end of kernel end trace frame: 0x78b115a24cc0, count: -15 ddb{0}> machine ddbcpu 1 Stopped at db_enter+0x25: addq $0x8,%rsp ddb{1}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438 witness_checkorder(fffffd8068b8c580,9,0) at witness_checkorder+0x10b9 sys/kern/subr_witness.c:-1 rw_do_enter_write(fffffd8068b8c568,1) at rw_do_enter_write+0xba sys/kern/kern_rwlock.c:234 rrw_enter(fffffd8068b8c568,1) at rrw_enter+0xc6 sys/kern/kern_rwlock.c:621 VOP_LOCK(fffffd806e11a8c0,2001) at VOP_LOCK+0xa3 sys/kern/vfs_vops.c:527 vn_lock(fffffd806e11a8c0,2001) at vn_lock+0xa4 sys/kern/vfs_vnops.c:570 vfs_getcwd_common(fffffd806e11a8c0,fffffd80621ff018,0,0,200,0,44fe00c032bcbf6) at vfs_getcwd_common+0xd1 sys/kern/vfs_getcwd.c:287 vn_isunder(fffffd806e11a8c0,fffffd80621ff018,ffff80002a314038) at vn_isunder+0x56 sys/kern/vfs_vnops.c:694 unp_externalize(fffffd8062ed8200,f0,0) at unp_externalize+0x26f sys/kern/uipc_usrreq.c:1090 soreceive(ffff80000142d1b0,ffff80003c459478,ffff80003c459428,0,ffff80003c459460,ffff80003c4595ec,5c1ec78f915974c8) at soreceive+0xc24 sys/kern/uipc_socket.c:1009 recvit(ffff80002a314038,5,ffff80003c4595c0,0,ffff80003c459670) at recvit+0x40b sys/kern/uipc_syscalls.c:1078 sys_recvmsg(ffff80002a314038,ffff80003c459720,ffff80003c459670) at sys_recvmsg+0x1bf sys/kern/uipc_syscalls.c:878 syscall(ffff80003c459720) at syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] syscall(ffff80003c459720) at syscall+0xbd4 sys/arch/amd64/amd64/trap.c:746 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xdcdd7b28ce0, count: -14