witness: lock order reversal: 1st 0xffff800000e28c30 sbufsnd (&so->so_snd.sb_lock) 2nd 0xfffffd806e15d2c0 inode (&ip->i_lock) lock order [1] sbufsnd (&so->so_snd.sb_lock) -> [2] inode (&ip->i_lock) lock order data 0xffffffff8292c146 -> 0xffffffff828d64a9 is missing lock order [2] inode (&ip->i_lock) -> [3] sbufrcv (&so->so_rcv.sb_lock) #0 rw_enter+0xd1 #1 sblock+0x8a sys/kern/uipc_socket2.c:548 #2 soreceive+0x201 sys/kern/uipc_socket.c:876 #3 fifo_read+0xd2 sys/miscfs/fifofs/fifo_vnops.c:256 #4 VOP_READ+0xc3 sys/kern/vfs_vops.c:227 #5 vn_rdwr+0x10a #6 vndsetcred+0x86 sys/dev/vnd.c:684 #7 vndioctl+0xac1 sys/dev/vnd.c:485 #8 VOP_IOCTL+0x9a sys/kern/vfs_vops.c:264 #9 vn_ioctl+0xc0 sys/kern/vfs_vnops.c:525 #10 sys_ioctl+0x4a9 #11 syscall+0x8cf mi_syscall sys/sys/syscall_mi.h:180 [inline] #11 syscall+0x8cf sys/arch/amd64/amd64/trap.c:577 #12 Xsyscall+0x128 lock order [3] sbufrcv (&so->so_rcv.sb_lock) -> [1] sbufsnd (&so->so_snd.sb_lock) #0 rw_enter+0xd1 #1 sblock+0x8a sys/kern/uipc_socket2.c:548 #2 sosplice+0x2a7 sys/kern/uipc_socket.c:1385 #3 sys_setsockopt+0x1d7 sys/kern/uipc_syscalls.c:1231 #4 syscall+0x8cf mi_syscall sys/sys/syscall_mi.h:180 [inline] #4 syscall+0x8cf sys/arch/amd64/amd64/trap.c:577 #5 Xsyscall+0x128 Stopped at db_enter+0x1c: addq $0x8,%rsp ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic the kernel did not panic ddb{1}> trace db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 witness_checkorder(fffffd806e15d2c0,9,0) at witness_checkorder+0xc05 rw_enter(fffffd806e15d2b0,1) at rw_enter+0xd1 rrw_enter(fffffd806e15d2b0,1) at rrw_enter+0x91 sys/kern/kern_rwlock.c:464 VOP_LOCK(fffffd805eebd2a0,2001) at VOP_LOCK+0x8b sys/kern/vfs_vops.c:524 vn_lock(fffffd805eebd2a0,2001) at vn_lock+0x85 sys/kern/vfs_vnops.c:564 vfs_lookup(ffff80002a2b1df8) at vfs_lookup+0xd3 sys/kern/vfs_lookup.c:418 namei(ffff80002a2b1df8) at namei+0x56a sys/kern/vfs_lookup.c:250 unp_connect(ffff800000e28a88,fffffd807f002c00,ffff8000373979f0) at unp_connect+0x1ce sys/kern/uipc_usrreq.c:864 uipc_dgram_send(ffff800000e28a88,fffffd806cc09800,fffffd807f002c00,0) at uipc_dgram_send+0xd7 sys/kern/uipc_usrreq.c:599 sosend(ffff800000e28a88,fffffd807f002c00,ffff80002a2b2088,0,0,0) at sosend+0x7aa sendit(ffff8000373979f0,9,ffff80002a2b2200,0,ffff80002a2b22c0) at sendit+0x54c sys/kern/uipc_syscalls.c:786 sys_sendmsg(ffff8000373979f0,ffff80002a2b2370,ffff80002a2b22c0) at sys_sendmsg+0x19b sys/kern/uipc_syscalls.c:604 syscall(ffff80002a2b2370) at syscall+0x8cf mi_syscall sys/sys/syscall_mi.h:180 [inline] syscall(ffff80002a2b2370) at syscall+0x8cf sys/arch/amd64/amd64/trap.c:577 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x9b7077b0150, count: -15 ddb{1}> show registers rdi 0 rsi 0x40000 acpi_pdirpa+0x2be71 rbp 0xffff80002a2b1a50 rbx 0xffffffff828e9404 pp_r600_decoded_lanes+0x2e88d rdx 0xffff800000df3c40 rcx 0xffff8000373979f0 rax 0x3ffff acpi_pdirpa+0x2be70 r8 0xffff80002a2b1930 r9 0x8080808080808080 r10 0x4ae71304fb9c9836 r11 0xb4c6db88e1d60437 r12 0xfffffd8003b26130 r13 0xfffffd800333ddc0 r14 0x3 r15 0xffffffff rip 0xffffffff81cae62c db_enter+0x1c cs 0x8 rflags 0x246 rsp 0xffff80002a2b1a40 ss 0x10 db_enter+0x1c: addq $0x8,%rsp ddb{1}> show proc PROC (syz-executor.4) tid=162085 pid=55262 tcnt=3 stat=onproc flags process=8000000 proc=4000000 runpri=32, usrpri=86, slppri=32, nice=20 wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0 forw=0xffffffffffffffff, list=0xffff800037396a90,0xffffffff82ea1428 process=0xffff8000373c8028 user=0xffff80002a2ad000, vmspace=0xfffffd806abca380 estcpu=36, cpticks=0, pctcpu=0.0, user=0, sys=0, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 55262 324639 53244 0 7 0x8000000 syz-executor.4 55262 369761 53244 0 3 0xc000080 tunread syz-executor.4 *55262 162085 53244 0 7 0xc000000 syz-executor.4 77823 128455 58714 0 3 0x8000080 nanoslp syz-executor.5 77823 179448 58714 0 3 0xc000080 rest syz-executor.5 2091 460396 11587 0 3 0x8000082 nanoslp syz-executor.7 53244 338101 11587 0 3 0x8000082 nanoslp syz-executor.4 57418 135259 11587 0 3 0x8000082 nanoslp syz-executor.3 69692 443254 11587 0 3 0x8000002 biowait syz-executor.2 7360 109291 11587 0 3 0x8000082 nanoslp syz-executor.1 7982 180358 0 0 3 0x14200 bored sosplice 54418 32856 0 0 3 0x14280 nfsidl nfsio 78166 84444 0 0 3 0x14280 nfsidl nfsio 90526 270853 0 0 3 0x14280 nfsidl nfsio 91202 20470 0 0 3 0x14280 nfsidl nfsio 4175 418923 0 0 3 0x14280 nfsidl nfsio 21390 47323 0 0 3 0x14280 nfsidl nfsio 7619 499433 0 0 3 0x14280 nfsidl nfsio 37389 494716 0 0 3 0x14280 nfsidl nfsio 39078 14878 0 0 3 0x14280 nfsidl nfsio 32219 293926 0 0 3 0x14280 nfsidl nfsio 55027 55631 0 0 3 0x14280 nfsidl nfsio 63241 500329 0 0 3 0x14280 nfsidl nfsio 64304 507851 0 0 3 0x14280 nfsidl nfsio 93083 326306 0 0 3 0x14280 nfsidl nfsio 87724 243292 0 0 3 0x14280 nfsidl nfsio 54142 471715 0 0 3 0x14280 nfsidl nfsio 7497 273421 0 0 3 0x14280 nfsidl nfsio 81146 361849 0 0 3 0x14280 nfsidl nfsio 61755 163638 0 0 3 0x14280 nfsidl nfsio 54808 480342 0 0 3 0x14280 nfsidl nfsio 58714 82775 11587 0 3 0x8000082 nanoslp syz-executor.5 79033 162704 11587 0 3 0x8000002 biowait syz-executor.6 73135 118676 11587 0 3 0x8000082 nanoslp syz-executor.0 11587 138900 67656 0 3 0x1a000082 wait syz-fuzzer 11587 231443 67656 0 3 0x1e000082 nanoslp syz-fuzzer 11587 454206 67656 0 3 0x1e000082 thrsleep syz-fuzzer 11587 155546 67656 0 3 0x1e000082 wait syz-fuzzer 11587 360709 67656 0 3 0x1e000082 wait syz-fuzzer 11587 386598 67656 0 3 0x1e000082 kqread syz-fuzzer 11587 402330 67656 0 3 0x1e000082 wait syz-fuzzer 11587 284221 67656 0 3 0x1e000082 wait syz-fuzzer 11587 394551 67656 0 3 0x1e000082 thrsleep syz-fuzzer 11587 459398 67656 0 3 0x1e000082 thrsleep syz-fuzzer 11587 4672 67656 0 3 0x1e000082 thrsleep syz-fuzzer 11587 427616 67656 0 3 0x1e000082 thrsleep syz-fuzzer 11587 182531 67656 0 3 0x1e000082 wait syz-fuzzer 11587 440934 67656 0 3 0x1e000082 wait syz-fuzzer 11587 183334 67656 0 3 0x1e000082 thrsleep syz-fuzzer 11587 463824 67656 0 3 0x1e000082 wait syz-fuzzer 67656 101185 1742 0 3 0x810008a sigsusp ksh 1742 334474 33091 0 3 0x1800009a kqread sshd 58165 438128 1 0 3 0x18100083 ttyin getty 33091 488456 1 0 3 0x18000088 kqread sshd 48222 217390 65940 73 3 0x19100010 biowait syslogd 65940 361030 1 0 3 0x18100082 sbwait syslogd 33374 477160 1 0 3 0x18100080 kqread resolvd 71329 441556 77872 77 3 0x18100092 kqread dhcpleased 39420 216306 77872 77 3 0x18100092 kqread dhcpleased 77872 155254 1 0 3 0x18000080 kqread dhcpleased 51894 149723 0 0 3 0x14200 bored smr 73357 149592 0 0 3 0x14200 pgzero zerothread 57431 17968 0 0 3 0x14200 aiodoned aiodoned 70339 336999 0 0 3 0x14200 syncer update 99212 113154 0 0 3 0x14200 cleaner cleaner 95689 247872 0 0 3 0x14200 reaper reaper 65381 133010 0 0 3 0x14200 pgdaemon pagedaemon 54806 311579 0 0 3 0x14200 bored viomb 65598 154238 0 0 3 0x40014200 acpi0 acpi0 22787 175303 0 0 3 0x40014200 idle1 39581 35113 0 0 3 0x14200 bored softnet3 8761 282427 0 0 3 0x14200 bored softnet2 29602 404177 0 0 3 0x14200 bored softnet1 57927 375415 0 0 3 0x14200 bored softnet0 15895 246731 0 0 3 0x14200 bored systqmp 84092 461727 0 0 3 0x14200 bored systq 89688 245540 0 0 3 0x14200 tmoslp softclockmp 42554 222893 0 0 3 0x40014200 tmoslp softclock 12027 263012 0 0 3 0x40014200 idle0 1 286922 0 0 3 0x8000082 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{1}> show all locks Process 55262 (syz-executor.4) thread 0xffff800037396d20 (369761) exclusive rrwlock inode r = 0 (0xfffffd806acf65f8) #0 witness_lock+0x446 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x446 sys/kern/subr_witness.c:1157 #1 rw_enter+0x32d sys/kern/kern_rwlock.c:309 #2 rrw_enter+0x91 sys/kern/kern_rwlock.c:464 #3 VOP_LOCK+0x8b sys/kern/vfs_vops.c:524 #4 vn_lock+0x85 sys/kern/vfs_vnops.c:564 #5 spec_open+0x230 sys/kern/spec_vnops.c:151 #6 VOP_OPEN+0x79 sys/kern/vfs_vops.c:138 #7 vn_open+0x4e4 sys/kern/vfs_vnops.c:177 #8 vndioctl+0x988 sys/dev/vnd.c:457 #9 VOP_IOCTL+0x9a sys/kern/vfs_vops.c:264 #10 vn_ioctl+0xc0 sys/kern/vfs_vnops.c:525 #11 sys_ioctl+0x4a9 #12 syscall+0x8cf mi_syscall sys/sys/syscall_mi.h:180 [inline] #12 syscall+0x8cf sys/arch/amd64/amd64/trap.c:577 #13 Xsyscall+0x128 Process 55262 (syz-executor.4) thread 0xffff8000373979f0 (162085) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff82dea520) #0 witness_lock+0x446 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x446 sys/kern/subr_witness.c:1157 #1 unp_connect+0x1c6 sys/kern/uipc_usrreq.c:864 #2 uipc_dgram_send+0xd7 sys/kern/uipc_usrreq.c:599 #3 sosend+0x7aa #4 sendit+0x54c sys/kern/uipc_syscalls.c:786 #5 sys_sendmsg+0x19b sys/kern/uipc_syscalls.c:604 #6 syscall+0x8cf mi_syscall sys/sys/syscall_mi.h:180 [inline] #6 syscall+0x8cf sys/arch/amd64/amd64/trap.c:577 #7 Xsyscall+0x128 exclusive rwlock sbufsnd r = 0 (0xffff800000e28c30) #0 witness_lock+0x446 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x446 sys/kern/subr_witness.c:1157 #1 rw_enter+0x32d sys/kern/kern_rwlock.c:309 #2 sblock+0x8a sys/kern/uipc_socket2.c:548 #3 sosend+0x259 sys/kern/uipc_socket.c:610 #4 sendit+0x54c sys/kern/uipc_syscalls.c:786 #5 sys_sendmsg+0x19b sys/kern/uipc_syscalls.c:604 #6 syscall+0x8cf mi_syscall sys/sys/syscall_mi.h:180 [inline] #6 syscall+0x8cf sys/arch/amd64/amd64/trap.c:577 #7 Xsyscall+0x128 Process 69692 (syz-executor.2) thread 0xffff800037391230 (443254) exclusive rrwlock inode r = 0 (0xfffffd806f4b42c0) #0 witness_lock+0x446 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x446 sys/kern/subr_witness.c:1157 #1 rw_enter+0x32d sys/kern/kern_rwlock.c:309 #2 rrw_enter+0x91 sys/kern/kern_rwlock.c:464 #3 VOP_LOCK+0x8b sys/kern/vfs_vops.c:524 #4 vn_lock+0x85 sys/kern/vfs_vnops.c:564 #5 vget+0x1fd sys/kern/vfs_subr.c:676 #6 ufs_ihashget+0x121 sys/ufs/ufs/ufs_ihash.c:119 #7 ffs_vget+0x7c sys/ufs/ffs/ffs_vfsops.c:1201 #8 ufs_lookup+0x1373 sys/ufs/ufs/ufs_lookup.c:478 #9 VOP_LOOKUP+0x5c sys/kern/vfs_vops.c:85 #10 vfs_lookup+0x6f8 sys/kern/vfs_lookup.c:566 #11 namei+0x56a sys/kern/vfs_lookup.c:250 #12 dounlinkat+0x9d sys/kern/vfs_syscalls.c:1856 #13 syscall+0x854 mi_syscall sys/sys/syscall_mi.h:180 [inline] #13 syscall+0x854 sys/arch/amd64/amd64/trap.c:577 #14 Xsyscall+0x128 exclusive rrwlock inode r = 0 (0xfffffd806e3892b8) #0 witness_lock+0x446 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x446 sys/kern/subr_witness.c:1157 #1 rw_enter+0x32d sys/kern/kern_rwlock.c:309 #2 rrw_enter+0x91 sys/kern/kern_rwlock.c:464 #3 VOP_LOCK+0x8b sys/kern/vfs_vops.c:524 #4 vn_lock+0x85 sys/kern/vfs_vnops.c:564 #5 vfs_lookup+0xd3 sys/kern/vfs_lookup.c:418 #6 namei+0x56a sys/kern/vfs_lookup.c:250 #7 dounlinkat+0x9d sys/kern/vfs_syscalls.c:1856 #8 syscall+0x854 mi_syscall sys/sys/syscall_mi.h:180 [inline] #8 syscall+0x854 sys/arch/amd64/amd64/trap.c:577 #9 Xsyscall+0x128 Process 79033 (syz-executor.6) thread 0xffff8000ffff7c48 (162704) exclusive rrwlock inode r = 0 (0xfffffd806ff83d68) #0 witness_lock+0x446 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x446 sys/kern/subr_witness.c:1157 #1 rw_enter+0x32d sys/kern/kern_rwlock.c:309 #2 rrw_enter+0x91 sys/kern/kern_rwlock.c:464 #3 VOP_LOCK+0x8b sys/kern/vfs_vops.c:524 #4 ufs_ihashins+0x46 #5 ffs_vget+0x141 sys/ufs/ffs/ffs_vfsops.c:1230 #6 ffs_inode_alloc+0x1e4 sys/ufs/ffs/ffs_alloc.c:393 #7 ufs_mkdir+0xe6 sys/ufs/ufs/ufs_vnops.c:1112 #8 VOP_MKDIR+0xc3 sys/kern/vfs_vops.c:394 #9 domkdirat+0x125 sys/kern/vfs_syscalls.c:3104 #10 syscall+0x854 mi_syscall sys/sys/syscall_mi.h:180 [inline] #10 syscall+0x854 sys/arch/amd64/amd64/trap.c:577 #11 Xsyscall+0x128 exclusive rrwlock inode r = 0 (0xfffffd80691cc3d0) #0 witness_lock+0x446 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x446 sys/kern/subr_witness.c:1157 #1 rw_enter+0x32d sys/kern/kern_rwlock.c:309 #2 rrw_enter+0x91 sys/kern/kern_rwlock.c:464 #3 VOP_LOCK+0x8b sys/kern/vfs_vops.c:524 #4 vn_lock+0x85 sys/kern/vfs_vnops.c:564 #5 vfs_lookup+0xd3 sys/kern/vfs_lookup.c:418 #6 namei+0x56a sys/kern/vfs_lookup.c:250 #7 domkdirat+0x79 sys/kern/vfs_syscalls.c:3089 #8 syscall+0x854 mi_syscall sys/sys/syscall_mi.h:180 [inline] #8 syscall+0x854 sys/arch/amd64/amd64/trap.c:577 #9 Xsyscall+0x128 Process 48222 (syslogd) thread 0xffff8000ffffdc40 (217390) exclusive rrwlock inode r = 0 (0xfffffd806e74f2c0) #0 witness_lock+0x446 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x446 sys/kern/subr_witness.c:1157 #1 rw_enter+0x32d sys/kern/kern_rwlock.c:309 #2 rrw_enter+0x91 sys/kern/kern_rwlock.c:464 #3 VOP_LOCK+0x8b sys/kern/vfs_vops.c:524 #4 vn_lock+0x85 sys/kern/vfs_vnops.c:564 #5 sys_fsync+0xf1 sys/kern/vfs_syscalls.c:2931 #6 syscall+0x854 mi_syscall sys/sys/syscall_mi.h:180 [inline] #6 syscall+0x854 sys/arch/amd64/amd64/trap.c:577 #7 Xsyscall+0x128 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10218 6562K 7116K 166960K 13462 0 pcb 17 12K 12K 166960K 204 0 rtable 196 8K 8K 166960K 774 0 pf 35 10K 10K 166960K 94 0 ifaddr 41 14K 15K 166960K 107 0 ifgroup 62 2K 2K 166960K 142 0 sysctl 3 0K 1K 166960K 6 0 counters 68 36K 37K 166960K 112 0 ioctlops 0 0K 4K 166960K 1542 0 iov 0 0K 18K 166960K 62 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1382 87K 87K 166960K 2468 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 5K 13K 166960K 30 0 VM map 2 1K 1K 166960K 2 0 sem 12 0K 0K 166960K 91 0 dirhash 12 2K 3K 166960K 51 0 ACPI 1697 195K 286K 166960K 12548 0 file desc 13 45K 89K 166960K 1229 0 sigio 0 0K 0K 166960K 18 0 proc 58 79K 115K 166960K 839 0 subproc 104 6K 7K 166960K 249 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 2 0K 0K 166960K 190 0 in_multi 77 5K 7K 166960K 249 0 ether_multi 1 0K 0K 166960K 9 0 mrt 0 0K 0K 166960K 3 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 97 440K 440K 166960K 97 0 exec 0 0K 1K 166960K 648 0 tdb 3 0K 0K 166960K 3 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 261 77K 105K 166960K 12859 0 UVM aobj 52 2K 2K 166960K 53 0 pinsyscall 33 66K 100K 166960K 2620 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 1K 166960K 85 0 NDP 13 0K 1K 166960K 70 0 temp 75 6808K 6888K 166960K 52611 0 kqueue 12 18K 28K 166960K 193 0 SYN cache 2 16K 16K 166960K 2 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 24 0 0 1 0 1 1 0 8 0 rtpcb 120 146 0 142 1 0 1 1 0 8 0 rtentry 112 247 0 162 4 0 4 4 0 8 0 unpcb 144 949 0 934 2 0 2 2 0 8 1 syncache 336 9 0 9 3 2 1 1 0 8 1 tcpqe 32 1 0 1 1 0 1 1 0 8 1 tcpcb 808 349 0 343 5 3 2 2 0 8 1 arp 120 44 0 28 1 0 1 1 0 8 0 inpcb 384 1255 0 1243 9 6 3 3 0 8 1 nd6 136 59 0 37 1 0 1 1 0 8 0 pkpcb 40 11 0 11 3 2 1 1 0 8 1 kcovpl 48 19 0 11 1 0 1 1 0 8 0 ppxss 1168 6 0 6 3 3 0 1 0 8 0 pffrag 232 12 0 8 2 1 1 1 0 482 0 pffrnode 88 12 0 8 2 1 1 1 0 8 0 pffrent 40 146 0 142 3 2 1 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfstitem 24 114 0 74 1 0 1 1 0 8 0 pfstkey 128 114 0 74 2 0 2 2 0 8 0 pfstate 376 114 0 74 5 1 4 4 0 8 0 pfrule 1344 21 0 16 2 1 1 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 947 0 580 33 9 24 29 0 8 0 art_table 32 948 0 580 4 0 4 4 0 8 0 art_node 16 243 0 168 1 0 1 1 0 8 0 sysvmsgpl 40 13 0 7 1 0 1 1 0 8 0 semupl 112 1 0 1 1 1 0 1 0 8 0 semapl 112 88 0 78 1 0 1 1 0 8 0 shmpl 112 50 0 1 2 0 2 2 0 8 0 dirhash 1024 43 0 26 3 0 3 3 0 8 0 dino2pl 256 3387 0 1869 96 0 96 96 0 8 0 ffsino 272 3387 0 1869 102 0 102 102 0 8 0 nchpl 144 5316 0 3574 67 0 67 67 0 8 0 uvmvnodes 80 4311 0 0 88 0 88 88 0 8 0 vnodes 216 4311 0 0 240 0 240 240 0 8 0 namei 1024 18755 0 18753 5 4 1 2 0 8 0 percpumem 16 70 0 22 1 0 1 1 0 8 0 vcpupl 3904 2 0 0 1 0 1 1 0 8 0 vmpool 696 9 0 7 1 0 1 1 0 8 0 kstatmem 264 70 0 44 2 0 2 2 0 8 0 scsiplug 72 1 0 1 1 1 0 1 0 8 0 scxspl 216 33378 0 33375 15 14 1 8 1 8 0 plimitpl 152 212 0 195 1 0 1 1 0 8 0 sigapl 424 1542 0 1476 8 0 8 8 0 8 0 futexpl 64 20259 0 20259 4 3 1 1 0 8 1 knotepl 120 623 0 0 18 0 18 18 0 8 0 kqueuepl 216 407 0 398 1 0 1 1 0 8 0 pipepl 320 316 0 288 3 0 3 3 0 8 0 fdescpl 496 1500 0 1476 6 2 4 5 0 8 0 filepl 152 9651 0 9399 12 0 12 12 0 8 0 lockfpl 104 471 0 468 1 0 1 1 0 8 0 lockfspl 48 216 0 213 1 0 1 1 0 8 0 sessionpl 144 32 0 16 1 0 1 1 0 8 0 pgrppl 48 41 0 24 1 0 1 1 0 8 0 ucredpl 104 1398 0 1382 1 0 1 1 0 8 0 zombiepl 144 1480 0 1476 1 0 1 1 0 8 0 processpl 1144 1542 0 1476 6 0 6 6 0 8 0 procpl 656 2759 0 2675 11 2 9 9 0 8 0 srpgc 96 15 0 15 3 3 0 1 0 8 0 sosppl 168 4 0 4 3 2 1 1 0 8 1 sockpl 664 2372 0 2341 9 4 5 5 0 8 1 mcl64k 65536 20 0 0 3 1 2 3 0 8 0 mcl16k 16384 4 0 0 1 0 1 1 0 8 0 mcl12k 12288 6 0 0 1 0 1 1 0 8 0 mcl9k 9216 2 0 0 1 0 1 1 0 8 0 mcl8k 8192 3 0 0 1 0 1 1 0 8 0 mcl4k 4096 3 0 0 1 0 1 1 0 8 0 mcl2k 2048 320 0 0 40 0 40 40 0 8 0 mtagpl 96 26 0 0 1 0 1 1 0 8 0 mbufpl 256 416 0 0 23 0 23 23 0 8 0 bufpl 280 9418 0 2356 505 0 505 505 0 8 0 anonpl 24 366588 0 360596 171 108 63 160 0 186 1 amapchunkpl 152 44334 0 43743 80 43 37 72 0 158 6 amappl16 200 8299 0 8158 49 29 20 21 0 8 8 amappl15 192 9 0 9 1 1 0 1 0 8 0 amappl14 184 176 0 165 2 1 1 2 0 8 0 amappl13 176 13 0 13 1 1 0 1 0 8 0 amappl12 168 2278 0 2251 2 0 2 2 0 8 0 amappl11 160 76 0 58 1 0 1 1 0 8 0 amappl10 152 80 0 75 1 0 1 1 0 8 0 amappl9 144 192 0 192 2 2 0 1 0 8 0 amappl8 136 183 0 152 2 0 2 2 0 8 0 amappl7 128 54 0 38 1 0 1 1 0 8 0 amappl6 120 435 0 421 2 1 1 2 0 8 0 amappl5 112 204 0 192 1 0 1 1 0 8 0 amappl4 104 616 0 583 2 0 2 2 0 8 0 amappl3 96 8404 0 8340 3 0 3 3 0 8 0 amappl2 88 1936 0 1868 4 2 2 4 0 8 0 amappl1 80 13495 0 13006 22 10 12 22 0 8 0 amappl 88 12152 0 11983 6 0 6 6 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 52 0 1 1 0 1 1 0 8 0 uaddrrnd 24 1509 0 1483 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 1509 0 1483 1 0 1 1 0 8 0 vmmpekpl 168 14522 0 14460 4 0 4 4 0 8 0 vmmpepl 168 113105 0 111391 141 45 96 118 0 357 5 vmsppl 440 1508 0 1483 4 0 4 4 0 8 0 rwobjpl 56 37722 0 32292 85 4 81 81 0 8 3 pdppl 4096 3025 0 2968 139 72 67 77 0 8 10 pvpl 32 49275 0 0 400 2 398 398 0 265 0 pmappl 248 1508 0 1483 4 1 3 3 0 8 0 extentpl 40 56 0 38 1 0 1 1 0 8 0 phpool 112 462 0 96 11 0 11 11 0 8 0 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+0x1e: addq $0x8,%rsp ddb{0}> trace x86_ipi_db(ffffffff82c95ff0) at x86_ipi_db+0x1e sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 __mp_lock(ffffffff82dea318) at __mp_lock+0x122 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff82dea318) at __mp_lock+0x122 sys/kern/kern_lock.c:147 intr_handler(ffff800030d6c7c0,ffff800000069c00) at intr_handler+0xa4 sys/arch/amd64/amd64/intr.c:549 Xintr_ioapic_edge16_untramp() at Xintr_ioapic_edge16_untramp+0x18f __sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc+0x33 kd_curproc sys/dev/kcov.c:590 [inline] __sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc+0x33 sys/dev/kcov.c:158 __mp_lock(ffffffff82dea318) at __mp_lock+0x133 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff82dea318) at __mp_lock+0x133 sys/kern/kern_lock.c:147 intr_handler(ffff800030d6c930,ffff80000067b200) at intr_handler+0xa4 sys/arch/amd64/amd64/intr.c:549 Xintr_ioapic_edge25_untramp() at Xintr_ioapic_edge25_untramp+0x18f __mp_lock(ffffffff82dea318) at __mp_lock+0x122 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff82dea318) at __mp_lock+0x122 sys/kern/kern_lock.c:147 intr_handler(ffff800030d6ca90,ffff80000006bc00) at intr_handler+0xa4 sys/arch/amd64/amd64/intr.c:549 Xintr_ioapic_edge23_untramp() at Xintr_ioapic_edge23_untramp+0x18f end of kernel end trace frame: 0x70ef43db6ec0, count: -13 ddb{0}> machine ddbcpu 1 Stopped at db_enter+0x1c: addq $0x8,%rsp ddb{1}> trace db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 witness_checkorder(fffffd806e15d2c0,9,0) at witness_checkorder+0xc05 rw_enter(fffffd806e15d2b0,1) at rw_enter+0xd1 rrw_enter(fffffd806e15d2b0,1) at rrw_enter+0x91 sys/kern/kern_rwlock.c:464 VOP_LOCK(fffffd805eebd2a0,2001) at VOP_LOCK+0x8b sys/kern/vfs_vops.c:524 vn_lock(fffffd805eebd2a0,2001) at vn_lock+0x85 sys/kern/vfs_vnops.c:564 vfs_lookup(ffff80002a2b1df8) at vfs_lookup+0xd3 sys/kern/vfs_lookup.c:418 namei(ffff80002a2b1df8) at namei+0x56a sys/kern/vfs_lookup.c:250 unp_connect(ffff800000e28a88,fffffd807f002c00,ffff8000373979f0) at unp_connect+0x1ce sys/kern/uipc_usrreq.c:864 uipc_dgram_send(ffff800000e28a88,fffffd806cc09800,fffffd807f002c00,0) at uipc_dgram_send+0xd7 sys/kern/uipc_usrreq.c:599 sosend(ffff800000e28a88,fffffd807f002c00,ffff80002a2b2088,0,0,0) at sosend+0x7aa sendit(ffff8000373979f0,9,ffff80002a2b2200,0,ffff80002a2b22c0) at sendit+0x54c sys/kern/uipc_syscalls.c:786 sys_sendmsg(ffff8000373979f0,ffff80002a2b2370,ffff80002a2b22c0) at sys_sendmsg+0x19b sys/kern/uipc_syscalls.c:604 syscall(ffff80002a2b2370) at syscall+0x8cf mi_syscall sys/sys/syscall_mi.h:180 [inline] syscall(ffff80002a2b2370) at syscall+0x8cf sys/arch/amd64/amd64/trap.c:577 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x9b7077b0150, count: -15