witness: lock order reversal: 1st 0xffffffff82cc8bd0 &sched_lock (&sched_lock) 2nd 0xffff8000ffff7798 &pr->ps_mtx (&pr->ps_mtx) lock order "&pr->ps_mtx"(mutex) -> "&sched_lock"(sched_lock) first seen at: #0 __mp_lock+0xa2 read_rflags machine/cpufunc.h:195 [inline] #0 __mp_lock+0xa2 intr_disable machine/cpufunc.h:216 [inline] #0 __mp_lock+0xa2 sys/kern/kern_lock.c:142 #1 donice+0x178 sys/kern/kern_resource.c:218 #2 syscall+0x5e2 mi_syscall sys/sys/syscall_mi.h:110 [inline] #2 syscall+0x5e2 sys/arch/amd64/amd64/trap.c:623 #3 Xsyscall+0x128 lock order "&sched_lock"(sched_lock) -> "&pr->ps_mtx"(mutex) first seen at: #0 mtx_enter+0x3e sys/kern/kern_lock.c:265 #1 single_thread_set+0x35a single_thread_wait sys/kern/kern_sig.c:2172 [inline] #1 single_thread_set+0x35a sys/kern/kern_sig.c:2155 #2 sigexit+0x90 sys/kern/kern_sig.c:1562 #3 postsig+0x4a9 sys/kern/kern_sig.c:1494 #4 userret+0x16e sys/kern/kern_sig.c:1987 #5 syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline] #5 syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644 #6 Xsyscall+0x128 Stopped at db_enter+0x1c: addq $0x8,%rsp ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic the kernel did not panic ddb{1}> trace db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 witness_checkorder(ffff8000ffff7798,9,0) at witness_checkorder+0x10b7 witness_debugger sys/kern/subr_witness.c:2510 [inline] witness_checkorder(ffff8000ffff7798,9,0) at witness_checkorder+0x10b7 sys/kern/subr_witness.c:1110 mtx_enter(ffff8000ffff7788) at mtx_enter+0x3e sys/kern/kern_lock.c:265 single_thread_set(ffff800021238558,0,1) at single_thread_set+0x35a single_thread_wait sys/kern/kern_sig.c:2172 [inline] single_thread_set(ffff800021238558,0,1) at single_thread_set+0x35a sys/kern/kern_sig.c:2155 sigexit(ffff800021238558,6) at sigexit+0x90 sys/kern/kern_sig.c:1562 postsig(ffff800021238558,6,ffff80002e449218) at postsig+0x4a9 sys/kern/kern_sig.c:1494 userret(ffff800021238558) at userret+0x16e sys/kern/kern_sig.c:1987 syscall(ffff80002e449350) at syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline] syscall(ffff80002e449350) at syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x6b0829e4790, count: -9 ddb{1}> show registers rdi 0x3 rsi 0xffffffff82bf22f0 __sancov_gen_cov_switch_values.134 rbp 0xffff80002e448f20 rbx 0x3 rdx 0 rcx 0xffff800021238558 rax 0xffff800020d58ff0 r8 0xffff80002e448e90 r9 0x8080808080808080 r10 0x8d62916dfe04607b r11 0x7c7292e9821955bb r12 0xfffffd8003a0a7a0 r13 0 r14 0xfffffd80039f4c70 r15 0xfffffd80031c7bc0 rip 0xffffffff81b6e27c db_enter+0x1c cs 0x8 rflags 0x246 rsp 0xffff80002e448f10 ss 0x10 db_enter+0x1c: addq $0x8,%rsp ddb{1}> show proc PROC (syz-executor.6) pid=345086 stat=onproc flags process=500010 proc=4002000 pri=32, usrpri=68, nice=20 forw=0xffffffffffffffff, list=0xffff800021200d60,0xffff800021238018 process=0xffff8000ffff7688 user=0xffff80002e444000, vmspace=0xfffffd806ca0f018 estcpu=18, cpticks=0, pctcpu=0.0 user=0, sys=0, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 50392 107663 77320 32767 3 0x90 nanoslp syz-executor.5 50392 35569 77320 32767 3 0x4000090 fsleep syz-executor.5 62157 202966 56232 32767 2 0x10 syz-executor.1 69372 82201 95638 32767 2 0x10 syz-executor.3 21948 350030 3871 32767 7 0x10 syz-executor.0 8347 209731 73245 32767 4 0x582010 syz-executor.6 * 8347 345086 73245 32767 7 0x4502010 syz-executor.6 8347 122620 73245 32767 4 0x4580090 fsleep syz-executor.6 90481 229539 34730 32767 2 0x10 syz-executor.4 90481 426452 34730 32767 2 0x4000010 syz-executor.4 90481 367780 34730 32767 3 0x4000090 fsleep syz-executor.4 3871 447299 32365 32767 3 0x90 nanoslp syz-executor.0 32365 221741 24757 0 3 0x82 wait syz-executor.0 64753 382445 39135 32767 3 0x10 biowait syz-executor.2 39135 44447 24757 0 3 0x82 wait syz-executor.2 34730 101982 105 32767 3 0x90 nanoslp syz-executor.4 105 60240 24757 0 3 0x82 wait syz-executor.4 95638 208950 32963 32767 3 0x90 nanoslp syz-executor.3 32963 514971 24757 0 3 0x82 wait syz-executor.3 43241 479490 90082 32767 3 0x10 biowait syz-executor.7 90082 423765 24757 0 3 0x82 wait syz-executor.7 77320 107307 26455 32767 3 0x90 nanoslp syz-executor.5 26455 51116 24757 0 3 0x82 wait syz-executor.5 56232 264084 77587 32767 3 0x90 nanoslp syz-executor.1 77587 225253 24757 0 3 0x82 wait syz-executor.1 73245 71878 25613 32767 3 0x90 nanoslp syz-executor.6 25613 467841 24757 0 3 0x82 wait syz-executor.6 23169 389723 0 0 3 0x14200 bored sosplice 24757 146375 12192 0 3 0x2000082 thrsleep syz-fuzzer 24757 357753 12192 0 3 0x6000082 nanoslp syz-fuzzer 24757 432205 12192 0 3 0x6000082 wait syz-fuzzer 24757 118275 12192 0 3 0x6000082 thrsleep syz-fuzzer 24757 80048 12192 0 3 0x6000082 wait syz-fuzzer 24757 120755 12192 0 3 0x6000082 wait syz-fuzzer 24757 474169 12192 0 3 0x6000082 kqread syz-fuzzer 24757 285534 12192 0 3 0x6000082 thrsleep syz-fuzzer 24757 178559 12192 0 3 0x6000082 thrsleep syz-fuzzer 24757 127254 12192 0 3 0x6000082 thrsleep syz-fuzzer 24757 356615 12192 0 3 0x6000082 wait syz-fuzzer 24757 157285 12192 0 3 0x6000082 wait syz-fuzzer 24757 168792 12192 0 3 0x6000082 thrsleep syz-fuzzer 24757 384025 12192 0 3 0x6000082 wait syz-fuzzer 24757 38402 12192 0 3 0x6000082 wait syz-fuzzer 24757 121262 12192 0 3 0x6000082 wait syz-fuzzer 12192 76444 59103 0 3 0x10008a sigsusp ksh 59103 438431 2297 0 3 0x9a kqread sshd 78313 442695 1 0 3 0x100083 ttyin getty 2297 473655 1 0 3 0x88 kqread sshd 90662 236593 99978 73 3 0x1100090 kqread syslogd 99978 55359 1 0 3 0x100082 netio syslogd 9994 137229 1 0 3 0x100080 kqread resolvd 22757 372113 6296 77 3 0x100092 kqread dhcpleased 21680 320912 6296 77 3 0x100092 kqread dhcpleased 6296 312254 1 0 3 0x80 kqread dhcpleased 77013 355561 0 0 3 0x14200 bored smr 46437 401206 0 0 3 0x14200 pgzero zerothread 12185 53365 0 0 3 0x14200 aiodoned aiodoned 69156 251053 0 0 3 0x14200 syncer update 88926 104333 0 0 3 0x14200 cleaner cleaner 67346 366029 0 0 3 0x14200 reaper reaper 37895 371343 0 0 3 0x14200 pgdaemon pagedaemon 42297 344576 0 0 3 0x14200 bored viomb 56214 222957 0 0 3 0x40014200 acpi0 acpi0 97467 101680 0 0 3 0x40014200 idle1 75762 455767 0 0 3 0x14200 bored softnet3 39064 197912 0 0 3 0x14200 bored softnet2 27271 94891 0 0 3 0x14200 bored softnet1 46848 421108 0 0 3 0x14200 bored softnet0 72463 14235 0 0 3 0x14200 bored systqmp 99011 420360 0 0 3 0x14200 bored systq 103 175500 0 0 3 0x40014200 bored softclock 12247 184842 0 0 3 0x40014200 idle0 1 477965 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{1}> show all locks CPU 0: exclusive mutex &uvm.fpageqlock r = 0 (0xffffffff82cc9298) #0 witness_lock+0x447 #1 mtx_enter_try+0x104 #2 mtx_enter+0x4f sys/kern/kern_lock.c:266 #3 uvm_pmr_getpages+0xeac sys/uvm/uvm_pmemrange.c:1205 #4 uvm_pagealloc+0x1a8 sys/uvm/uvm_page.c:910 #5 pmap_get_ptp+0x18d sys/arch/amd64/amd64/pmap.c:1209 #6 pmap_enter+0x2be sys/arch/amd64/amd64/pmap.c:2716 #7 uvm_fault_upper_lookup+0x2c0 sys/uvm/uvm_fault.c:895 #8 uvm_fault+0x129 sys/uvm/uvm_fault.c:605 #9 upageflttrap+0x86 sys/arch/amd64/amd64/trap.c:188 #10 usertrap+0x226 sys/arch/amd64/amd64/trap.c:436 #11 recall_trap+0x8 exclusive mutex &pmap->pm_mtx r = 0 (0xfffffd806fed6108) #0 witness_lock+0x447 #1 mtx_enter_try+0x104 #2 mtx_enter+0x4f sys/kern/kern_lock.c:266 #3 pmap_enter+0x1c3 pmap_map_ptes sys/arch/amd64/amd64/pmap.c:423 [inline] #3 pmap_enter+0x1c3 sys/arch/amd64/amd64/pmap.c:2710 #4 uvm_fault_upper_lookup+0x2c0 sys/uvm/uvm_fault.c:895 #5 uvm_fault+0x129 sys/uvm/uvm_fault.c:605 #6 upageflttrap+0x86 sys/arch/amd64/amd64/trap.c:188 #7 usertrap+0x226 sys/arch/amd64/amd64/trap.c:436 #8 recall_trap+0x8 CPU 1: exclusive sched_lock &sched_lock r = 0 (0xffffffff82cc8bd0) #0 witness_lock+0x447 #1 single_thread_set+0x179 sys/kern/kern_sig.c:2113 #2 sigexit+0x90 sys/kern/kern_sig.c:1562 #3 postsig+0x4a9 sys/kern/kern_sig.c:1494 #4 userret+0x16e sys/kern/kern_sig.c:1987 #5 syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline] #5 syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644 #6 Xsyscall+0x128 Process 21948 (syz-executor.0) thread 0xffff8000212012b0 (350030) exclusive rwlock amaplk r = 0 (0xfffffd80761702f0) #0 witness_lock+0x447 #1 uvm_fault_check+0x41a sys/uvm/uvm_fault.c:782 #2 uvm_fault+0xf2 sys/uvm/uvm_fault.c:600 #3 upageflttrap+0x86 sys/arch/amd64/amd64/trap.c:188 #4 usertrap+0x226 sys/arch/amd64/amd64/trap.c:436 #5 recall_trap+0x8 shared rwlock vmmaplk r = 0 (0xfffffd806ca0fa20) #0 witness_lock+0x447 #1 uvmfault_lookup+0xd9 sys/uvm/uvm_fault.c:1785 #2 uvm_fault_check+0x3e sys/uvm/uvm_fault.c:672 #3 uvm_fault+0xf2 sys/uvm/uvm_fault.c:600 #4 upageflttrap+0x86 sys/arch/amd64/amd64/trap.c:188 #5 usertrap+0x226 sys/arch/amd64/amd64/trap.c:436 #6 recall_trap+0x8 Process 8347 (syz-executor.6) thread 0xffff800021238558 (345086) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff82ca0228) #0 witness_lock+0x447 #1 postsig+0x49e sys/kern/kern_sig.c:1500 #2 userret+0x16e sys/kern/kern_sig.c:1987 #3 syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline] #3 syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644 #4 Xsyscall+0x128 exclusive sched_lock &sched_lock r = 0 (0xffffffff82cc8bd0) #0 witness_lock+0x447 #1 single_thread_set+0x179 sys/kern/kern_sig.c:2113 #2 sigexit+0x90 sys/kern/kern_sig.c:1562 #3 postsig+0x4a9 sys/kern/kern_sig.c:1494 #4 userret+0x16e sys/kern/kern_sig.c:1987 #5 syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline] #5 syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644 #6 Xsyscall+0x128 Process 64753 (syz-executor.2) thread 0xffff800021231d38 (382445) exclusive rrwlock inode r = 0 (0xfffffd8077c02c50) #0 witness_lock+0x447 #1 rw_enter+0x3c8 sys/kern/kern_rwlock.c:309 #2 rrw_enter+0x8c sys/kern/kern_rwlock.c:464 #3 VOP_LOCK+0x8b sys/kern/vfs_vops.c:518 #4 ufs_ihashins+0x46 sys/ufs/ufs/ufs_ihash.c:140 #5 ffs_vget+0x141 sys/ufs/ffs/ffs_vfsops.c:1343 #6 ffs_inode_alloc+0x1c2 sys/ufs/ffs/ffs_alloc.c:394 #7 ufs_mkdir+0xf8 sys/ufs/ufs/ufs_vnops.c:1149 #8 VOP_MKDIR+0xc3 sys/kern/vfs_vops.c:388 #9 domkdirat+0x125 sys/kern/vfs_syscalls.c:3073 #10 syscall+0x5e2 mi_syscall sys/sys/syscall_mi.h:110 [inline] #10 syscall+0x5e2 sys/arch/amd64/amd64/trap.c:623 #11 Xsyscall+0x128 exclusive rrwlock inode r = 0 (0xfffffd8077c02810) #0 witness_lock+0x447 #1 rw_enter+0x3c8 sys/kern/kern_rwlock.c:309 #2 rrw_enter+0x8c sys/kern/kern_rwlock.c:464 #3 VOP_LOCK+0x8b sys/kern/vfs_vops.c:518 #4 vn_lock+0x84 sys/kern/vfs_vnops.c:564 #5 vfs_lookup+0xd5 sys/kern/vfs_lookup.c:418 #6 namei+0x55a sys/kern/vfs_lookup.c:250 #7 domkdirat+0x79 sys/kern/vfs_syscalls.c:3058 #8 syscall+0x5e2 mi_syscall sys/sys/syscall_mi.h:110 [inline] #8 syscall+0x5e2 sys/arch/amd64/amd64/trap.c:623 #9 Xsyscall+0x128 Process 43241 (syz-executor.7) thread 0xffff8000212397f0 (479490) exclusive rrwlock inode r = 0 (0xfffffd806953bf80) #0 witness_lock+0x447 #1 rw_enter+0x3c8 sys/kern/kern_rwlock.c:309 #2 rrw_enter+0x8c sys/kern/kern_rwlock.c:464 #3 VOP_LOCK+0x8b sys/kern/vfs_vops.c:518 #4 vn_lock+0x84 sys/kern/vfs_vnops.c:564 #5 vget+0x200 sys/kern/vfs_subr.c:676 #6 ufs_ihashget+0x121 sys/ufs/ufs/ufs_ihash.c:119 #7 ffs_vget+0x7c sys/ufs/ffs/ffs_vfsops.c:1314 #8 ufs_lookup+0x13ba sys/ufs/ufs/ufs_lookup.c:487 #9 VOP_LOOKUP+0x5c sys/kern/vfs_vops.c:85 #10 vfs_lookup+0x6e2 sys/kern/vfs_lookup.c:566 #11 namei+0x55a sys/kern/vfs_lookup.c:250 #12 dounlinkat+0x9d sys/kern/vfs_syscalls.c:1847 #13 syscall+0x5e2 mi_syscall sys/sys/syscall_mi.h:110 [inline] #13 syscall+0x5e2 sys/arch/amd64/amd64/trap.c:623 #14 Xsyscall+0x128 exclusive rrwlock inode r = 0 (0xfffffd80625c4098) #0 witness_lock+0x447 #1 rw_enter+0x3c8 sys/kern/kern_rwlock.c:309 #2 rrw_enter+0x8c sys/kern/kern_rwlock.c:464 #3 VOP_LOCK+0x8b sys/kern/vfs_vops.c:518 #4 vn_lock+0x84 sys/kern/vfs_vnops.c:564 #5 vget+0x200 sys/kern/vfs_subr.c:676 #6 cache_lookup+0x2b4 sys/kern/vfs_cache.c:222 #7 ufs_lookup+0x1ac sys/ufs/ufs/ufs_lookup.c:162 #8 VOP_LOOKUP+0x5c sys/kern/vfs_vops.c:85 #9 vfs_lookup+0x6e2 sys/kern/vfs_lookup.c:566 #10 namei+0x55a sys/kern/vfs_lookup.c:250 #11 dounlinkat+0x9d sys/kern/vfs_syscalls.c:1847 #12 syscall+0x5e2 mi_syscall sys/sys/syscall_mi.h:110 [inline] #12 syscall+0x5e2 sys/arch/amd64/amd64/trap.c:623 #13 Xsyscall+0x128 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10197 6409K 6420K 78643K 11347 0 pcb 13 12K 14K 78643K 17 0 rtable 252 7K 7K 78643K 1765 0 pf 29 8K 8K 78643K 77 0 ifaddr 44 15K 16K 78643K 146 0 ifgroup 50 2K 2K 78643K 146 0 sysctl 3 1K 1K 78643K 3 0 counters 60 35K 35K 78643K 108 0 ioctlops 0 0K 2K 78643K 199 0 iov 0 0K 24K 78643K 1618 0 mount 1 1K 1K 78643K 1 0 log 0 0K 0K 78643K 4 0 vnodes 1279 80K 80K 78643K 3530 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 9K 78643K 307 0 VM map 2 1K 1K 78643K 2 0 sem 12 0K 0K 78643K 4404 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1697 195K 286K 78643K 12548 0 file desc 24 89K 129K 78643K 17110 0 sigio 0 0K 0K 78643K 102 0 proc 56 78K 115K 78643K 2123 0 subproc 104 6K 6K 78643K 416 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 1845 0 in_multi 99 7K 7K 78643K 486 0 ether_multi 1 0K 0K 78643K 9 0 mrt 1 0K 0K 78643K 1 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 121 546K 546K 78643K 121 0 exec 0 0K 1K 78643K 3620 0 tdb 3 0K 0K 78643K 3 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 8 62K 64K 78643K 10 0 UVM amap 456 99K 111K 78643K 169569 0 UVM aobj 131 4K 4K 78643K 133 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 0K 78643K 474 0 NDP 11 0K 2K 78643K 99 0 temp 74 5920K 6000K 78643K 44476 0 kqueue 12 18K 34K 78643K 1520 0 SYN cache 2 16K 16K 78643K 2 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 22 0 0 1 0 1 1 0 8 0 rtpcb 120 886 0 883 14 13 1 4 0 8 0 rtentry 112 406 0 287 4 0 4 4 0 8 0 unpcb 144 9242 0 9229 117 116 1 10 0 8 0 syncache 304 185 0 185 33 33 0 1 0 8 0 tcpqe 32 247 0 247 31 31 0 1 0 8 0 tcpcb 808 6873 0 6859 174 169 5 16 0 8 0 arp 120 70 0 51 1 0 1 1 0 8 0 ipq 40 27 0 27 10 10 0 1 0 8 0 ipqe 40 103 0 103 10 10 0 1 0 8 0 inpcb 368 12368 0 12348 191 187 4 15 0 8 0 nd6 136 123 0 91 7 5 2 2 0 8 0 kcovpl 48 32 0 24 1 0 1 1 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 1627 0 1154 32 2 30 31 0 8 0 art_table 32 1628 0 1154 4 0 4 4 0 8 0 art_node 16 405 0 296 1 0 1 1 0 8 0 sysvmsgpl 40 9 0 1 1 0 1 1 0 8 0 semapl 112 4400 0 4390 1 0 1 1 0 8 0 shmpl 112 130 0 2 4 0 4 4 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 26055 0 24600 92 0 92 92 0 8 0 ffsino 272 26055 0 24600 98 0 98 98 0 8 0 nchpl 144 50949 0 49307 63 0 63 63 0 8 0 uvmvnodes 80 5926 0 0 121 0 121 121 0 8 0 vnodes 216 5926 0 0 330 0 330 330 0 8 0 namei 1024 172412 0 172411 8 7 1 2 0 8 0 percpumem 16 67 0 24 1 0 1 1 0 8 0 kstatmem 264 70 0 48 2 0 2 2 0 8 0 scxspl 216 161378 0 161376 51 50 1 8 1 8 0 plimitpl 152 2930 0 2907 22 21 1 2 0 8 0 sigapl 424 17354 0 17301 7 0 7 7 0 8 1 futexpl 64 150970 0 150967 5 4 1 1 0 8 0 knotepl 120 961 0 0 11 1 10 11 0 8 0 kqueuepl 216 4563 0 4555 85 84 1 8 0 8 0 pipepl 320 3838 0 3810 82 79 3 8 0 8 0 fdescpl 496 17336 0 17301 7 2 5 6 0 8 0 filepl 152 113630 0 113391 188 178 10 20 0 8 0 lockfpl 104 3291 0 3289 3 2 1 2 0 8 0 lockfspl 48 863 0 861 1 0 1 1 0 8 0 sessionpl 144 47 0 31 1 0 1 1 0 8 0 pgrppl 48 1562 0 1546 1 0 1 1 0 8 0 ucredpl 104 13131 0 13113 1 0 1 1 0 8 0 zombiepl 144 17301 0 17301 1 0 1 1 0 8 1 processpl 1072 17354 0 17301 5 1 4 5 0 8 0 procpl 680 47873 0 47800 39 32 7 9 0 8 0 sosppl 168 224 0 224 24 24 0 1 0 8 0 sockpl 488 23076 0 23043 473 466 7 37 0 8 0 mcl64k 65536 39 0 0 4 1 3 3 0 8 0 mcl16k 16384 26 0 0 3 0 3 3 0 8 0 mcl12k 12288 33 0 0 2 0 2 2 0 8 0 mcl9k 9216 25 0 0 2 0 2 2 0 8 0 mcl8k 8192 25 0 0 3 0 3 3 0 8 0 mcl4k 4096 128 0 0 6 2 4 4 0 8 0 mcl2k2 2112 16 0 0 2 0 2 2 0 8 0 mcl2k 2048 441 0 0 31 14 17 31 0 8 0 mtagpl 96 8 0 0 1 0 1 1 0 8 0 mbufpl 256 1670 0 0 58 0 58 58 0 8 0 bufpl 288 33884 0 27553 453 0 453 453 0 8 0 anonpl 24 1710808 0 1699256 246 154 92 118 0 186 0 amapchunkpl 152 541817 0 540945 119 80 39 50 0 158 0 amappl16 200 36517 0 36218 227 210 17 40 0 8 1 amappl15 192 24 0 23 1 0 1 1 0 8 0 amappl14 184 224 0 207 2 1 1 2 0 8 0 amappl13 176 27 0 27 7 7 0 1 0 8 0 amappl12 168 18327 0 18291 3 0 3 3 0 8 0 amappl11 160 64 0 53 1 0 1 1 0 8 0 amappl10 152 62 0 46 1 0 1 1 0 8 0 amappl9 144 354 0 354 35 34 1 1 0 8 1 amappl8 136 777 0 603 8 1 7 7 0 8 0 amappl7 128 158 0 145 2 0 2 2 0 8 0 amappl6 120 596 0 570 2 1 1 2 0 8 0 amappl5 112 674 0 665 1 0 1 1 0 8 0 amappl4 104 1044 0 1003 2 0 2 2 0 8 0 amappl3 96 105212 0 105129 4 1 3 4 0 8 0 amappl2 88 18495 0 18420 3 1 2 3 0 8 0 amappl1 80 69585 0 69063 25 13 12 22 0 8 0 amappl 88 168263 0 168006 7 0 7 7 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 132 0 2 3 0 3 3 0 8 0 uaddrrnd 24 17336 0 17301 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 17336 0 17301 1 0 1 1 0 8 0 vmmpekpl 168 133701 0 133630 4 0 4 4 0 8 0 vmmpepl 168 1010709 0 1008230 314 198 116 130 0 357 0 vmsppl 464 17335 0 17301 7 2 5 6 0 8 0 rwobjpl 56 252984 0 245513 122 13 109 111 0 8 0 pdppl 4096 34680 0 34602 585 505 80 98 0 8 2 pvpl 32 4921733 0 4904030 610 444 166 353 0 265 0 pmappl 248 17335 0 17301 4 1 3 3 0 8 0 extentpl 40 56 0 38 1 0 1 1 0 8 0 phpool 112 1784 0 921 26 0 26 26 0 8 0 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+0x1e: addq $0x8,%rsp ddb{0}> trace x86_ipi_db(ffffffff82c05ff0) at x86_ipi_db+0x1e sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 __mp_lock(ffffffff82cc89c8) at __mp_lock+0x129 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff82cc89c8) at __mp_lock+0x129 sys/kern/kern_lock.c:147 wakeup_n(ffffffff82cc9104,ffffffff) at wakeup_n+0x37 sys/kern/kern_synch.c:541 uvm_pmr_getpages(1,0,0,1,0,1,140ad26550d9480e,0) at uvm_pmr_getpages+0xeef sys/uvm/uvm_pmemrange.c:1207 uvm_pagealloc(fffffd806fed6128,3c10702000,0,3) at uvm_pagealloc+0x1a8 sys/uvm/uvm_page.c:910 pmap_get_ptp(fffffd806fed60f8,7820e04c2000) at pmap_get_ptp+0x18d sys/arch/amd64/amd64/pmap.c:1209 pmap_enter(fffffd806fed60f8,7820e04c2000,5d3b8000,1,20) at pmap_enter+0x2be sys/arch/amd64/amd64/pmap.c:2716 uvm_fault_upper_lookup(ffff8000295cb6c0,ffff8000295cb6f8,ffff8000295cb5c0,ffff8000295cb640) at uvm_fault_upper_lookup+0x2c0 sys/uvm/uvm_fault.c:895 uvm_fault(fffffd806ca0f928,7820e04c3000,0,1) at uvm_fault+0x129 sys/uvm/uvm_fault.c:605 upageflttrap(ffff8000295cb840,7820e04c3210) at upageflttrap+0x86 sys/arch/amd64/amd64/trap.c:188 usertrap(ffff8000295cb840) at usertrap+0x226 sys/arch/amd64/amd64/trap.c:436 recall_trap() at recall_trap+0x8 end of kernel end trace frame: 0x7820e04c3240, count: -14 ddb{0}> machine ddbcpu 1 Stopped at db_enter+0x1c: addq $0x8,%rsp ddb{1}> trace db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 witness_checkorder(ffff8000ffff7798,9,0) at witness_checkorder+0x10b7 witness_debugger sys/kern/subr_witness.c:2510 [inline] witness_checkorder(ffff8000ffff7798,9,0) at witness_checkorder+0x10b7 sys/kern/subr_witness.c:1110 mtx_enter(ffff8000ffff7788) at mtx_enter+0x3e sys/kern/kern_lock.c:265 single_thread_set(ffff800021238558,0,1) at single_thread_set+0x35a single_thread_wait sys/kern/kern_sig.c:2172 [inline] single_thread_set(ffff800021238558,0,1) at single_thread_set+0x35a sys/kern/kern_sig.c:2155 sigexit(ffff800021238558,6) at sigexit+0x90 sys/kern/kern_sig.c:1562 postsig(ffff800021238558,6,ffff80002e449218) at postsig+0x4a9 sys/kern/kern_sig.c:1494 userret(ffff800021238558) at userret+0x16e sys/kern/kern_sig.c:1987 syscall(ffff80002e449350) at syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline] syscall(ffff80002e449350) at syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x6b0829e4790, count: -9