witness: lock order reversal: 1st 0xffffffff82c65f30 &sched_lock (&sched_lock) 2nd 0xffff8000ffff4db8 &pr->ps_mtx (&pr->ps_mtx) lock order "&pr->ps_mtx"(mutex) -> "&sched_lock"(sched_lock) first seen at: #0 __mp_lock+0xa2 read_rflags machine/cpufunc.h:195 [inline] #0 __mp_lock+0xa2 intr_disable machine/cpufunc.h:216 [inline] #0 __mp_lock+0xa2 sys/kern/kern_lock.c:142 #1 donice+0x178 sys/kern/kern_resource.c:218 #2 syscall+0x5e2 mi_syscall sys/sys/syscall_mi.h:110 [inline] #2 syscall+0x5e2 sys/arch/amd64/amd64/trap.c:623 #3 Xsyscall+0x128 lock order "&sched_lock"(sched_lock) -> "&pr->ps_mtx"(mutex) first seen at: #0 mtx_enter+0x3e sys/kern/kern_lock.c:265 #1 single_thread_set+0x35a single_thread_wait sys/kern/kern_sig.c:2172 [inline] #1 single_thread_set+0x35a sys/kern/kern_sig.c:2155 #2 sigexit+0x90 sys/kern/kern_sig.c:1562 #3 postsig+0x4a9 sys/kern/kern_sig.c:1494 #4 userret+0x16e sys/kern/kern_sig.c:1987 #5 syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline] #5 syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644 #6 Xsyscall+0x128 Stopped at db_enter+0x1c: addq $0x8,%rsp ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic the kernel did not panic ddb{1}> trace db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 witness_checkorder(ffff8000ffff4db8,9,0) at witness_checkorder+0x10b7 witness_debugger sys/kern/subr_witness.c:2510 [inline] witness_checkorder(ffff8000ffff4db8,9,0) at witness_checkorder+0x10b7 sys/kern/subr_witness.c:1110 mtx_enter(ffff8000ffff4da8) at mtx_enter+0x3e sys/kern/kern_lock.c:265 single_thread_set(ffff800021263ab0,0,1) at single_thread_set+0x35a single_thread_wait sys/kern/kern_sig.c:2172 [inline] single_thread_set(ffff800021263ab0,0,1) at single_thread_set+0x35a sys/kern/kern_sig.c:2155 sigexit(ffff800021263ab0,6) at sigexit+0x90 sys/kern/kern_sig.c:1562 postsig(ffff800021263ab0,6,ffff80002e4914c8) at postsig+0x4a9 sys/kern/kern_sig.c:1494 userret(ffff800021263ab0) at userret+0x16e sys/kern/kern_sig.c:1987 syscall(ffff80002e491600) at syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline] syscall(ffff80002e491600) at syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xf54c3b3a650, count: -9 ddb{1}> show registers rdi 0x3 rsi 0xffffffff82befdd0 __sancov_gen_cov_switch_values.134 rbp 0xffff80002e4911d0 rbx 0x3 rdx 0 rcx 0xffff800021263ab0 rax 0xffff800020d58ff0 r8 0xffff80002e491140 r9 0x8080808080808080 r10 0x1f4cf1f37fcae817 r11 0x5085c660f7d1d7a1 r12 0xfffffd8003a0d7a0 r13 0 r14 0xfffffd80039f7c70 r15 0xfffffd80031cabc0 rip 0xffffffff8165630c db_enter+0x1c cs 0x8 rflags 0x246 rsp 0xffff80002e4911c0 ss 0x10 db_enter+0x1c: addq $0x8,%rsp ddb{1}> show proc PROC (syz-executor.6) pid=10277 stat=onproc flags process=500010 proc=4002000 pri=32, usrpri=81, nice=20 forw=0xffffffffffffffff, list=0xffff8000212a2d50,0xffff800021263818 process=0xffff8000ffff4ca8 user=0xffff80002e48c000, vmspace=0xfffffd806600b598 estcpu=36, cpticks=0, pctcpu=0.0 user=0, sys=0, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 73845 335432 51816 32767 7 0x10 syz-executor.7 63335 431084 45467 32767 4 0x582010 syz-executor.6 *63335 10277 45467 32767 7 0x4502010 syz-executor.6 84683 185426 34767 32767 3 0x90 nanoslp syz-executor.0 84683 357288 34767 32767 3 0x4000090 fsleep syz-executor.0 84683 302521 34767 32767 3 0x4000090 fsleep syz-executor.0 7274 249898 33491 32767 3 0x90 nanoslp syz-executor.5 33491 141661 30347 0 3 0x82 wait syz-executor.5 15249 229056 31770 32767 3 0x90 nanoslp syz-executor.3 31770 276864 30347 0 3 0x82 wait syz-executor.3 45467 93445 86554 32767 3 0x90 nanoslp syz-executor.6 86554 410335 30347 0 3 0x82 wait syz-executor.6 40933 482029 85964 32767 3 0x90 nanoslp syz-executor.2 85964 209176 30347 0 3 0x82 wait syz-executor.2 97463 510916 47639 32767 3 0x90 nanoslp syz-executor.4 47639 456102 30347 0 3 0x82 wait syz-executor.4 34767 296842 79050 32767 3 0x90 nanoslp syz-executor.0 79050 5755 30347 0 3 0x82 wait syz-executor.0 51816 517486 96177 32767 3 0x90 nanoslp syz-executor.7 96177 165332 30347 0 3 0x82 wait syz-executor.7 90469 80515 0 0 3 0x14200 bored sosplice 67374 493616 15841 32767 3 0x10 biowait syz-executor.1 15841 488849 30347 0 3 0x82 wait syz-executor.1 30347 411747 10474 0 3 0x2000082 thrsleep syz-fuzzer 30347 334950 10474 0 3 0x6000082 nanoslp syz-fuzzer 30347 346552 10474 0 3 0x6000082 wait syz-fuzzer 30347 156186 10474 0 3 0x6000082 wait syz-fuzzer 30347 417365 10474 0 3 0x6000082 thrsleep syz-fuzzer 30347 345973 10474 0 3 0x6000082 wait syz-fuzzer 30347 51482 10474 0 3 0x6000082 thrsleep syz-fuzzer 30347 364953 10474 0 3 0x6000082 thrsleep syz-fuzzer 30347 521512 10474 0 3 0x6000082 wait syz-fuzzer 30347 427296 10474 0 3 0x6000082 wait syz-fuzzer 30347 170321 10474 0 3 0x6000082 thrsleep syz-fuzzer 30347 60378 10474 0 3 0x6000082 wait syz-fuzzer 30347 115173 10474 0 3 0x6000082 wait syz-fuzzer 30347 430719 10474 0 3 0x6000082 wait syz-fuzzer 30347 105166 10474 0 3 0x6000082 kqread syz-fuzzer 30347 84281 10474 0 3 0x6000082 thrsleep syz-fuzzer 10474 380108 78614 0 3 0x10008a sigsusp ksh 78614 72187 74713 0 3 0x9a kqread sshd 63825 436090 1 0 3 0x100083 ttyin getty 74713 183524 1 0 3 0x88 kqread sshd 53430 370791 6875 73 3 0x1100090 kqread syslogd 6875 140852 1 0 3 0x100082 netio syslogd 78214 292075 1 0 3 0x100080 kqread resolvd 47665 107701 32658 77 3 0x100092 kqread dhcpleased 93314 374653 32658 77 3 0x100092 kqread dhcpleased 32658 326081 1 0 3 0x80 kqread dhcpleased 73895 336315 0 0 3 0x14200 bored smr 421 354014 0 0 2 0x14200 zerothread 3601 350487 0 0 3 0x14200 aiodoned aiodoned 4679 263522 0 0 3 0x14200 syncer update 75500 136017 0 0 3 0x14200 cleaner cleaner 2695 163371 0 0 3 0x14200 reaper reaper 53096 255977 0 0 3 0x14200 pgdaemon pagedaemon 45685 163682 0 0 3 0x14200 bored viomb 54797 465826 0 0 3 0x40014200 acpi0 acpi0 32459 3194 0 0 3 0x40014200 idle1 16117 155930 0 0 3 0x14200 bored softnet3 38031 452139 0 0 3 0x14200 bored softnet2 56761 306729 0 0 3 0x14200 bored softnet1 61451 74135 0 0 3 0x14200 bored softnet0 87214 310885 0 0 3 0x14200 bored systqmp 59930 147921 0 0 3 0x14200 bored systq 20301 203296 0 0 3 0x40014200 bored softclock 25744 299751 0 0 3 0x40014200 idle0 1 428447 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{1}> show all locks CPU 0: exclusive mutex &uvm.fpageqlock r = 0 (0xffffffff82d03148) #0 witness_lock+0x447 #1 mtx_enter_try+0x104 #2 mtx_enter+0x4f sys/kern/kern_lock.c:266 #3 uvm_pmr_getpages+0xeac sys/uvm/uvm_pmemrange.c:1205 #4 uvm_pagealloc+0x1a8 sys/uvm/uvm_page.c:910 #5 pmap_get_ptp+0x18d sys/arch/amd64/amd64/pmap.c:1209 #6 pmap_enter+0x2be sys/arch/amd64/amd64/pmap.c:2716 #7 uvm_fault_lower_lookup+0x2a4 sys/uvm/uvm_fault.c:1192 #8 uvm_fault_lower+0x63 sys/uvm/uvm_fault.c:1227 #9 uvm_fault+0x238 #10 upageflttrap+0x86 sys/arch/amd64/amd64/trap.c:188 #11 usertrap+0x226 sys/arch/amd64/amd64/trap.c:436 #12 recall_trap+0x8 exclusive mutex &pmap->pm_mtx r = 0 (0xfffffd806600a6d8) #0 witness_lock+0x447 #1 mtx_enter_try+0x104 #2 mtx_enter+0x4f sys/kern/kern_lock.c:266 #3 pmap_enter+0x1c3 pmap_map_ptes sys/arch/amd64/amd64/pmap.c:423 [inline] #3 pmap_enter+0x1c3 sys/arch/amd64/amd64/pmap.c:2710 #4 uvm_fault_lower_lookup+0x2a4 sys/uvm/uvm_fault.c:1192 #5 uvm_fault_lower+0x63 sys/uvm/uvm_fault.c:1227 #6 uvm_fault+0x238 #7 upageflttrap+0x86 sys/arch/amd64/amd64/trap.c:188 #8 usertrap+0x226 sys/arch/amd64/amd64/trap.c:436 #9 recall_trap+0x8 CPU 1: exclusive sched_lock &sched_lock r = 0 (0xffffffff82c65f30) #0 witness_lock+0x447 #1 single_thread_set+0x179 sys/kern/kern_sig.c:2113 #2 sigexit+0x90 sys/kern/kern_sig.c:1562 #3 postsig+0x4a9 sys/kern/kern_sig.c:1494 #4 userret+0x16e sys/kern/kern_sig.c:1987 #5 syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline] #5 syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644 #6 Xsyscall+0x128 Process 73845 (syz-executor.7) thread 0xffff800021263808 (335432) exclusive rwlock uobjlk r = 0 (0xfffffd8071fee780) #0 witness_lock+0x447 #1 rw_enter+0x3c8 sys/kern/kern_rwlock.c:309 #2 uvm_fault_lower_lookup+0x4a sys/uvm/uvm_fault.c:1126 #3 uvm_fault_lower+0x63 sys/uvm/uvm_fault.c:1227 #4 uvm_fault+0x238 #5 upageflttrap+0x86 sys/arch/amd64/amd64/trap.c:188 #6 usertrap+0x226 sys/arch/amd64/amd64/trap.c:436 #7 recall_trap+0x8 shared rwlock vmmaplk r = 0 (0xfffffd806600b4c0) #0 witness_lock+0x447 #1 uvmfault_lookup+0xd9 sys/uvm/uvm_fault.c:1785 #2 uvm_fault_check+0x3e sys/uvm/uvm_fault.c:672 #3 uvm_fault+0xf2 sys/uvm/uvm_fault.c:600 #4 upageflttrap+0x86 sys/arch/amd64/amd64/trap.c:188 #5 usertrap+0x226 sys/arch/amd64/amd64/trap.c:436 #6 recall_trap+0x8 Process 63335 (syz-executor.6) thread 0xffff800021263ab0 (10277) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff82d03430) #0 witness_lock+0x447 #1 postsig+0x49e sys/kern/kern_sig.c:1500 #2 userret+0x16e sys/kern/kern_sig.c:1987 #3 syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline] #3 syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644 #4 Xsyscall+0x128 exclusive sched_lock &sched_lock r = 0 (0xffffffff82c65f30) #0 witness_lock+0x447 #1 single_thread_set+0x179 sys/kern/kern_sig.c:2113 #2 sigexit+0x90 sys/kern/kern_sig.c:1562 #3 postsig+0x4a9 sys/kern/kern_sig.c:1494 #4 userret+0x16e sys/kern/kern_sig.c:1987 #5 syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline] #5 syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644 #6 Xsyscall+0x128 Process 67374 (syz-executor.1) thread 0xffff80002120dd50 (493616) exclusive rrwlock inode r = 0 (0xfffffd807219d2c0) #0 witness_lock+0x447 #1 rw_enter+0x3c8 sys/kern/kern_rwlock.c:309 #2 rrw_enter+0x8c sys/kern/kern_rwlock.c:464 #3 VOP_LOCK+0x8b sys/kern/vfs_vops.c:518 #4 ufs_ihashins+0x46 sys/ufs/ufs/ufs_ihash.c:140 #5 ffs_vget+0x141 sys/ufs/ffs/ffs_vfsops.c:1343 #6 ffs_inode_alloc+0x1c2 sys/ufs/ffs/ffs_alloc.c:394 #7 ufs_mkdir+0xf8 sys/ufs/ufs/ufs_vnops.c:1149 #8 VOP_MKDIR+0xc3 sys/kern/vfs_vops.c:388 #9 domkdirat+0x125 sys/kern/vfs_syscalls.c:3073 #10 syscall+0x5e2 mi_syscall sys/sys/syscall_mi.h:110 [inline] #10 syscall+0x5e2 sys/arch/amd64/amd64/trap.c:623 #11 Xsyscall+0x128 exclusive rrwlock inode r = 0 (0xfffffd806d636e78) #0 witness_lock+0x447 #1 rw_enter+0x3c8 sys/kern/kern_rwlock.c:309 #2 rrw_enter+0x8c sys/kern/kern_rwlock.c:464 #3 VOP_LOCK+0x8b sys/kern/vfs_vops.c:518 #4 vn_lock+0x84 sys/kern/vfs_vnops.c:564 #5 vfs_lookup+0xd5 sys/kern/vfs_lookup.c:418 #6 namei+0x55a sys/kern/vfs_lookup.c:250 #7 domkdirat+0x79 sys/kern/vfs_syscalls.c:3058 #8 syscall+0x5e2 mi_syscall sys/sys/syscall_mi.h:110 [inline] #8 syscall+0x5e2 sys/arch/amd64/amd64/trap.c:623 #9 Xsyscall+0x128 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10236 6415K 6420K 78643K 11415 0 pcb 13 16K 20K 78643K 19 0 rtable 248 7K 7K 78643K 2197 0 pf 29 8K 8K 78643K 95 0 ifaddr 44 15K 16K 78643K 182 0 ifgroup 50 2K 2K 78643K 182 0 sysctl 3 1K 1K 78643K 3 0 counters 60 35K 35K 78643K 126 0 ioctlops 0 0K 2K 78643K 498 0 iov 0 0K 28K 78643K 3072 0 mount 1 1K 1K 78643K 1 0 log 0 0K 0K 78643K 4 0 vnodes 1279 80K 80K 78643K 4508 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 9K 78643K 303 0 VM map 2 1K 1K 78643K 2 0 sem 12 0K 0K 78643K 2151 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1697 195K 286K 78643K 12548 0 file desc 21 77K 129K 78643K 18934 0 sigio 1 0K 0K 78643K 456 0 proc 56 78K 127K 78643K 2729 0 subproc 104 6K 6K 78643K 533 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 1073 0 in_multi 99 7K 7K 78643K 640 0 ether_multi 1 0K 0K 78643K 21 0 mrt 1 0K 0K 78643K 1 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 355 1579K 1579K 78643K 355 0 exec 0 0K 1K 78643K 2425 0 tdb 3 0K 0K 78643K 3 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 8 62K 64K 78643K 10 0 UVM amap 448 98K 114K 78643K 185490 0 UVM aobj 131 4K 4K 78643K 131 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 0K 78643K 418 0 NDP 11 0K 2K 78643K 126 0 temp 74 5920K 6048K 78643K 48768 0 kqueue 12 18K 47K 78643K 2068 0 SYN cache 2 16K 16K 78643K 2 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 22 0 0 1 0 1 1 0 8 0 rtpcb 120 862 0 859 11 10 1 3 0 8 0 rtentry 112 522 0 405 4 0 4 4 0 8 0 unpcb 144 14990 0 14977 140 139 1 11 0 8 0 syncache 304 221 0 221 38 38 0 1 0 8 0 tcpqe 32 570 0 570 33 33 0 1 0 8 0 tcpcb 808 20965 0 20946 305 293 12 24 0 8 8 arp 120 88 0 69 1 0 1 1 0 8 0 ipq 40 102 0 102 9 9 0 1 0 8 0 ipqe 40 285 0 285 9 9 0 1 0 8 0 inpcb 368 27292 0 27270 271 259 12 22 0 8 7 nd6 136 167 0 137 5 3 2 2 0 8 0 kcovpl 48 41 0 33 1 0 1 1 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 2072 0 1601 34 4 30 31 0 8 0 art_table 32 2073 0 1601 4 0 4 4 0 8 0 art_node 16 521 0 414 1 0 1 1 0 8 0 sysvmsgpl 40 10 0 3 1 0 1 1 0 8 0 semupl 112 3 0 3 1 1 0 1 0 8 0 semapl 112 2147 0 2137 1 0 1 1 0 8 0 shmpl 112 128 0 0 4 0 4 4 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 25059 0 23601 92 0 92 92 0 8 0 ffsino 272 25059 0 23601 98 0 98 98 0 8 0 nchpl 144 49135 0 47487 63 0 63 63 0 8 0 uvmvnodes 80 5926 0 0 121 0 121 121 0 8 0 vnodes 216 5926 0 0 330 0 330 330 0 8 0 namei 1024 162955 0 162954 6 5 1 2 0 8 0 percpumem 16 76 0 33 1 0 1 1 0 8 0 kstatmem 264 88 0 66 2 0 2 2 0 8 0 scxspl 216 144660 0 144659 43 42 1 8 1 8 0 plimitpl 152 2287 0 2264 25 24 1 2 0 8 0 sigapl 424 19161 0 19107 7 0 7 7 0 8 0 futexpl 64 151311 0 151309 6 5 1 1 0 8 0 knotepl 120 1214 0 0 19 1 18 18 0 8 0 kqueuepl 216 4513 0 4505 78 73 5 8 0 8 4 pipepl 320 5679 0 5651 123 120 3 11 0 8 0 fdescpl 496 19143 0 19111 7 2 5 6 0 8 0 filepl 152 126572 0 126339 229 215 14 21 0 8 4 lockfpl 104 4290 0 4288 1 0 1 1 0 8 0 lockfspl 48 1332 0 1330 1 0 1 1 0 8 0 sessionpl 144 56 0 40 1 0 1 1 0 8 0 pgrppl 48 390 0 374 1 0 1 1 0 8 0 ucredpl 104 15993 0 15975 1 0 1 1 0 8 0 zombiepl 144 19111 0 19107 1 0 1 1 0 8 0 processpl 1072 19161 0 19107 5 1 4 5 0 8 0 procpl 680 51969 0 51897 32 24 8 8 0 8 1 sosppl 168 334 0 334 21 20 1 1 0 8 1 sockpl 488 44325 0 44287 749 732 17 36 0 8 8 mcl64k 65536 25 0 0 3 0 3 3 0 8 0 mcl16k 16384 25 0 0 4 1 3 3 0 8 0 mcl12k 12288 25 0 0 2 0 2 2 0 8 0 mcl9k 9216 13 0 0 1 0 1 1 0 8 0 mcl8k 8192 49 0 0 4 1 3 3 0 8 0 mcl4k 4096 75 0 0 6 3 3 4 0 8 0 mcl2k2 2112 35 0 0 2 0 2 2 0 8 0 mcl2k 2048 452 0 0 32 15 17 32 0 8 0 mtagpl 96 17 0 0 1 0 1 1 0 8 0 mbufpl 256 1672 0 0 74 0 74 74 0 8 0 bufpl 288 29563 0 23235 453 0 453 453 0 8 0 anonpl 24 1868577 0 1857132 257 148 109 109 0 186 14 amapchunkpl 152 597819 0 596972 144 99 45 55 0 158 5 amappl16 200 38669 0 38378 259 233 26 29 0 8 8 amappl15 192 14 0 12 1 0 1 1 0 8 0 amappl14 184 283 0 266 2 1 1 2 0 8 0 amappl13 176 38 0 37 1 0 1 1 0 8 0 amappl12 168 20285 0 20250 2 0 2 2 0 8 0 amappl11 160 51 0 41 1 0 1 1 0 8 0 amappl10 152 146 0 132 1 0 1 1 0 8 0 amappl9 144 339 0 339 11 10 1 1 0 8 1 amappl8 136 1099 0 892 8 0 8 8 0 8 0 amappl7 128 123 0 109 2 1 1 2 0 8 0 amappl6 120 629 0 606 2 1 1 2 0 8 0 amappl5 112 733 0 724 1 0 1 1 0 8 0 amappl4 104 1013 0 969 4 2 2 3 0 8 0 amappl3 96 115149 0 115068 12 9 3 4 0 8 0 amappl2 88 20199 0 20122 3 1 2 3 0 8 0 amappl1 80 76381 0 75874 21 9 12 21 0 8 0 amappl 88 184034 0 183786 10 3 7 8 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 130 0 0 3 0 3 3 0 8 0 uaddrrnd 24 19143 0 19111 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 19143 0 19111 1 0 1 1 0 8 0 vmmpekpl 168 152552 0 152486 4 0 4 4 0 8 0 vmmpepl 168 1113566 0 1111185 297 170 127 131 0 357 7 vmsppl 464 19142 0 19111 7 2 5 6 0 8 0 rwobjpl 56 273753 0 266268 128 18 110 110 0 8 0 pdppl 4096 38294 0 38222 690 608 82 98 0 8 10 pvpl 32 5270689 0 5253221 672 471 201 362 0 265 32 pmappl 248 19142 0 19111 4 1 3 3 0 8 0 extentpl 40 56 0 38 1 0 1 1 0 8 0 phpool 112 2111 0 1215 27 0 27 27 0 8 0 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+0x1e: addq $0x8,%rsp ddb{0}> trace x86_ipi_db(ffffffff82bc3ff0) at x86_ipi_db+0x1e sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 __mp_lock(ffffffff82c65d28) at __mp_lock+0x129 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff82c65d28) at __mp_lock+0x129 sys/kern/kern_lock.c:147 wakeup_n(ffffffff82d02fb4,ffffffff) at wakeup_n+0x37 sys/kern/kern_synch.c:541 uvm_pmr_getpages(1,0,0,1,0,1,ecfed32093ae4f2e,0) at uvm_pmr_getpages+0xeef sys/uvm/uvm_pmemrange.c:1207 uvm_pagealloc(fffffd806600a6f8,759dcb000,0,3) at uvm_pagealloc+0x1a8 sys/uvm/uvm_page.c:910 pmap_get_ptp(fffffd806600a6c8,eb3b971f000) at pmap_get_ptp+0x18d sys/arch/amd64/amd64/pmap.c:1209 pmap_enter(fffffd806600a6c8,eb3b971f000,63825000,4,20) at pmap_enter+0x2be sys/arch/amd64/amd64/pmap.c:2716 uvm_fault_lower_lookup(ffff80002e4973c0,ffff80002e4973f8,ffff80002e497340) at uvm_fault_lower_lookup+0x2a4 sys/uvm/uvm_fault.c:1192 uvm_fault_lower(ffff80002e4973c0,ffff80002e4973f8,ffff80002e497340,0) at uvm_fault_lower+0x63 sys/uvm/uvm_fault.c:1227 uvm_fault(fffffd806600b3c8,eb3b9722000,0,4) at uvm_fault+0x238 upageflttrap(ffff80002e497540,eb3b9722c9b) at upageflttrap+0x86 sys/arch/amd64/amd64/trap.c:188 usertrap(ffff80002e497540) at usertrap+0x226 sys/arch/amd64/amd64/trap.c:436 recall_trap() at recall_trap+0x8 end of kernel end trace frame: 0x7caedea6c590, count: -15 ddb{0}> machine ddbcpu 1 Stopped at db_enter+0x1c: addq $0x8,%rsp ddb{1}> trace db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 witness_checkorder(ffff8000ffff4db8,9,0) at witness_checkorder+0x10b7 witness_debugger sys/kern/subr_witness.c:2510 [inline] witness_checkorder(ffff8000ffff4db8,9,0) at witness_checkorder+0x10b7 sys/kern/subr_witness.c:1110 mtx_enter(ffff8000ffff4da8) at mtx_enter+0x3e sys/kern/kern_lock.c:265 single_thread_set(ffff800021263ab0,0,1) at single_thread_set+0x35a single_thread_wait sys/kern/kern_sig.c:2172 [inline] single_thread_set(ffff800021263ab0,0,1) at single_thread_set+0x35a sys/kern/kern_sig.c:2155 sigexit(ffff800021263ab0,6) at sigexit+0x90 sys/kern/kern_sig.c:1562 postsig(ffff800021263ab0,6,ffff80002e4914c8) at postsig+0x4a9 sys/kern/kern_sig.c:1494 userret(ffff800021263ab0) at userret+0x16e sys/kern/kern_sig.c:1987 syscall(ffff80002e491600) at syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline] syscall(ffff80002e491600) at syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xf54c3b3a650, count: -9