witness: lock order reversal: 1st 0xffffffff82c75c58 &sched_lock (&sched_lock) 2nd 0xffff8000212c0988 &pr->ps_mtx (&pr->ps_mtx) lock order "&pr->ps_mtx"(mutex) -> "&sched_lock"(sched_lock) first seen at: #0 __mp_lock+0xa2 read_rflags machine/cpufunc.h:195 [inline] #0 __mp_lock+0xa2 intr_disable machine/cpufunc.h:216 [inline] #0 __mp_lock+0xa2 sys/kern/kern_lock.c:142 #1 donice+0x178 sys/kern/kern_resource.c:218 #2 syscall+0x5e2 mi_syscall sys/sys/syscall_mi.h:110 [inline] #2 syscall+0x5e2 sys/arch/amd64/amd64/trap.c:623 #3 Xsyscall+0x128 lock order "&sched_lock"(sched_lock) -> "&pr->ps_mtx"(mutex) first seen at: #0 mtx_enter+0x3e sys/kern/kern_lock.c:265 #1 single_thread_set+0x35a single_thread_wait sys/kern/kern_sig.c:2172 [inline] #1 single_thread_set+0x35a sys/kern/kern_sig.c:2155 #2 sigexit+0x90 sys/kern/kern_sig.c:1562 #3 postsig+0x4a9 sys/kern/kern_sig.c:1494 #4 userret+0x16e sys/kern/kern_sig.c:1987 #5 syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline] #5 syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644 #6 Xsyscall+0x128 Stopped at db_enter+0x1c: addq $0x8,%rsp ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic the kernel did not panic ddb{0}> trace db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 witness_checkorder(ffff8000212c0988,9,0) at witness_checkorder+0x10b7 witness_debugger sys/kern/subr_witness.c:2510 [inline] witness_checkorder(ffff8000212c0988,9,0) at witness_checkorder+0x10b7 sys/kern/subr_witness.c:1110 mtx_enter(ffff8000212c0978) at mtx_enter+0x3e sys/kern/kern_lock.c:265 single_thread_set(ffff80002129f2a8,0,1) at single_thread_set+0x35a single_thread_wait sys/kern/kern_sig.c:2172 [inline] single_thread_set(ffff80002129f2a8,0,1) at single_thread_set+0x35a sys/kern/kern_sig.c:2155 sigexit(ffff80002129f2a8,6) at sigexit+0x90 sys/kern/kern_sig.c:1562 postsig(ffff80002129f2a8,6,ffff80002c9c7688) at postsig+0x4a9 sys/kern/kern_sig.c:1494 userret(ffff80002129f2a8) at userret+0x16e sys/kern/kern_sig.c:1987 syscall(ffff80002c9c77c0) at syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline] syscall(ffff80002c9c77c0) at syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xf2c76a3ee30, count: -9 ddb{0}> show registers rdi 0x3 rsi 0x40000 acpi_pdirpa+0x2be63 rbp 0xffff80002c9c7390 rbx 0x3 rdx 0xffff800000d17fc0 rcx 0xffff80002129f2a8 rax 0x3ffff acpi_pdirpa+0x2be62 r8 0xffff80002c9c7300 r9 0x8080808080808080 r10 0x38cf7e11771b0439 r11 0x9b6bf874e7eb1444 r12 0xfffffd8003a0d850 r13 0 r14 0xfffffd80039fa710 r15 0xfffffd80031cabc0 rip 0xffffffff82131f9c db_enter+0x1c cs 0x8 rflags 0x246 rsp 0xffff80002c9c7380 ss 0x10 db_enter+0x1c: addq $0x8,%rsp ddb{0}> show proc PROC (syz-executor.0) pid=33975 stat=onproc flags process=500000 proc=4002000 pri=32, usrpri=82, nice=20 forw=0xffffffffffffffff, list=0xffff80002129e808,0xffffffff82d74600 process=0xffff8000212c0878 user=0xffff80002c9c2000, vmspace=0xfffffd806b841cc8 estcpu=36, cpticks=0, pctcpu=0.0 user=0, sys=0, intr=0 ddb{0}> ps PID TID PPID UID S FLAGS WAIT COMMAND 31679 179307 36160 0 4 0x582000 syz-executor.0 *31679 33975 36160 0 7 0x4502000 syz-executor.0 71646 509441 59255 0 2 0 syz-executor.6 92503 373365 72792 0 2 0 syz-executor.5 92503 86249 72792 0 2 0x4000000 syz-executor.5 92503 401151 72792 0 3 0x4000080 fsleep syz-executor.5 79245 15061 79134 0 2 0x2 syz-executor.7 59255 195575 79134 0 3 0x82 nanoslp syz-executor.6 72792 165067 79134 0 3 0x82 nanoslp syz-executor.5 48304 460190 79134 0 3 0x82 piperd syz-executor.4 20465 469639 79134 0 7 0x2 syz-executor.3 19222 29141 79134 0 3 0x82 piperd syz-executor.2 59764 86119 79134 0 2 0x2 syz-executor.1 36160 383491 79134 0 3 0x82 nanoslp syz-executor.0 79134 503033 7075 0 3 0x2000082 wait syz-fuzzer 79134 307626 7075 0 3 0x6000082 nanoslp syz-fuzzer 79134 88786 7075 0 3 0x6000082 thrsleep syz-fuzzer 79134 235921 7075 0 3 0x6000082 wait syz-fuzzer 79134 48249 7075 0 3 0x6000082 wait syz-fuzzer 79134 221628 7075 0 3 0x6000082 kqread syz-fuzzer 79134 497356 7075 0 3 0x6000082 wait syz-fuzzer 79134 242036 7075 0 3 0x6000082 wait syz-fuzzer 79134 479605 7075 0 3 0x6000082 thrsleep syz-fuzzer 79134 447392 7075 0 3 0x6000082 wait syz-fuzzer 79134 403618 7075 0 3 0x6000082 thrsleep syz-fuzzer 79134 152572 7075 0 3 0x6000082 wait syz-fuzzer 79134 348806 7075 0 3 0x6000082 wait syz-fuzzer 79134 159938 7075 0 3 0x6000082 thrsleep syz-fuzzer 79134 42283 7075 0 3 0x6000082 thrsleep syz-fuzzer 79134 128920 7075 0 3 0x6000082 thrsleep syz-fuzzer 79134 465755 7075 0 3 0x6000082 thrsleep syz-fuzzer 7075 86825 58580 0 3 0x10008a sigsusp ksh 58580 128672 93708 0 3 0x9a kqread sshd 99734 378853 1 0 3 0x100083 ttyin getty 93708 2717 1 0 3 0x88 kqread sshd 95117 316470 88378 74 3 0x1100092 bpf pflogd 88378 441301 1 0 3 0x80 netio pflogd 59095 363913 61234 73 3 0x1100090 kqread syslogd 61234 71427 1 0 3 0x100082 netio syslogd 55426 248893 1 0 3 0x100080 kqread resolvd 7347 233371 3172 77 3 0x100092 kqread dhcpleased 11555 197560 3172 77 3 0x100092 kqread dhcpleased 3172 342798 1 0 3 0x80 kqread dhcpleased 88676 348008 0 0 3 0x14200 bored smr 7882 243603 0 0 2 0x14200 zerothread 49143 293279 0 0 3 0x14200 aiodoned aiodoned 75539 510569 0 0 3 0x14200 syncer update 87308 29767 0 0 3 0x14200 cleaner cleaner 57252 150114 0 0 3 0x14200 reaper reaper 33921 344376 0 0 3 0x14200 pgdaemon pagedaemon 61372 125730 0 0 3 0x14200 bored viomb 74556 503179 0 0 3 0x40014200 acpi0 acpi0 27337 69575 0 0 3 0x40014200 idle1 29380 3336 0 0 3 0x14200 bored softnet3 60611 5694 0 0 3 0x14200 bored softnet2 50649 11067 0 0 3 0x14200 bored softnet1 42732 297853 0 0 3 0x14200 bored softnet0 38884 96255 0 0 3 0x14200 bored systqmp 14701 219599 0 0 3 0x14200 bored systq 66329 302540 0 0 3 0x40014200 bored softclock 32807 397623 0 0 3 0x40014200 idle0 1 20203 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{0}> show all locks CPU 0: exclusive sched_lock &sched_lock r = 0 (0xffffffff82c75c58) #0 witness_lock+0x447 #1 single_thread_set+0x179 sys/kern/kern_sig.c:2113 #2 sigexit+0x90 sys/kern/kern_sig.c:1562 #3 postsig+0x4a9 sys/kern/kern_sig.c:1494 #4 userret+0x16e sys/kern/kern_sig.c:1987 #5 syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline] #5 syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644 #6 Xsyscall+0x128 Process 31679 (syz-executor.0) thread 0xffff80002129f2a8 (33975) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff82d78a60) #0 witness_lock+0x447 #1 postsig+0x49e sys/kern/kern_sig.c:1500 #2 userret+0x16e sys/kern/kern_sig.c:1987 #3 syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline] #3 syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644 #4 Xsyscall+0x128 exclusive sched_lock &sched_lock r = 0 (0xffffffff82c75c58) #0 witness_lock+0x447 #1 single_thread_set+0x179 sys/kern/kern_sig.c:2113 #2 sigexit+0x90 sys/kern/kern_sig.c:1562 #3 postsig+0x4a9 sys/kern/kern_sig.c:1494 #4 userret+0x16e sys/kern/kern_sig.c:1987 #5 syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline] #5 syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644 #6 Xsyscall+0x128 ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10188 6482K 6515K 78643K 17417 0 pcb 13 8K 8K 78643K 91 0 rtable 234 6K 6K 78643K 354 0 pf 32 9K 10K 78643K 43 0 ifaddr 45 15K 15K 78643K 47 0 ifgroup 55 2K 2K 78643K 55 0 counters 60 35K 35K 78643K 60 0 ioctlops 0 0K 4K 78643K 3935 0 iov 0 0K 1K 78643K 62 0 mount 1 1K 1K 78643K 1 0 log 0 0K 0K 78643K 4 0 vnodes 1409 88K 88K 78643K 5748 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 9K 78643K 92 0 VM map 2 1K 1K 78643K 2 0 sem 12 0K 0K 78643K 729 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1697 195K 286K 78643K 12548 0 file desc 13 45K 93K 78643K 14138 0 proc 67 91K 115K 78643K 526 0 subproc 104 6K 6K 78643K 104 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 in_multi 99 7K 7K 78643K 99 0 ether_multi 1 0K 0K 78643K 1 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 25 122K 122K 78643K 25 0 exec 0 0K 1K 78643K 387 0 tdb 3 0K 0K 78643K 3 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 8 62K 64K 78643K 10 0 UVM amap 300 85K 86K 78643K 122032 0 UVM aobj 131 7K 7K 78643K 131 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 NDP 12 0K 1K 78643K 28 0 temp 73 5916K 6044K 78643K 38058 0 kqueue 12 18K 20K 78643K 102 0 SYN cache 2 16K 16K 78643K 2 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 22 0 0 1 0 1 1 0 8 0 rtpcb 120 36 0 33 1 0 1 1 0 8 0 rtentry 112 111 0 1 4 0 4 4 0 8 0 unpcb 144 2720 0 2705 9 8 1 6 0 8 0 syncache 304 4 0 4 1 1 0 1 0 8 0 tcpqe 32 150 0 150 1 1 0 1 0 8 0 tcpcb 808 8 0 4 1 0 1 1 0 8 0 arp 120 18 0 0 1 0 1 1 0 8 0 inpcb 368 252 0 245 1 0 1 1 0 8 0 nd6 136 24 0 0 1 0 1 1 0 8 0 kcovpl 48 8 0 0 1 0 1 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfstitem 24 27 0 25 1 0 1 1 0 8 0 pfstkey 128 27 0 25 1 0 1 1 0 8 0 pfstate 376 27 0 25 3 2 1 3 0 8 0 pfrule 1344 21 0 16 2 1 1 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 452 0 0 29 0 29 29 0 8 0 art_table 32 453 0 0 4 0 4 4 0 8 0 art_node 16 110 0 10 1 0 1 1 0 8 0 sysvmsgpl 40 3 0 0 1 0 1 1 0 8 0 semapl 112 727 0 717 1 0 1 1 0 8 0 shmpl 112 128 0 0 4 0 4 4 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 15941 0 14494 91 0 91 91 0 8 0 ffsino 272 15941 0 14494 97 0 97 97 0 8 0 nchpl 144 36833 0 35148 64 0 64 64 0 8 0 uvmvnodes 80 5926 0 0 121 0 121 121 0 8 0 vnodes 216 5926 0 0 330 0 330 330 0 8 0 namei 1024 107016 0 107016 2 1 1 2 0 8 1 percpumem 16 43 0 0 1 0 1 1 0 8 0 kstatmem 264 24 0 0 2 0 2 2 0 8 0 scxspl 216 93387 0 93387 10 9 1 8 1 8 1 plimitpl 152 34 0 18 1 0 1 1 0 8 0 sigapl 424 14458 0 14415 8 2 6 7 0 8 0 futexpl 64 134281 0 134280 1 0 1 1 0 8 0 knotepl 120 114 0 0 4 0 4 4 0 8 0 kqueuepl 216 3860 0 3852 38 37 1 8 0 8 0 pipepl 320 141 0 113 3 0 3 3 0 8 0 fdescpl 496 14441 0 14415 5 0 5 5 0 8 0 filepl 152 59906 0 59665 58 48 10 17 0 8 0 lockfpl 104 594 0 592 1 0 1 1 0 8 0 lockfspl 48 269 0 267 1 0 1 1 0 8 0 sessionpl 144 24 0 7 1 0 1 1 0 8 0 pgrppl 48 1628 0 1611 1 0 1 1 0 8 0 ucredpl 104 62 0 50 1 0 1 1 0 8 0 zombiepl 144 14415 0 14415 1 0 1 1 0 8 1 processpl 1072 14458 0 14415 4 0 4 4 0 8 0 procpl 680 30219 0 30157 8 1 7 7 0 8 0 sockpl 488 3008 0 2983 31 27 4 20 0 8 0 mcl64k 65536 65 0 0 4 1 3 3 0 8 0 mcl16k 16384 93 0 0 6 4 2 3 0 8 0 mcl12k 12288 102 0 0 3 1 2 2 0 8 0 mcl9k 9216 30 0 0 2 1 1 2 0 8 0 mcl8k 8192 88 0 0 6 4 2 3 0 8 0 mcl4k 4096 63 0 0 4 2 2 3 0 8 0 mcl2k2 2112 19 0 0 2 0 2 2 0 8 0 mcl2k 2048 299 0 0 36 6 30 36 0 8 0 mtagpl 96 2 0 0 1 0 1 1 0 8 0 mbufpl 256 1029 0 0 19 1 18 19 0 8 0 bufpl 288 18580 0 12253 453 0 453 453 0 8 0 anonpl 24 1143616 0 1135498 60 8 52 59 0 186 0 amapchunkpl 152 383972 0 383257 131 99 32 35 0 158 1 amappl16 200 17804 0 17624 10 0 10 10 0 8 0 amappl15 192 15 0 14 1 0 1 1 0 8 0 amappl14 184 165 0 151 2 1 1 2 0 8 0 amappl13 176 47 0 46 1 0 1 1 0 8 0 amappl12 168 15132 0 15105 3 1 2 2 0 8 0 amappl11 160 59 0 45 1 0 1 1 0 8 0 amappl10 152 40 0 29 1 0 1 1 0 8 0 amappl9 144 715 0 715 48 48 0 1 0 8 0 amappl8 136 343 0 272 3 0 3 3 0 8 0 amappl7 128 147 0 132 2 0 2 2 0 8 0 amappl6 120 1743 0 1723 2 1 1 2 0 8 0 amappl5 112 853 0 840 1 0 1 1 0 8 0 amappl4 104 1273 0 1227 3 1 2 3 0 8 0 amappl3 96 78467 0 78398 4 1 3 3 0 8 0 amappl2 88 14992 0 14924 3 1 2 3 0 8 0 amappl1 80 55271 0 54723 24 10 14 23 0 8 0 amappl 88 121486 0 121298 7 1 6 6 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 130 0 0 3 0 3 3 0 8 0 uaddrrnd 24 14441 0 14415 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 14441 0 14415 1 0 1 1 0 8 0 vmmpekpl 168 68110 0 68060 3 0 3 3 0 8 0 vmmpepl 168 779048 0 777082 271 175 96 119 0 357 2 vmsppl 464 14440 0 14415 5 1 4 5 0 8 0 rwobjpl 56 169374 0 162061 105 1 104 104 0 8 0 pdppl 4096 28890 0 28830 1578 1512 66 84 0 8 6 pvpl 32 4118367 0 4104214 415 293 122 369 0 265 0 pmappl 248 14440 0 14415 3 1 2 3 0 8 0 extentpl 40 56 0 38 1 0 1 1 0 8 0 phpool 112 2342 0 1534 24 0 24 24 0 8 0 ddb{0}> machine ddbcpu 0 Invalid cpu 0 ddb{0}> trace db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 witness_checkorder(ffff8000212c0988,9,0) at witness_checkorder+0x10b7 witness_debugger sys/kern/subr_witness.c:2510 [inline] witness_checkorder(ffff8000212c0988,9,0) at witness_checkorder+0x10b7 sys/kern/subr_witness.c:1110 mtx_enter(ffff8000212c0978) at mtx_enter+0x3e sys/kern/kern_lock.c:265 single_thread_set(ffff80002129f2a8,0,1) at single_thread_set+0x35a single_thread_wait sys/kern/kern_sig.c:2172 [inline] single_thread_set(ffff80002129f2a8,0,1) at single_thread_set+0x35a sys/kern/kern_sig.c:2155 sigexit(ffff80002129f2a8,6) at sigexit+0x90 sys/kern/kern_sig.c:1562 postsig(ffff80002129f2a8,6,ffff80002c9c7688) at postsig+0x4a9 sys/kern/kern_sig.c:1494 userret(ffff80002129f2a8) at userret+0x16e sys/kern/kern_sig.c:1987 syscall(ffff80002c9c77c0) at syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline] syscall(ffff80002c9c77c0) at syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xf2c76a3ee30, count: -9 ddb{0}> machine ddbcpu 1 Stopped at x86_ipi_db+0x1e: addq $0x8,%rsp ddb{1}> trace x86_ipi_db(ffff800020d58ff0) at x86_ipi_db+0x1e sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 __sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc+0xf sys/dev/kcov.c:154 syscall(ffff800026153b00) at syscall+0x5cd mi_syscall sys/sys/syscall_mi.h:110 [inline] syscall(ffff800026153b00) at syscall+0x5cd sys/arch/amd64/amd64/trap.c:623 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7de5362cef60, count: -6