witness: lock order reversal: 1st 0xffffffff82d0ad08 &sched_lock (&sched_lock) 2nd 0xffff80002121c120 &pr->ps_mtx (&pr->ps_mtx) lock order "&pr->ps_mtx"(mutex) -> "&sched_lock"(sched_lock) first seen at: #0 __mp_lock+0xa2 read_rflags machine/cpufunc.h:195 [inline] #0 __mp_lock+0xa2 intr_disable machine/cpufunc.h:216 [inline] #0 __mp_lock+0xa2 sys/kern/kern_lock.c:142 #1 donice+0x178 sys/kern/kern_resource.c:218 #2 syscall+0x5e2 mi_syscall sys/sys/syscall_mi.h:110 [inline] #2 syscall+0x5e2 sys/arch/amd64/amd64/trap.c:623 #3 Xsyscall+0x128 lock order "&sched_lock"(sched_lock) -> "&pr->ps_mtx"(mutex) first seen at: #0 mtx_enter+0x3e sys/kern/kern_lock.c:265 #1 single_thread_set+0x35a single_thread_wait sys/kern/kern_sig.c:2172 [inline] #1 single_thread_set+0x35a sys/kern/kern_sig.c:2155 #2 sigexit+0x90 sys/kern/kern_sig.c:1562 #3 postsig+0x4a9 sys/kern/kern_sig.c:1494 #4 userret+0x16e sys/kern/kern_sig.c:1987 #5 syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline] #5 syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644 #6 Xsyscall+0x128 Stopped at db_enter+0x1c: addq $0x8,%rsp ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic the kernel did not panic ddb{1}> trace db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 witness_checkorder(ffff80002121c120,9,0) at witness_checkorder+0x10b7 witness_debugger sys/kern/subr_witness.c:2510 [inline] witness_checkorder(ffff80002121c120,9,0) at witness_checkorder+0x10b7 sys/kern/subr_witness.c:1110 mtx_enter(ffff80002121c110) at mtx_enter+0x3e sys/kern/kern_lock.c:265 single_thread_set(ffff8000212cdd58,0,1) at single_thread_set+0x35a single_thread_wait sys/kern/kern_sig.c:2172 [inline] single_thread_set(ffff8000212cdd58,0,1) at single_thread_set+0x35a sys/kern/kern_sig.c:2155 sigexit(ffff8000212cdd58,6) at sigexit+0x90 sys/kern/kern_sig.c:1562 postsig(ffff8000212cdd58,6,ffff80002e442f88) at postsig+0x4a9 sys/kern/kern_sig.c:1494 userret(ffff8000212cdd58) at userret+0x16e sys/kern/kern_sig.c:1987 syscall(ffff80002e4430c0) at syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline] syscall(ffff80002e4430c0) at syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xb47c8a2cab0, count: -9 ddb{1}> show registers rdi 0x3 rsi 0xffffffff82b6a550 __sancov_gen_cov_switch_values.134 rbp 0xffff80002e442c90 rbx 0x3 rdx 0 rcx 0xffff8000212cdd58 rax 0xffff800020d58ff0 r8 0xffff80002e442c00 r9 0x8080808080808080 r10 0x950c0d0b8d3e85e4 r11 0x330eec4c1f73c3b9 r12 0xfffffd8003a0a4e0 r13 0 r14 0xfffffd80039f8790 r15 0xfffffd80031c7bc0 rip 0xffffffff81334fdc db_enter+0x1c cs 0x8 rflags 0x246 rsp 0xffff80002e442c80 ss 0x10 db_enter+0x1c: addq $0x8,%rsp ddb{1}> show proc PROC (syz-executor.3) pid=178166 stat=onproc flags process=500010 proc=4002000 pri=32, usrpri=80, nice=20 forw=0xffffffffffffffff, list=0xffff8000212cc2c8,0xffff8000212cc828 process=0xffff80002121c010 user=0xffff80002e43e000, vmspace=0xfffffd806baad758 estcpu=30, cpticks=0, pctcpu=0.0 user=0, sys=0, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 3671 83507 999 32767 2 0x10 syz-executor.1 3671 235707 999 32767 3 0x4000090 fsleep syz-executor.1 1049 291082 5867 32767 2 0x10 syz-executor.4 1049 159759 5867 32767 3 0x4000090 fsleep syz-executor.4 4045 265277 52554 32767 2 0x10 syz-executor.2 4045 146303 52554 32767 2 0x4000010 syz-executor.2 30749 110892 8895 32767 2 0x10 syz-executor.6 30749 370097 8895 32767 3 0x4000090 fsleep syz-executor.6 30749 397859 8895 32767 3 0x4000090 fsleep syz-executor.6 30749 70165 8895 32767 3 0x4000090 fsleep syz-executor.6 70474 520498 12246 32767 2 0x10 syz-executor.5 70474 172752 12246 32767 3 0x4000090 fsleep syz-executor.5 70474 470319 12246 32767 3 0x4000090 ttyin syz-executor.5 96320 121949 10470 32767 2 0x10 syz-executor.7 96320 380441 10470 32767 3 0x4000090 fsleep syz-executor.7 96320 136456 10470 32767 3 0x4000090 fsleep syz-executor.7 90926 98700 5310 32767 7 0x10 syz-executor.0 90926 482876 5310 32767 2 0x4000090 syz-executor.0 90926 79706 5310 32767 3 0x4000090 fsleep syz-executor.0 28621 64396 66947 32767 4 0x582010 syz-executor.3 *28621 178166 66947 32767 7 0x4502010 syz-executor.3 28621 238549 66947 32767 4 0x4580090 fsleep syz-executor.3 66947 318733 67878 32767 3 0x90 nanoslp syz-executor.3 67878 175667 66605 0 3 0x82 wait syz-executor.3 10470 323488 85664 32767 2 0x10 syz-executor.7 85664 1297 66605 0 3 0x82 wait syz-executor.7 8895 295456 47893 32767 3 0x90 nanoslp syz-executor.6 47893 383764 66605 0 3 0x82 wait syz-executor.6 999 12390 26222 32767 3 0x90 nanoslp syz-executor.1 26222 408379 66605 0 3 0x82 wait syz-executor.1 65218 111620 0 0 3 0x14200 bored sosplice 5867 344050 64782 32767 3 0x90 nanoslp syz-executor.4 64782 96320 66605 0 3 0x82 wait syz-executor.4 12246 303421 25003 32767 3 0x90 nanoslp syz-executor.5 25003 2529 66605 0 3 0x82 wait syz-executor.5 52554 100537 87226 32767 3 0x90 nanoslp syz-executor.2 5310 76776 63863 32767 3 0x90 nanoslp syz-executor.0 87226 295906 66605 0 3 0x82 wait syz-executor.2 63863 23109 66605 0 3 0x82 wait syz-executor.0 66605 49325 66869 0 3 0x2000082 wait syz-fuzzer 66605 290051 66869 0 3 0x6000082 thrsleep syz-fuzzer 66605 315550 66869 0 3 0x6000082 thrsleep syz-fuzzer 66605 479696 66869 0 3 0x6000082 thrsleep syz-fuzzer 66605 442189 66869 0 3 0x6000082 wait syz-fuzzer 66605 347791 66869 0 3 0x6000082 kqread syz-fuzzer 66605 370812 66869 0 3 0x6000082 wait syz-fuzzer 66605 333712 66869 0 3 0x6000082 wait syz-fuzzer 66605 389571 66869 0 3 0x6000082 wait syz-fuzzer 66605 158733 66869 0 3 0x6000082 thrsleep syz-fuzzer 66605 371462 66869 0 3 0x6000082 thrsleep syz-fuzzer 66605 199967 66869 0 3 0x6000082 wait syz-fuzzer 66605 249486 66869 0 3 0x6000082 thrsleep syz-fuzzer 66605 343835 66869 0 3 0x6000082 thrsleep syz-fuzzer 66605 455971 66869 0 3 0x6000082 wait syz-fuzzer 66605 469092 66869 0 3 0x6000082 wait syz-fuzzer 66869 313875 85486 0 3 0x10008a sigsusp ksh 85486 17478 6589 0 3 0x9a kqread sshd 63314 47711 1 0 3 0x100083 ttyin getty 6589 274601 1 0 3 0x88 kqread sshd 318 280648 48712 73 3 0x1100090 kqread syslogd 48712 202011 1 0 3 0x100082 netio syslogd 33120 524034 1 0 3 0x100080 kqread resolvd 17341 235424 48216 77 3 0x100092 kqread dhcpleased 78686 378337 48216 77 3 0x100092 kqread dhcpleased 48216 496359 1 0 3 0x80 kqread dhcpleased 62948 415575 0 0 3 0x14200 bored smr 82547 513459 0 0 2 0x14200 zerothread 44105 369127 0 0 3 0x14200 aiodoned aiodoned 73741 106317 0 0 3 0x14200 syncer update 86982 115671 0 0 3 0x14200 cleaner cleaner 21726 506973 0 0 3 0x14200 reaper reaper 4010 384498 0 0 3 0x14200 pgdaemon pagedaemon 7060 345579 0 0 3 0x14200 bored viomb 86876 8840 0 0 3 0x40014200 acpi0 acpi0 51274 31108 0 0 3 0x40014200 idle1 19903 97428 0 0 3 0x14200 bored softnet3 67368 256727 0 0 3 0x14200 bored softnet2 15834 424656 0 0 3 0x14200 bored softnet1 77779 197353 0 0 3 0x14200 bored softnet0 44107 2733 0 0 3 0x14200 bored systqmp 65165 474219 0 0 3 0x14200 bored systq 91062 485328 0 0 3 0x40014200 bored softclock 79207 11047 0 0 3 0x40014200 idle0 1 435088 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{1}> show all locks CPU 1: exclusive sched_lock &sched_lock r = 0 (0xffffffff82d0ad08) #0 witness_lock+0x447 #1 single_thread_set+0x179 sys/kern/kern_sig.c:2113 #2 sigexit+0x90 sys/kern/kern_sig.c:1562 #3 postsig+0x4a9 sys/kern/kern_sig.c:1494 #4 userret+0x16e sys/kern/kern_sig.c:1987 #5 syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline] #5 syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644 #6 Xsyscall+0x128 Process 28621 (syz-executor.3) thread 0xffff8000212cdd58 (178166) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff82c795f0) #0 witness_lock+0x447 #1 postsig+0x49e sys/kern/kern_sig.c:1500 #2 userret+0x16e sys/kern/kern_sig.c:1987 #3 syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline] #3 syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644 #4 Xsyscall+0x128 exclusive sched_lock &sched_lock r = 0 (0xffffffff82d0ad08) #0 witness_lock+0x447 #1 single_thread_set+0x179 sys/kern/kern_sig.c:2113 #2 sigexit+0x90 sys/kern/kern_sig.c:1562 #3 postsig+0x4a9 sys/kern/kern_sig.c:1494 #4 userret+0x16e sys/kern/kern_sig.c:1987 #5 syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline] #5 syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644 #6 Xsyscall+0x128 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10203 6410K 6420K 78643K 11312 0 pcb 13 12K 14K 78643K 17 0 rtable 244 6K 7K 78643K 958 0 pf 29 8K 8K 78643K 49 0 ifaddr 44 15K 15K 78643K 88 0 ifgroup 50 2K 2K 78643K 90 0 sysctl 2 0K 0K 78643K 2 0 counters 60 35K 35K 78643K 80 0 ioctlops 0 0K 2K 78643K 102 0 iov 0 0K 34K 78643K 526 0 mount 1 1K 1K 78643K 1 0 log 0 0K 0K 78643K 4 0 vnodes 1279 80K 80K 78643K 2701 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 9K 78643K 139 0 VM map 2 1K 1K 78643K 2 0 sem 12 0K 0K 78643K 767 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1697 195K 286K 78643K 12548 0 file desc 26 97K 117K 78643K 6535 0 sigio 0 0K 0K 78643K 175 0 proc 56 78K 115K 78643K 1125 0 subproc 104 6K 6K 78643K 234 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 350 0 in_multi 99 7K 7K 78643K 265 0 ether_multi 1 0K 0K 78643K 8 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 157 705K 705K 78643K 157 0 exec 0 0K 1K 78643K 1330 0 tdb 3 0K 0K 78643K 3 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 8 62K 64K 78643K 10 0 UVM amap 454 96K 105K 78643K 66904 0 UVM aobj 131 4K 4K 78643K 144 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 0K 78643K 154 0 NDP 11 0K 2K 78643K 57 0 temp 75 5916K 5996K 78643K 19229 0 kqueue 12 18K 45K 78643K 2465 0 SYN cache 2 16K 16K 78643K 2 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 22 0 0 1 0 1 1 0 8 0 rtpcb 120 347 0 344 5 4 1 3 0 8 0 rtentry 112 233 0 118 4 0 4 4 0 8 0 unpcb 144 13774 0 13760 65 60 5 11 0 8 4 syncache 304 112 0 112 16 15 1 1 0 8 1 tcpqe 32 911 0 911 14 13 1 1 0 8 1 tcpcb 808 3194 0 3165 83 79 4 17 0 8 1 arp 120 39 0 21 1 0 1 1 0 8 0 ipq 40 17 0 16 5 4 1 1 0 8 0 ipqe 40 94 0 93 5 4 1 1 0 8 0 inpcb 368 5631 0 5599 88 79 9 16 0 8 5 nd6 136 65 0 36 2 1 1 2 0 8 0 kcovpl 48 18 0 10 1 0 1 1 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 945 0 469 31 0 31 31 0 8 0 art_table 32 946 0 469 4 0 4 4 0 8 0 art_node 16 232 0 127 1 0 1 1 0 8 0 sysvmsgpl 40 17 0 8 1 0 1 1 0 8 0 semupl 112 1 0 1 1 1 0 1 0 8 0 semapl 112 762 0 752 1 0 1 1 0 8 0 shmpl 112 141 0 13 4 0 4 4 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 9979 0 8531 91 0 91 91 0 8 0 ffsino 272 9979 0 8531 97 0 97 97 0 8 0 nchpl 144 19011 0 17369 63 0 63 63 0 8 0 uvmvnodes 80 5926 0 0 121 0 121 121 0 8 0 vnodes 216 5926 0 0 330 0 330 330 0 8 0 namei 1024 61063 0 61063 2 1 1 2 0 8 1 percpumem 16 53 0 10 1 0 1 1 0 8 0 kstatmem 264 42 0 20 2 0 2 2 0 8 0 scxspl 216 55056 0 55056 28 25 3 8 1 8 3 plimitpl 152 748 0 725 9 8 1 2 0 8 0 sigapl 424 6810 0 6755 7 0 7 7 0 8 0 futexpl 64 57640 0 57630 1 0 1 1 0 8 0 knotepl 120 439 0 0 12 0 12 12 0 8 0 kqueuepl 216 11333 0 11324 74 66 8 9 0 8 7 pipepl 320 2034 0 2005 45 40 5 11 0 8 2 fdescpl 496 6792 0 6755 7 2 5 6 0 8 0 filepl 152 59501 0 59259 110 92 18 27 0 8 8 lockfpl 104 1107 0 1105 2 1 1 2 0 8 0 lockfspl 48 291 0 289 1 0 1 1 0 8 0 sessionpl 144 33 0 17 1 0 1 1 0 8 0 pgrppl 48 231 0 215 1 0 1 1 0 8 0 ucredpl 104 6876 0 6858 1 0 1 1 0 8 0 zombiepl 144 6755 0 6755 1 0 1 1 0 8 1 processpl 1072 6810 0 6755 4 0 4 4 0 8 0 procpl 680 18801 0 18717 21 13 8 9 0 8 1 sosppl 168 147 0 147 7 6 1 1 0 8 1 sockpl 488 19996 0 19947 362 342 20 38 0 8 12 mcl64k 65536 17 0 0 3 0 3 3 0 8 0 mcl16k 16384 17 0 0 3 0 3 3 0 8 0 mcl12k 12288 17 0 0 2 0 2 2 0 8 0 mcl9k 9216 16 0 0 2 0 2 2 0 8 0 mcl8k 8192 20 0 0 3 0 3 3 0 8 0 mcl4k 4096 67 0 0 7 4 3 6 0 8 0 mcl2k2 2112 6 0 0 1 0 1 1 0 8 0 mcl2k 2048 307 0 0 35 6 29 35 0 8 0 mtagpl 96 4 0 0 1 0 1 1 0 8 0 mbufpl 256 1032 0 0 59 0 59 59 0 8 0 bufpl 288 13762 0 7434 453 0 453 453 0 8 0 anonpl 24 813014 0 801483 160 69 91 96 0 186 0 amapchunkpl 152 210967 0 209973 74 35 39 46 0 158 0 amappl16 200 19438 0 19119 133 115 18 31 0 8 1 amappl15 192 13 0 12 1 0 1 1 0 8 0 amappl14 184 215 0 202 2 1 1 2 0 8 0 amappl13 176 10 0 9 1 0 1 1 0 8 0 amappl12 168 7572 0 7534 2 0 2 2 0 8 0 amappl11 160 55 0 45 1 0 1 1 0 8 0 amappl10 152 39 0 29 1 0 1 1 0 8 0 amappl9 144 240 0 240 7 6 1 1 0 8 1 amappl8 136 417 0 301 5 0 5 5 0 8 0 amappl7 128 93 0 77 2 0 2 2 0 8 0 amappl6 120 380 0 353 2 1 1 2 0 8 0 amappl5 112 377 0 367 1 0 1 1 0 8 0 amappl4 104 720 0 676 4 2 2 4 0 8 0 amappl3 96 41656 0 41543 7 4 3 4 0 8 0 amappl2 88 7360 0 7285 3 1 2 3 0 8 0 amappl1 80 32497 0 31969 23 11 12 22 0 8 0 amappl 88 66066 0 65771 7 0 7 7 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 143 0 13 3 0 3 3 0 8 0 uaddrrnd 24 6792 0 6755 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 6792 0 6755 1 0 1 1 0 8 0 vmmpekpl 168 59636 0 59580 4 0 4 4 0 8 0 vmmpepl 168 412819 0 410249 220 108 112 123 0 357 0 vmsppl 464 6791 0 6755 7 2 5 6 0 8 0 rwobjpl 56 114825 0 107351 123 16 107 109 0 8 0 pdppl 4096 13592 0 13510 294 212 82 92 0 8 0 pvpl 32 2059667 0 2041656 415 257 158 358 0 265 0 pmappl 248 6791 0 6755 4 1 3 3 0 8 0 extentpl 40 56 0 38 1 0 1 1 0 8 0 phpool 112 1277 0 396 26 0 26 26 0 8 0 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+0x1e: addq $0x8,%rsp ddb{0}> trace x86_ipi_db(ffffffff82c29ff0) at x86_ipi_db+0x1e sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 __mp_lock(ffffffff82d0ab00) at __mp_lock+0x122 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff82d0ab00) at __mp_lock+0x122 sys/kern/kern_lock.c:147 preempt() at preempt+0x37 sys/kern/sched_bsd.c:340 ast(ffff800029dbbe90) at ast+0x109 mi_ast sys/sys/syscall_mi.h:192 [inline] ast(ffff800029dbbe90) at ast+0x109 sys/arch/amd64/amd64/trap.c:541 Xsyscall() at Xsyscall+0x156 end of kernel end trace frame: 0x773bd5d9bff0, count: -7 ddb{0}> machine ddbcpu 1 Stopped at db_enter+0x1c: addq $0x8,%rsp ddb{1}> trace db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 witness_checkorder(ffff80002121c120,9,0) at witness_checkorder+0x10b7 witness_debugger sys/kern/subr_witness.c:2510 [inline] witness_checkorder(ffff80002121c120,9,0) at witness_checkorder+0x10b7 sys/kern/subr_witness.c:1110 mtx_enter(ffff80002121c110) at mtx_enter+0x3e sys/kern/kern_lock.c:265 single_thread_set(ffff8000212cdd58,0,1) at single_thread_set+0x35a single_thread_wait sys/kern/kern_sig.c:2172 [inline] single_thread_set(ffff8000212cdd58,0,1) at single_thread_set+0x35a sys/kern/kern_sig.c:2155 sigexit(ffff8000212cdd58,6) at sigexit+0x90 sys/kern/kern_sig.c:1562 postsig(ffff8000212cdd58,6,ffff80002e442f88) at postsig+0x4a9 sys/kern/kern_sig.c:1494 userret(ffff8000212cdd58) at userret+0x16e sys/kern/kern_sig.c:1987 syscall(ffff80002e4430c0) at syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline] syscall(ffff80002e4430c0) at syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xb47c8a2cab0, count: -9