witness: lock order reversal: 1st 0xffffffff82c65f30 &sched_lock (&sched_lock) 2nd 0xffff8000295a3bc8 &pr->ps_mtx (&pr->ps_mtx) lock order "&pr->ps_mtx"(mutex) -> "&sched_lock"(sched_lock) first seen at: #0 __mp_lock+0xa2 read_rflags machine/cpufunc.h:195 [inline] #0 __mp_lock+0xa2 intr_disable machine/cpufunc.h:216 [inline] #0 __mp_lock+0xa2 sys/kern/kern_lock.c:142 #1 donice+0x178 sys/kern/kern_resource.c:218 #2 syscall+0x5e2 mi_syscall sys/sys/syscall_mi.h:110 [inline] #2 syscall+0x5e2 sys/arch/amd64/amd64/trap.c:623 #3 Xsyscall+0x128 lock order "&sched_lock"(sched_lock) -> "&pr->ps_mtx"(mutex) first seen at: #0 mtx_enter+0x3e sys/kern/kern_lock.c:265 #1 single_thread_set+0x35a single_thread_wait sys/kern/kern_sig.c:2172 [inline] #1 single_thread_set+0x35a sys/kern/kern_sig.c:2155 #2 sigexit+0x90 sys/kern/kern_sig.c:1562 #3 postsig+0x4a9 sys/kern/kern_sig.c:1494 #4 userret+0x16e sys/kern/kern_sig.c:1987 #5 syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline] #5 syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644 #6 Xsyscall+0x128 Stopped at db_enter+0x1c: addq $0x8,%rsp ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic the kernel did not panic ddb{1}> trace db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 witness_checkorder(ffff8000295a3bc8,9,0) at witness_checkorder+0x10b7 witness_debugger sys/kern/subr_witness.c:2510 [inline] witness_checkorder(ffff8000295a3bc8,9,0) at witness_checkorder+0x10b7 sys/kern/subr_witness.c:1110 mtx_enter(ffff8000295a3bb8) at mtx_enter+0x3e sys/kern/kern_lock.c:265 single_thread_set(ffff800021200018,0,1) at single_thread_set+0x35a single_thread_wait sys/kern/kern_sig.c:2172 [inline] single_thread_set(ffff800021200018,0,1) at single_thread_set+0x35a sys/kern/kern_sig.c:2155 sigexit(ffff800021200018,6) at sigexit+0x90 sys/kern/kern_sig.c:1562 postsig(ffff800021200018,6,ffff80002afb8ea8) at postsig+0x4a9 sys/kern/kern_sig.c:1494 userret(ffff800021200018) at userret+0x16e sys/kern/kern_sig.c:1987 syscall(ffff80002afb8fe0) at syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline] syscall(ffff80002afb8fe0) at syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xf9a996ffbe0, count: -9 ddb{1}> show registers rdi 0x3 rsi 0xffffffff82befdd0 __sancov_gen_cov_switch_values.134 rbp 0xffff80002afb8bb0 rbx 0x3 rdx 0 rcx 0xffff800021200018 rax 0xffff800020d58ff0 r8 0xffff80002afb8b20 r9 0x8080808080808080 r10 0xfb4f0ee40a8b546c r11 0xd7c7a1c8d2e7e830 r12 0xfffffd8003a0d900 r13 0 r14 0xfffffd80039f91c0 r15 0xfffffd80031cabc0 rip 0xffffffff8165630c db_enter+0x1c cs 0x8 rflags 0x246 rsp 0xffff80002afb8ba0 ss 0x10 db_enter+0x1c: addq $0x8,%rsp ddb{1}> show proc PROC (syz-executor.5) pid=245359 stat=onproc flags process=500010 proc=4002000 pri=32, usrpri=79, nice=20 forw=0xffffffffffffffff, list=0xffff800021200810,0xffff800021201568 process=0xffff8000295a3ab8 user=0xffff80002afb4000, vmspace=0xfffffd80731e9018 estcpu=36, cpticks=0, pctcpu=0.0 user=0, sys=0, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 33163 379439 27581 32767 3 0x90 nanoslp syz-executor.6 33163 324510 27581 32767 2 0x4000010 syz-executor.6 33163 89449 27581 32767 3 0x4000090 fsleep syz-executor.6 2068 82275 20462 32767 4 0x582010 syz-executor.5 * 2068 245359 20462 32767 7 0x4502010 syz-executor.5 2068 493031 20462 32767 4 0x4580090 fsleep syz-executor.5 5735 24279 10981 32767 2 0x10 syz-executor.1 5735 445279 10981 32767 2 0x4000010 syz-executor.1 31001 40325 91043 32767 3 0x90 nanoslp syz-executor.3 31001 485989 91043 32767 3 0x4000090 kqread syz-executor.3 31001 7966 91043 32767 3 0x4000090 fsleep syz-executor.3 31001 495863 91043 32767 3 0x4000090 fsleep syz-executor.3 93259 135490 51320 32767 3 0x90 nanoslp syz-executor.4 93259 191633 51320 32767 3 0x4000090 fifor syz-executor.4 18310 97796 59343 32767 3 0x90 nanoslp syz-executor.7 18310 342560 59343 32767 3 0x4000090 kqread syz-executor.7 18310 23245 59343 32767 3 0x4000090 kqread syz-executor.7 18310 416538 59343 32767 3 0x4000090 fsleep syz-executor.7 51320 371205 53027 32767 3 0x90 nanoslp syz-executor.4 53027 480913 5837 0 3 0x82 wait syz-executor.4 10981 215280 52474 32767 3 0x90 nanoslp syz-executor.1 52474 108004 5837 0 3 0x82 wait syz-executor.1 15110 244109 21171 32767 3 0x90 nanoslp syz-executor.0 21171 129671 5837 0 3 0x82 wait syz-executor.0 45521 348303 51380 32767 3 0x90 piperd syz-executor.2 51380 54590 5837 0 3 0x82 wait syz-executor.2 27581 238814 28419 32767 3 0x90 nanoslp syz-executor.6 28419 109099 5837 0 3 0x82 wait syz-executor.6 50163 62190 0 0 3 0x14200 bored sosplice 59343 338624 63842 32767 3 0x90 nanoslp syz-executor.7 63842 352541 5837 0 3 0x82 wait syz-executor.7 20462 236389 81451 32767 3 0x90 nanoslp syz-executor.5 91043 302167 97659 32767 3 0x90 nanoslp syz-executor.3 81451 348139 5837 0 3 0x82 wait syz-executor.5 97659 337463 5837 0 3 0x82 wait syz-executor.3 5837 252581 38806 0 3 0x2000082 wait syz-fuzzer 5837 386415 38806 0 3 0x6000082 thrsleep syz-fuzzer 5837 451018 38806 0 3 0x6000082 thrsleep syz-fuzzer 5837 341882 38806 0 3 0x6000082 thrsleep syz-fuzzer 5837 369163 38806 0 3 0x6000082 wait syz-fuzzer 5837 411489 38806 0 3 0x6000082 wait syz-fuzzer 5837 273472 38806 0 3 0x6000082 thrsleep syz-fuzzer 5837 57652 38806 0 3 0x6000082 thrsleep syz-fuzzer 5837 121009 38806 0 3 0x6000082 wait syz-fuzzer 5837 166298 38806 0 3 0x6000082 thrsleep syz-fuzzer 5837 292332 38806 0 3 0x6000082 wait syz-fuzzer 5837 126755 38806 0 3 0x6000082 wait syz-fuzzer 5837 290651 38806 0 3 0x6000082 wait syz-fuzzer 5837 380502 38806 0 3 0x6000082 thrsleep syz-fuzzer 5837 116403 38806 0 3 0x6000082 wait syz-fuzzer 5837 511872 38806 0 3 0x6000082 kqread syz-fuzzer 38806 498400 6219 0 3 0x10008a sigsusp ksh 6219 119921 88272 0 3 0x9a kqread sshd 3326 97179 1 0 3 0x100083 ttyin getty 88272 353521 1 0 3 0x88 kqread sshd 64421 424368 85756 73 3 0x1100090 kqread syslogd 85756 460246 1 0 3 0x100082 netio syslogd 17882 449194 1 0 3 0x100080 kqread resolvd 71421 496293 46569 77 3 0x100092 kqread dhcpleased 93071 281197 46569 77 3 0x100092 kqread dhcpleased 46569 70798 1 0 3 0x80 kqread dhcpleased 21869 361840 0 0 3 0x14200 bored smr 69024 128787 0 0 7 0x14200 zerothread 56472 87552 0 0 3 0x14200 aiodoned aiodoned 41441 66532 0 0 3 0x14200 syncer update 18240 120418 0 0 3 0x14200 cleaner cleaner 85131 505348 0 0 3 0x14200 reaper reaper 82402 301763 0 0 3 0x14200 pgdaemon pagedaemon 53204 153745 0 0 3 0x14200 bored viomb 26882 161685 0 0 3 0x40014200 acpi0 acpi0 60762 145868 0 0 3 0x40014200 idle1 29745 276764 0 0 3 0x14200 bored softnet3 55943 279474 0 0 3 0x14200 bored softnet2 7059 491482 0 0 3 0x14200 bored softnet1 61724 229598 0 0 3 0x14200 bored softnet0 39500 274188 0 0 3 0x14200 bored systqmp 38173 371415 0 0 3 0x14200 bored systq 74387 458535 0 0 3 0x40014200 bored softclock 50310 214677 0 0 3 0x40014200 idle0 1 407619 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{1}> show all locks CPU 1: exclusive sched_lock &sched_lock r = 0 (0xffffffff82c65f30) #0 witness_lock+0x447 #1 single_thread_set+0x179 sys/kern/kern_sig.c:2113 #2 sigexit+0x90 sys/kern/kern_sig.c:1562 #3 postsig+0x4a9 sys/kern/kern_sig.c:1494 #4 userret+0x16e sys/kern/kern_sig.c:1987 #5 syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline] #5 syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644 #6 Xsyscall+0x128 Process 2068 (syz-executor.5) thread 0xffff800021200018 (245359) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff82d03430) #0 witness_lock+0x447 #1 postsig+0x49e sys/kern/kern_sig.c:1500 #2 userret+0x16e sys/kern/kern_sig.c:1987 #3 syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline] #3 syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644 #4 Xsyscall+0x128 exclusive sched_lock &sched_lock r = 0 (0xffffffff82c65f30) #0 witness_lock+0x447 #1 single_thread_set+0x179 sys/kern/kern_sig.c:2113 #2 sigexit+0x90 sys/kern/kern_sig.c:1562 #3 postsig+0x4a9 sys/kern/kern_sig.c:1494 #4 userret+0x16e sys/kern/kern_sig.c:1987 #5 syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline] #5 syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644 #6 Xsyscall+0x128 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10188 6408K 6420K 78643K 11311 0 pcb 13 16K 20K 78643K 19 0 rtable 240 6K 7K 78643K 1213 0 pf 29 8K 8K 78643K 59 0 ifaddr 44 15K 15K 78643K 108 0 ifgroup 50 2K 2K 78643K 110 0 sysctl 3 1K 1K 78643K 7 0 counters 60 35K 35K 78643K 90 0 ioctlops 0 0K 2K 78643K 53 0 iov 0 0K 20K 78643K 1053 0 mount 1 1K 1K 78643K 1 0 log 0 0K 0K 78643K 4 0 vnodes 1280 80K 80K 78643K 3369 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 9K 78643K 231 0 VM map 2 1K 1K 78643K 2 0 sem 12 1K 1K 78643K 142 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1697 195K 286K 78643K 12548 0 file desc 24 89K 121K 78643K 12984 0 sigio 0 0K 0K 78643K 147 0 proc 56 78K 127K 78643K 1660 0 subproc 104 6K 6K 78643K 299 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 302 0 in_multi 99 7K 7K 78643K 342 0 ether_multi 1 0K 0K 78643K 8 0 mrt 1 0K 0K 78643K 1 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 67 307K 307K 78643K 67 0 exec 0 0K 1K 78643K 1703 0 tdb 3 0K 0K 78643K 3 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 8 62K 64K 78643K 10 0 UVM amap 451 92K 101K 78643K 132925 0 UVM aobj 131 4K 4K 78643K 131 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 0K 78643K 256 0 NDP 11 0K 2K 78643K 72 0 temp 74 5920K 6000K 78643K 35300 0 kqueue 23 32K 35K 78643K 1188 0 SYN cache 2 16K 16K 78643K 2 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 22 0 0 1 0 1 1 0 8 0 rtpcb 120 726 0 723 11 10 1 4 0 8 0 rtentry 112 310 0 197 4 0 4 4 0 8 0 unpcb 144 10206 0 10177 99 97 2 12 0 8 0 syncache 304 120 0 120 25 24 1 1 0 8 1 sackhl 24 2 0 2 2 2 0 1 0 8 0 tcpqe 32 456 0 456 20 19 1 1 0 8 1 tcpcb 808 3184 0 3179 102 99 3 15 0 8 2 arp 120 51 0 33 1 0 1 1 0 8 0 ipq 40 29 0 28 11 10 1 1 0 8 0 ipqe 40 168 0 167 11 10 1 1 0 8 0 inpcb 368 15857 0 15849 278 273 5 24 0 8 3 nd6 136 100 0 73 1 0 1 1 0 8 0 kcovpl 48 23 0 15 1 0 1 1 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 1319 0 847 38 8 30 30 0 8 0 art_table 32 1320 0 847 4 0 4 4 0 8 0 art_node 16 309 0 206 1 0 1 1 0 8 0 semupl 112 5 0 5 1 1 0 1 0 8 0 semapl 112 140 0 130 1 0 1 1 0 8 0 shmpl 112 128 0 0 4 0 4 4 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 19089 0 17644 91 0 91 91 0 8 0 ffsino 272 19089 0 17644 97 0 97 97 0 8 0 nchpl 144 37048 0 35402 63 0 63 63 0 8 0 uvmvnodes 80 5926 0 0 121 0 121 121 0 8 0 vnodes 216 5926 0 0 330 0 330 330 0 8 0 namei 1024 119031 0 119031 7 6 1 2 0 8 1 percpumem 16 58 0 15 1 0 1 1 0 8 0 kstatmem 264 52 0 30 2 0 2 2 0 8 0 scxspl 216 113594 0 113594 34 33 1 8 1 8 1 plimitpl 152 1313 0 1289 15 14 1 2 0 8 0 sigapl 424 13238 0 13184 7 0 7 7 0 8 0 futexpl 64 109366 0 109361 5 4 1 1 0 8 0 knotepl 120 793 0 0 18 2 16 17 0 8 1 kqueuepl 216 9295 0 9278 105 100 5 8 0 8 4 pipepl 320 2470 0 2441 87 84 3 8 0 8 0 fdescpl 496 13220 0 13185 7 2 5 6 0 8 0 filepl 152 85359 0 85096 172 158 14 24 0 8 2 lockfpl 104 1308 0 1306 1 0 1 1 0 8 0 lockfspl 48 391 0 389 1 0 1 1 0 8 0 sessionpl 144 38 0 22 1 0 1 1 0 8 0 pgrppl 48 1325 0 1309 1 0 1 1 0 8 0 ucredpl 104 10308 0 10290 1 0 1 1 0 8 0 zombiepl 144 13185 0 13184 2 1 1 1 0 8 0 processpl 1072 13238 0 13184 5 1 4 5 0 8 0 procpl 680 38448 0 38367 28 20 8 9 0 8 1 sosppl 168 594 0 594 17 16 1 1 0 8 1 sockpl 488 27238 0 27198 594 585 9 45 0 8 3 mcl64k 65536 25 0 0 3 0 3 3 0 8 0 mcl16k 16384 40 0 0 4 2 2 3 0 8 0 mcl12k 12288 17 0 0 2 0 2 2 0 8 0 mcl9k 9216 17 0 0 2 0 2 2 0 8 0 mcl8k 8192 25 0 0 3 0 3 3 0 8 0 mcl4k 4096 43 0 0 4 0 4 4 0 8 0 mcl2k2 2112 24 0 0 2 0 2 2 0 8 0 mcl2k 2048 387 0 0 32 14 18 32 0 8 0 mtagpl 96 7 0 0 1 0 1 1 0 8 0 mbufpl 256 1498 0 0 59 0 59 59 0 8 0 bufpl 288 22468 0 16142 453 0 453 453 0 8 0 anonpl 24 1385064 0 1373283 217 125 92 100 0 186 0 amapchunkpl 152 417851 0 416875 124 84 40 50 0 158 1 amappl16 200 29446 0 29147 155 138 17 29 0 8 0 amappl15 192 27 0 25 1 0 1 1 0 8 0 amappl14 184 197 0 184 2 1 1 2 0 8 0 amappl13 176 28 0 27 1 0 1 1 0 8 0 amappl12 168 14060 0 14022 3 0 3 3 0 8 0 amappl11 160 54 0 44 1 0 1 1 0 8 0 amappl10 152 45 0 35 1 0 1 1 0 8 0 amappl9 144 303 0 302 1 0 1 1 0 8 0 amappl8 136 750 0 601 6 0 6 6 0 8 0 amappl7 128 133 0 120 2 0 2 2 0 8 0 amappl6 120 423 0 405 2 1 1 2 0 8 0 amappl5 112 468 0 459 1 0 1 1 0 8 0 amappl4 104 819 0 776 3 1 2 3 0 8 0 amappl3 96 81994 0 81887 12 9 3 4 0 8 0 amappl2 88 14041 0 13963 3 1 2 3 0 8 0 amappl1 80 55242 0 54709 22 10 12 22 0 8 0 amappl 88 131836 0 131549 8 0 8 8 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 130 0 0 3 0 3 3 0 8 0 uaddrrnd 24 13220 0 13185 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 13220 0 13185 1 0 1 1 0 8 0 vmmpekpl 168 108599 0 108533 5 1 4 4 0 8 0 vmmpepl 168 786194 0 783696 264 135 129 129 0 357 14 vmsppl 464 13219 0 13185 7 2 5 6 0 8 0 rwobjpl 56 203691 0 196198 116 7 109 109 0 8 0 pdppl 4096 26448 0 26370 416 334 82 96 0 8 4 pvpl 32 3889159 0 3870736 601 437 164 361 0 265 0 pmappl 248 13219 0 13185 4 1 3 3 0 8 0 extentpl 40 56 0 38 1 0 1 1 0 8 0 phpool 112 1622 0 758 26 0 26 26 0 8 0 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+0x1e: addq $0x8,%rsp ddb{0}> trace x86_ipi_db(ffffffff82bc3ff0) at x86_ipi_db+0x1e sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 __mp_lock(ffffffff82c65d28) at __mp_lock+0x122 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff82c65d28) at __mp_lock+0x122 sys/kern/kern_lock.c:147 yield() at yield+0x37 sys/kern/sched_bsd.c:321 uvm_pagezero_thread(ffff8000211b2aa8) at uvm_pagezero_thread+0x6d sys/uvm/uvm_pmemrange.c:2170 end trace frame: 0x0, count: -6 ddb{0}> machine ddbcpu 1 Stopped at db_enter+0x1c: addq $0x8,%rsp ddb{1}> trace db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 witness_checkorder(ffff8000295a3bc8,9,0) at witness_checkorder+0x10b7 witness_debugger sys/kern/subr_witness.c:2510 [inline] witness_checkorder(ffff8000295a3bc8,9,0) at witness_checkorder+0x10b7 sys/kern/subr_witness.c:1110 mtx_enter(ffff8000295a3bb8) at mtx_enter+0x3e sys/kern/kern_lock.c:265 single_thread_set(ffff800021200018,0,1) at single_thread_set+0x35a single_thread_wait sys/kern/kern_sig.c:2172 [inline] single_thread_set(ffff800021200018,0,1) at single_thread_set+0x35a sys/kern/kern_sig.c:2155 sigexit(ffff800021200018,6) at sigexit+0x90 sys/kern/kern_sig.c:1562 postsig(ffff800021200018,6,ffff80002afb8ea8) at postsig+0x4a9 sys/kern/kern_sig.c:1494 userret(ffff800021200018) at userret+0x16e sys/kern/kern_sig.c:1987 syscall(ffff80002afb8fe0) at syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline] syscall(ffff80002afb8fe0) at syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xf9a996ffbe0, count: -9