witness: lock order reversal: 1st 0xffffffff839c4440 &sched_lock (&sched_lock) 2nd 0xffff8000299dee30 &cq->cq_mtx (&cq->cq_mtx) lock order [1] &sched_lock (&sched_lock) -> [2] &cq->cq_mtx (&cq->cq_mtx) #0 mtx_enter+0x95 sys/kern/kern_lock.c:406 #1 clockintr_cancel+0x2f sys/kern/kern_clockintr.c:323 #2 mi_switch+0x187 sys/kern/sched_bsd.c:376 #3 ast+0x15a mi_ast sys/sys/syscall_mi.h:252 [inline] #3 ast+0x15a sys/arch/amd64/amd64/trap.c:739 #4 intr_user_exit+0x3c lock order [2] &cq->cq_mtx (&cq->cq_mtx) -> [1] &sched_lock (&sched_lock) #0 mtx_enter+0x95 sys/kern/kern_lock.c:406 #1 sleep_setup+0xed sys/kern/kern_synch.c:304 #2 msleep_nsec+0x10b sys/kern/kern_synch.c:217 #3 dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:589 #4 dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline] #4 dtclose+0x109 sys/dev/dt/dt_dev.c:239 #5 spec_close+0x466 sys/kern/spec_vnops.c:-1 #6 VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156 #7 vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline] #7 vn_closefile+0x12b sys/kern/vfs_vnops.c:615 #8 fdrop+0x121 sys/kern/kern_descrip.c:1280 #9 closef+0x192 sys/kern/kern_descrip.c:1264 #10 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #10 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:775 #11 Xsyscall+0x128 Stopped at db_enter+0x25: addq $0x8,%rsp ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic the kernel did not panic ddb{1}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438 witness_checkorder(ffff8000299dee30,9,0) at witness_checkorder+0x10d1 sys/kern/subr_witness.c:-1 mtx_enter(ffff8000299dee20) at mtx_enter+0x95 sys/kern/kern_lock.c:406 clockintr_cancel(ffff8000299de2f8) at clockintr_cancel+0x2f sys/kern/kern_clockintr.c:323 mi_switch() at mi_switch+0x187 sys/kern/sched_bsd.c:376 ast(ffff80003c41f540) at ast+0x15a mi_ast sys/sys/syscall_mi.h:252 [inline] ast(ffff80003c41f540) at ast+0x15a sys/arch/amd64/amd64/trap.c:739 intr_user_exit() at intr_user_exit+0x3c end of kernel end trace frame: 0x78ea0fe55b50, count: -7 ddb{1}> show registers rdi 0 rsi 0 rbp 0xffff80003c41f340 rbx 0 rdx 0 rcx 0xffff80003c400fb8 rax 0xffff8000299ddff0 r8 0xffff80003c41f220 r9 0x8080808080808080 r10 0x154809d71db2a30 r11 0x26a396e4b6c96989 r12 0xfffffd8003f54300 r13 0xfffffd8004744d58 r14 0x3 r15 0xffffffff8341be15 substchar+0x52f5 rip 0xffffffff814c93d5 db_enter+0x25 cs 0x8 rflags 0x246 rsp 0xffff80003c41f330 ss 0 db_enter+0x25: addq $0x8,%rsp ddb{1}> show proc PROC (syz-executor) tid=38742 pid=62541 tcnt=2 stat=run flags process=4000000 proc=0 runpri=50, usrpri=50, slppri=16, nice=20 wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0 forw=0x0, list=0xffff80003c401cb0,0xffff80003c400038 process=0xffff8000fffece88 user=0xffff80003c41a000, vmspace=0xfffffd800f7e9020 estcpu=36, cpticks=0, pctcpu=0.0, user=0, sys=0, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND *62541 38742 69154 0 2 0x4000000 syz-executor 62541 93217 69154 0 2 0x4000000 syz-executor 80550 134181 9697 0 2 0 syz-executor 80550 467973 9697 0 3 0x4000080 fsleep syz-executor 2731 450202 50768 0 2 0 syz-executor 22900 173589 45105 0 2 0 syz-executor 22900 307962 45105 0 2 0x4000000 syz-executor 63036 473707 24780 0 2 0 syz-executor 63036 219327 24780 0 3 0x4000080 kqpoll syz-executor 63036 55908 24780 0 2 0x4000000 syz-executor 63036 194534 24780 0 3 0x4000080 fsleep syz-executor 37298 458766 76659 0 3 0x80 nanoslp syz-executor 37298 69426 76659 0 3 0x4000080 fsleep syz-executor 37298 426373 76659 0 3 0x4000080 kqread syz-executor 37298 483562 76659 0 3 0x4000080 fsleep syz-executor 45105 235011 80617 0 3 0x82 nanoslp syz-executor 57086 319623 80617 0 3 0x82 piperd syz-executor 99943 347083 80617 0 3 0x82 nanoslp syz-executor 24780 478058 80617 0 3 0x82 nanoslp syz-executor 76659 190410 80617 0 3 0x82 nanoslp syz-executor 69154 226192 80617 0 3 0x82 nanoslp syz-executor 9697 77143 80617 0 3 0x82 nanoslp syz-executor 50768 454753 80617 0 3 0x82 nanoslp syz-executor 80617 194125 94339 0 2 0x2 syz-executor 94339 46454 8749 0 3 0x10008a sigsusp ksh 8749 447920 50452 0 3 0x98 kqread sshd-session 50452 504390 16356 0 3 0x92 kqread sshd-session 70185 351813 1 0 3 0x100083 ttyin getty 16356 261548 1 0 3 0x88 kqread sshd 920 6359 33379 74 3 0x1100092 bpf pflogd 33379 517626 1 0 3 0x80 sbwait pflogd 82517 483481 26581 73 3 0x1100090 kqread syslogd 26581 358606 1 0 3 0x100082 sbwait syslogd 495 244452 1 0 3 0x100080 kqread resolvd 33689 143493 46945 77 3 0x100092 kqread dhcpleased 78149 155215 46945 77 3 0x100092 kqread dhcpleased 46945 484222 1 0 3 0x80 kqread dhcpleased 25603 313171 0 0 3 0x14200 bored smr 32866 436073 0 0 2 0x14200 zerothread 45479 353379 0 0 3 0x14200 aiodoned aiodoned 57171 447812 0 0 3 0x14200 syncer update 9905 191418 0 0 3 0x14200 cleaner cleaner 33841 277466 0 0 3 0x14200 reaper reaper 13023 97333 0 0 3 0x14200 pgdaemon pagedaemon 7941 124929 0 0 3 0x14200 bored viomb 30109 520669 0 0 3 0x40014200 acpi0 acpi0 81581 395066 0 0 3 0x40014200 idle1 91595 399934 0 0 3 0x14200 bored softnet1 23888 368207 0 0 3 0x14200 bored softnet0 30647 492126 0 0 3 0x14200 bored systqmp 84644 363129 0 0 3 0x14200 bored systq 20163 176626 0 0 3 0x14200 tmoslp softclockmp 19185 359036 0 0 3 0x40014200 tmoslp softclock 5474 515203 0 0 3 0x40014200 idle0 1 299072 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{1}> show all locks CPU 0: exclusive mutex &uvm.fpageqlock r = 0 (0xffffffff838eeea8) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 mtx_enter+0x4b4 sys/kern/kern_lock.c:487 #2 uvm_pmr_freepages+0x1a8 sys/uvm/uvm_pmemrange.c:-1 #3 pmap_do_remove+0x8af sys/arch/amd64/amd64/pmap.c:1939 #4 uvm_unmap_kill_entry_withlock+0x269 sys/uvm/uvm_map.c:1863 #5 uvm_map_teardown+0x117 uvm_map_addr_RBT_LEFT sys/uvm/uvm_map.h:-1 [inline] #5 uvm_map_teardown+0x117 sys/uvm/uvm_map.c:2491 #6 exit1+0x6fc sys/kern/kern_exit.c:260 #7 sys_exit+0x1a sys/kern/kern_exit.c:-1 #8 syscall+0xb17 mi_syscall sys/sys/syscall_mi.h:176 [inline] #8 syscall+0xb17 sys/arch/amd64/amd64/trap.c:775 #9 Xsyscall+0x128 CPU 1: exclusive mutex &sched_lock r = 0 (0xffffffff839c4440) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 mtx_enter+0x4b4 sys/kern/kern_lock.c:487 #2 preempt+0x35 sys/kern/sched_bsd.c:338 #3 ast+0x15a mi_ast sys/sys/syscall_mi.h:252 [inline] #3 ast+0x15a sys/arch/amd64/amd64/trap.c:739 #4 intr_user_exit+0x3c Process 22900 (syz-executor) thread 0xffff80002a261ca0 (307962) exclusive rrwlock inode r = 0 (0xfffffd806c761458) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 rrw_enter+0xc6 sys/kern/kern_rwlock.c:621 #3 VOP_LOCK+0xbd sys/kern/vfs_vops.c:527 #4 vn_lock+0xa4 sys/kern/vfs_vnops.c:570 #5 vn_write+0x18f sys/kern/vfs_vnops.c:405 #6 dofilewritev+0x242 sys/kern/sys_generic.c:380 #7 sys_write+0xa2 sys/kern/sys_generic.c:300 #8 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #8 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:775 #9 Xsyscall+0x128 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 11058 12090K 12462K 166960K 12538 0 pcb 17 13K 14K 166960K 115 0 rtable 235 8K 8K 166960K 402 0 pf 35 17K 22K 166960K 73 0 ifaddr 40 7K 7K 166960K 58 0 ifgroup 54 2K 2K 166960K 83 0 sysctl 1 1K 9K 166960K 5 0 counters 68 36K 38K 166960K 98 0 ioctlops 0 0K 4K 166960K 1509 0 iov 0 0K 16K 166960K 80 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1398 88K 89K 166960K 1653 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 1K 5K 166960K 6 0 VM map 2 1K 1K 166960K 2 0 sem 12 0K 0K 166960K 20 0 dirhash 12 2K 2K 166960K 21 0 ACPI 1692 195K 286K 166960K 12470 0 file desc 16 57K 228K 166960K 442 0 sigio 0 0K 0K 166960K 8 0 proc 73 115K 180K 166960K 565 0 subproc 72 4K 4K 166960K 72 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 32 0 in_multi 92 6K 7K 166960K 115 0 ether_multi 1 0K 0K 166960K 9 0 mrt 0 0K 0K 166960K 6 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 73 334K 334K 166960K 73 0 exec 0 0K 1K 166960K 416 0 fusefs mount 1 32K 32K 166960K 1 0 tdb 3 0K 0K 166960K 3 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 234 167K 182K 166960K 5851 0 UVM aobj 11 4K 4K 166960K 12 0 pinsyscall 41 82K 106K 166960K 1553 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 0K 166960K 29 0 NDP 11 0K 2K 166960K 39 0 temp 49 8677K 8742K 166960K 14692 0 kqueue 14 22K 34K 166960K 97 0 SYN cache 2 16K 16K 166960K 2 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 26 0 0 1 0 1 1 0 8 0 rtpcb 120 50 0 46 1 0 1 1 0 8 0 rtentry 176 120 0 16 6 0 6 6 0 8 0 unpcb 144 211 0 192 4 0 4 4 0 8 3 syncache 336 3 0 3 1 1 0 1 0 8 0 tcpcb 736 83 0 78 1 0 1 1 0 8 0 arp 136 18 0 2 1 0 1 1 0 8 0 inpcb 328 324 0 316 4 0 4 4 0 8 2 nd6 152 28 0 5 2 0 2 2 0 8 0 kcovpl 48 8 0 0 1 0 1 1 0 8 0 ppxss 1192 12 0 12 1 0 1 1 0 8 1 pffrag 232 2 0 1 1 0 1 1 0 482 0 pffrnode 88 2 0 1 1 0 1 1 0 8 0 pffrent 40 4 0 3 1 0 1 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfrktable 1344 1 0 1 1 0 1 1 0 8 1 pfanchor 1288 1 0 0 1 0 1 1 0 8 0 pftag 88 4 0 4 1 0 1 1 0 8 1 pfstkey 128 2 0 2 1 1 0 1 0 8 0 pfstate 448 1 0 1 1 1 0 1 0 8 0 pfrule 1360 4 0 3 1 0 1 1 0 8 0 rttmr 136 2 0 2 1 0 1 1 0 8 1 art_heap8 4096 2 0 0 2 0 2 2 0 8 0 art_heap4 256 479 0 61 30 0 30 30 0 8 0 art_table 40 481 0 61 5 0 5 5 0 8 0 art_node 32 120 0 25 1 0 1 1 0 8 0 sysvmsgpl 40 2 0 1 1 0 1 1 0 8 0 semupl 112 1 0 1 1 0 1 1 0 8 1 semapl 112 16 0 6 1 0 1 1 0 8 0 shmpl 112 8 0 1 1 0 1 1 0 8 0 dirhash 1024 23 0 6 3 0 3 3 0 8 0 dino2pl 256 2146 0 641 95 0 95 95 0 8 0 ffsino 296 2146 0 641 117 0 117 117 0 8 0 nchpl 144 2766 0 1066 64 0 64 64 0 8 0 rtmask 32 2 0 2 1 0 1 1 0 8 1 vnodes 216 2409 0 0 134 0 134 134 0 8 0 namei 1024 9235 0 9235 3 1 2 2 0 8 2 percpumem 16 64 0 15 1 0 1 1 0 8 0 vcpupl 3968 1 0 0 1 0 1 1 0 8 0 vmpool 848 1 0 0 1 0 1 1 0 8 0 kstatmem 264 46 0 22 3 0 3 3 0 8 1 scxspl 216 15753 0 15753 6 4 2 5 1 8 2 plimitpl 152 152 0 133 1 0 1 1 0 8 0 sigapl 424 737 0 690 7 1 6 7 0 8 0 knotepl 120 538 0 0 17 0 17 17 0 8 0 kqueuepl 224 195 0 184 5 0 5 5 0 8 4 pipepl 344 171 0 144 6 1 5 6 0 8 2 fdescpl 528 721 0 691 3 0 3 3 0 8 0 filepl 160 3569 0 3317 18 1 17 17 0 8 5 lockfpl 104 236 0 233 2 1 1 2 0 8 0 lockfspl 48 109 0 106 1 0 1 1 0 8 0 sessionpl 144 22 0 13 1 0 1 1 0 8 0 pgrppl 48 46 0 29 1 0 1 1 0 8 0 ucredpl 104 366 0 353 1 0 1 1 0 8 0 zombiepl 144 691 0 690 1 0 1 1 0 8 0 processpl 1232 737 0 690 5 1 4 5 0 8 0 procpl 664 1263 0 1207 7 0 7 7 0 8 2 sosppl 176 1 0 1 1 0 1 1 0 8 1 sockpl 752 659 0 628 12 1 11 11 0 8 7 mcl64k 65536 4 0 0 1 0 1 1 0 8 0 mcl8k 8192 2 0 0 1 0 1 1 0 8 0 mcl4k 4096 123 0 0 16 0 16 16 0 8 0 mcl2k 2048 24 0 0 3 0 3 3 0 8 0 mtagpl 96 5 0 0 1 0 1 1 0 8 0 mbufpl 256 239 0 0 15 0 15 15 0 8 0 bufpl 280 6384 0 248 439 0 439 439 0 8 0 anonpl 32 10963 0 0 89 0 89 89 0 246 0 amapchunkpl 152 18252 0 17791 34 2 32 32 0 158 10 amappl16 200 2283 0 2233 29 6 23 23 0 8 16 amappl15 192 5 0 5 1 1 0 1 0 8 0 amappl14 184 6 0 6 1 1 0 1 0 8 0 amappl13 176 421 0 420 1 0 1 1 0 8 0 amappl12 168 1094 0 1053 3 1 2 3 0 8 0 amappl11 160 7 0 7 1 1 0 1 0 8 0 amappl10 152 49 0 35 1 0 1 1 0 8 0 amappl9 144 261 0 260 2 1 1 1 0 8 0 amappl8 136 24 0 21 1 0 1 1 0 8 0 amappl7 128 109 0 108 1 0 1 1 0 8 0 amappl6 120 268 0 255 1 0 1 1 0 8 0 amappl5 112 68 0 58 1 0 1 1 0 8 0 amappl4 104 412 0 383 1 0 1 1 0 8 0 amappl3 96 3017 0 2929 4 1 3 3 0 8 0 amappl2 88 829 0 754 2 0 2 2 0 8 0 amappl1 80 10296 0 9708 14 1 13 14 0 8 0 amappl 88 5074 0 4918 5 0 5 5 0 92 0 uvmvnodes 80 111 0 0 3 0 3 3 0 8 0 dma65536 65536 1 0 1 1 0 1 1 0 8 1 dma4096 4096 2 0 2 2 1 1 1 0 8 1 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 7 0 7 2 1 1 1 0 8 1 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 11 0 1 1 0 1 1 0 8 0 uaddrrnd 24 721 0 690 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 721 0 690 1 0 1 1 0 8 0 vmmpekpl 168 7626 0 7589 3 0 3 3 0 8 0 vmmpepl 168 52380 0 50456 104 0 104 104 0 357 9 vmsppl 488 720 0 690 5 1 4 5 0 8 0 rwobjpl 80 16887 0 15831 30 0 30 30 0 8 2 pdppl 4096 1452 0 1381 102 30 72 86 0 8 1 pvpl 32 18230 0 0 149 1 148 148 0 265 0 pmappl 256 721 0 690 3 1 2 3 0 8 0 extentpl 40 45 0 27 1 0 1 1 0 8 0 phpool 112 284 0 36 8 0 8 8 0 8 0 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+0x27: addq $0x8,%rsp ddb{0}> trace x86_ipi_db(ffffffff837d3ff0) at x86_ipi_db+0x27 sys/arch/amd64/amd64/db_interface.c:394 x86_ipi_handler() at x86_ipi_handler+0xd9 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 mtx_enter(ffffffff839c4430) at mtx_enter+0x377 sys/kern/kern_lock.c:454 wakeup_n(ffffffff838eecf8,ffffffff) at wakeup_n+0x54 sys/kern/kern_synch.c:581 uvm_pmr_freepages(fffffd80082bf180,1) at uvm_pmr_freepages+0x3e8 sys/uvm/uvm_pmemrange.c:1341 pmap_do_remove(fffffd800b062400,69f2a48e000,69f2a4ce000,0) at pmap_do_remove+0x8af sys/arch/amd64/amd64/pmap.c:1939 uvm_unmap_kill_entry_withlock(fffffd800f7e95d8,fffffd806c5f3d88,0) at uvm_unmap_kill_entry_withlock+0x269 sys/uvm/uvm_map.c:1863 uvm_map_teardown(fffffd800f7e95d8) at uvm_map_teardown+0x117 uvm_map_addr_RBT_LEFT sys/uvm/uvm_map.h:-1 [inline] uvm_map_teardown(fffffd800f7e95d8) at uvm_map_teardown+0x117 sys/uvm/uvm_map.c:2491 exit1(ffff80003c400028,b,0,1) at exit1+0x6fc sys/kern/kern_exit.c:260 sys_exit(ffff80003c400028,ffff80003c43d270,ffff80003c43d1c0) at sys_exit+0x1a sys/kern/kern_exit.c:-1 syscall(ffff80003c43d270) at syscall+0xb17 mi_syscall sys/sys/syscall_mi.h:176 [inline] syscall(ffff80003c43d270) at syscall+0xb17 sys/arch/amd64/amd64/trap.c:775 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x77696f4069b0, count: -13 ddb{0}> machine ddbcpu 1 Stopped at db_enter+0x25: addq $0x8,%rsp ddb{1}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438 witness_checkorder(ffff8000299dee30,9,0) at witness_checkorder+0x10d1 sys/kern/subr_witness.c:-1 mtx_enter(ffff8000299dee20) at mtx_enter+0x95 sys/kern/kern_lock.c:406 clockintr_cancel(ffff8000299de2f8) at clockintr_cancel+0x2f sys/kern/kern_clockintr.c:323 mi_switch() at mi_switch+0x187 sys/kern/sched_bsd.c:376 ast(ffff80003c41f540) at ast+0x15a mi_ast sys/sys/syscall_mi.h:252 [inline] ast(ffff80003c41f540) at ast+0x15a sys/arch/amd64/amd64/trap.c:739 intr_user_exit() at intr_user_exit+0x3c end of kernel end trace frame: 0x78ea0fe55b50, count: -7