witness: lock order reversal:
 1st 0xffff8000299bee30 &cq->cq_mtx (&cq->cq_mtx)
 2nd 0xffffffff83a4a1c8 &sched_lock (&sched_lock)
lock order [1] &cq->cq_mtx (&cq->cq_mtx) -> [2] &sched_lock (&sched_lock)
#0  mtx_enter+0x95 sys/kern/kern_lock.c:406
#1  sleep_setup+0xed sys/kern/kern_synch.c:304
#2  msleep_nsec+0x10b sys/kern/kern_synch.c:217
#3  dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:589
#4  dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline]
#4  dtclose+0x109 sys/dev/dt/dt_dev.c:239
#5  spec_close+0x466 sys/kern/spec_vnops.c:-1
#6  VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156
#7  vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline]
#7  vn_closefile+0x12b sys/kern/vfs_vnops.c:615
#8  fdrop+0x121 sys/kern/kern_descrip.c:1281
#9  closef+0x192 sys/kern/kern_descrip.c:1265
#10 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#10 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783
#11 Xsyscall+0x128
lock order [2] &sched_lock (&sched_lock) -> [1] &cq->cq_mtx (&cq->cq_mtx)
#0  mtx_enter+0x95 sys/kern/kern_lock.c:406
#1  clockintr_cancel+0x2f sys/kern/kern_clockintr.c:323
#2  mi_switch+0x187 sys/kern/sched_bsd.c:376
#3  ast+0x15c mi_ast sys/sys/syscall_mi.h:252 [inline]
#3  ast+0x15c sys/arch/amd64/amd64/trap.c:747
#4  intr_user_exit+0x3c
Stopped at      db_enter+0x25:  addq    $0x8,%rsp
ddb{0}> 
ddb{0}> set $lines = 0
ddb{0}> set $maxwidth = 0
ddb{0}> show panic
the kernel did not panic
ddb{0}> trace
db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438
witness_checkorder(ffffffff83a4a1c8,9,0) at witness_checkorder+0x10d1 sys/kern/subr_witness.c:-1
mtx_enter(ffffffff83a4a1b8) at mtx_enter+0x95 sys/kern/kern_lock.c:406
sleep_setup(ffff8000299bee80,220,ffffffff834cb92c) at sleep_setup+0xed sys/kern/kern_synch.c:304
msleep_nsec(ffff8000299bee80,ffff8000299bee20,220,ffffffff834cb92c,ffffffffffffffff) at msleep_nsec+0x10b sys/kern/kern_synch.c:217
dt_ioctl_record_stop(ffff800001669000) at dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:589
dtclose(21e5f,81,2000,ffff80003c3f6558) at dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline]
dtclose(21e5f,81,2000,ffff80003c3f6558) at dtclose+0x109 sys/dev/dt/dt_dev.c:239
spec_close(ffff80003c4017c0) at spec_close+0x466 sys/kern/spec_vnops.c:-1
VOP_CLOSE(fffffd805f7a62d8,81,fffffd80097fd2d8,ffff80003c3f6558) at VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156
vn_closefile(fffffd806c19eea0,ffff80003c3f6558) at vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline]
vn_closefile(fffffd806c19eea0,ffff80003c3f6558) at vn_closefile+0x12b sys/kern/vfs_vnops.c:615
fdrop(fffffd806c19eea0,ffff80003c3f6558) at fdrop+0x121 sys/kern/kern_descrip.c:1281
closef(fffffd806c19eea0,ffff80003c3f6558) at closef+0x192 sys/kern/kern_descrip.c:1265
syscall(ffff80003c401a20) at syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
syscall(ffff80003c401a20) at syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0xefd3ba36cb0, count: -14
ddb{0}> show registers
rdi                                0
rsi                          0x80000    acpi_pdirpa+0x6be71
rbp               0xffff80003c401490
rbx                                0
rdx               0xffff8000015f3780
rcx               0xffff80003c3f6558
rax                          0x7ffff    acpi_pdirpa+0x6be70
r8                0xffff80003c401370
r9                0x8080808080808080
r10               0x6d9c6ee3acdd3690
r11               0xbe6b39d38d36cf0c
r12               0xfffffd800403ce40
r13               0xfffffd80047e7330
r14                              0x3
r15               0xffffffff834b72b3    substchar+0x1f5f
rip               0xffffffff82669d45    db_enter+0x25
cs                               0x8
rflags                         0x246
rsp               0xffff80003c401480
ss                              0x10
db_enter+0x25:  addq    $0x8,%rsp
ddb{0}> show proc
PROC (syz-executor) tid=39841 pid=21809 tcnt=2 stat=onproc
    flags process=0 proc=4000000<THREAD>
    runpri=32, usrpri=50, slppri=32, nice=20
    wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0
    forw=0xffffffffffffffff, list=0xffff80003c3f67f0,0xffff80003c3f6a98
    process=0xffff80003c3f9d00 user=0xffff80003c3fc000, vmspace=0xfffffd800f263b90
    estcpu=36, cpticks=0, pctcpu=0.0, user=0, sys=0, intr=0
ddb{0}> ps
   PID     TID   PPID    UID  S       FLAGS  WAIT          COMMAND
 14793  404676  21374      0  7           0                syz-executor
 14793  310800  21374      0  2   0x4000000                syz-executor
  2357  310695  94124      0  2           0                syz-executor
  2357  460262  94124      0  3   0x4000080  fsleep        syz-executor
 21809  299932  45938      0  2           0                syz-executor
*21809   39841  45938      0  7   0x4000000                syz-executor
 28135  279617  34420      0  2           0                syz-executor
 28135  421725  34420      0  3   0x4000080  fsleep        syz-executor
 11993  387011   4788      0  2           0                syz-executor
 11993  437860   4788      0  3   0x4000080  fsleep        syz-executor
 11993  126139   4788      0  3   0x4000080  fsleep        syz-executor
 36959  117080  72570      0  2           0                syz-executor
 36959  118987  72570      0  3   0x4000080  fsleep        syz-executor
 40855  151952  63964      0  2           0                syz-executor
 40855  113540  63964      0  3   0x4000080  fsleep        syz-executor
 45938  132711  95568      0  3        0x82  nanoslp       syz-executor
 61971   56641  95568      0  3         0x2  biowait       syz-executor
 94124   89618  95568      0  3        0x82  nanoslp       syz-executor
 72570  268984  95568      0  3        0x82  nanoslp       syz-executor
 21374  505804  95568      0  3        0x82  nanoslp       syz-executor
 34420  391513  95568      0  3        0x82  nanoslp       syz-executor
 63964  143070  95568      0  3        0x82  nanoslp       syz-executor
  4788   19530  95568      0  3        0x82  nanoslp       syz-executor
 95568  243334  45468      0  3        0x82  kqread        syz-executor
 45468  367638  61800      0  3    0x10008a  sigsusp       ksh
 61800  307194  96473      0  3        0x98  kqread        sshd-session
 96473   99133   7515      0  3        0x92  kqread        sshd-session
 20277  454032      1      0  3    0x100083  ttyin         getty
  7515  194856      1      0  3        0x88  kqread        sshd
 48961  234457  79622     74  3   0x1100092  bpf           pflogd
 79622  199961      1      0  3        0x80  sbwait        pflogd
 53604  153694  71334     73  3   0x1100090  kqread        syslogd
 71334   93242      1      0  3    0x100082  sbwait        syslogd
 47449  187759      1      0  3    0x100080  kqread        resolvd
 98473  115573  71865     77  3    0x100092  kqread        dhcpleased
 17492  152196  71865     77  3    0x100092  kqread        dhcpleased
 71865  233369      1      0  3        0x80  kqread        dhcpleased
 43538  146707      0      0  3     0x14200  bored         smr
 42173  499177      0      0  2     0x14200                zerothread
 30960  202486      0      0  3     0x14200  aiodoned      aiodoned
 58013   45443      0      0  3     0x14200  syncer        update
 90859  243273      0      0  3     0x14200  cleaner       cleaner
 61062  488410      0      0  3     0x14200  reaper        reaper
 20369  289171      0      0  3     0x14200  pgdaemon      pagedaemon
 68998  230390      0      0  3     0x14200  bored         viomb
 99469  275876      0      0  3  0x40014200  acpi0         acpi0
 24696  357891      0      0  3  0x40014200                idle1
 15117   71973      0      0  3     0x14200  bored         softnet1
 42042  444334      0      0  3     0x14200  bored         softnet0
 44442   55654      0      0  3     0x14200  bored         systqmp
 66664  408761      0      0  3     0x14200  bored         systq
 85289   35222      0      0  3     0x14200  tmoslp        softclockmp
 77972  172335      0      0  3  0x40014200  tmoslp        softclock
 84884    6438      0      0  3  0x40014200                idle0
     1   82064      0      0  3        0x82  wait          init
     0       0     -1      0  3  0x10010200  scheduler     swapper
ddb{0}> show all locks
CPU 0:
exclusive mutex &cq->cq_mtx r = 0 (0xffff8000299bee30)
#0  witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0  witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1  mtx_enter+0x4b4 sys/kern/kern_lock.c:487
#2  clockintr_unbind+0x56 sys/kern/kern_clockintr.c:375
#3  dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:589
#4  dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline]
#4  dtclose+0x109 sys/dev/dt/dt_dev.c:239
#5  spec_close+0x466 sys/kern/spec_vnops.c:-1
#6  VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156
#7  vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline]
#7  vn_closefile+0x12b sys/kern/vfs_vnops.c:615
#8  fdrop+0x121 sys/kern/kern_descrip.c:1281
#9  closef+0x192 sys/kern/kern_descrip.c:1265
#10 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#10 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783
#11 Xsyscall+0x128
Process 21809 (syz-executor) thread 0xffff80003c3f6558 (39841)
exclusive rwlock dtlk r = 0 (0xffffffff838c6f38)
#0  witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0  witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1  rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320
#2  dt_ioctl_record_stop+0x2e sys/dev/dt/dt_dev.c:572
#3  dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline]
#3  dtclose+0x109 sys/dev/dt/dt_dev.c:239
#4  spec_close+0x466 sys/kern/spec_vnops.c:-1
#5  VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156
#6  vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline]
#6  vn_closefile+0x12b sys/kern/vfs_vnops.c:615
#7  fdrop+0x121 sys/kern/kern_descrip.c:1281
#8  closef+0x192 sys/kern/kern_descrip.c:1265
#9  syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#9  syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783
#10 Xsyscall+0x128
exclusive kernel_lock &kernel_lock r = 0 (0xffffffff83aba240)
#0  witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0  witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1  vn_closefile+0x41 sys/kern/vfs_vnops.c:608
#2  fdrop+0x121 sys/kern/kern_descrip.c:1281
#3  closef+0x192 sys/kern/kern_descrip.c:1265
#4  syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#4  syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783
#5  Xsyscall+0x128
exclusive mutex &cq->cq_mtx r = 0 (0xffff8000299bee30)
#0  witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0  witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1  mtx_enter+0x4b4 sys/kern/kern_lock.c:487
#2  clockintr_unbind+0x56 sys/kern/kern_clockintr.c:375
#3  dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:589
#4  dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline]
#4  dtclose+0x109 sys/dev/dt/dt_dev.c:239
#5  spec_close+0x466 sys/kern/spec_vnops.c:-1
#6  VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156
#7  vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline]
#7  vn_closefile+0x12b sys/kern/vfs_vnops.c:615
#8  fdrop+0x121 sys/kern/kern_descrip.c:1281
#9  closef+0x192 sys/kern/kern_descrip.c:1265
#10 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#10 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783
#11 Xsyscall+0x128
Process 61971 (syz-executor) thread 0xffff8000fffef770 (56641)
exclusive rrwlock inode r = 0 (0xfffffd806b6141f0)
#0  witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0  witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1  rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320
#2  rrw_enter+0xc6 sys/kern/kern_rwlock.c:621
#3  VOP_LOCK+0xbd sys/kern/vfs_vops.c:527
#4  ufs_ihashins+0x4f ufs_ihash sys/ufs/ufs/ufs_ihash.c:-1 [inline]
#4  ufs_ihashins+0x4f sys/ufs/ufs/ufs_ihash.c:159
#5  ffs_vget+0x187 sys/ufs/ffs/ffs_vfsops.c:1232
#6  ffs_inode_alloc+0x279 sys/ufs/ffs/ffs_alloc.c:393
#7  ufs_mkdir+0xfc sys/ufs/ufs/ufs_vnops.c:1112
#8  VOP_MKDIR+0x101 sys/kern/vfs_vops.c:394
#9  domkdirat+0x179 sys/kern/vfs_syscalls.c:3143
#10 syscall+0xb17 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#10 syscall+0xb17 sys/arch/amd64/amd64/trap.c:783
#11 Xsyscall+0x128
exclusive rrwlock inode r = 0 (0xfffffd806ce22d70)
#0  witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0  witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1  rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320
#2  rrw_enter+0xc6 sys/kern/kern_rwlock.c:621
#3  VOP_LOCK+0xbd sys/kern/vfs_vops.c:527
#4  vn_lock+0xa4 sys/kern/vfs_vnops.c:570
#5  vfs_lookup+0x11c sys/kern/vfs_lookup.c:-1
#6  namei+0x7ca sys/kern/vfs_lookup.c:250
#7  domkdirat+0x8b sys/kern/vfs_syscalls.c:3128
#8  syscall+0xb17 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#8  syscall+0xb17 sys/arch/amd64/amd64/trap.c:783
#9  Xsyscall+0x128
ddb{0}> show malloc
           Type InUse  MemUse  HighUse   Limit  Requests Type Lim
         devbuf 11074  12209K   12344K 166960K     12350        0
            pcb    17     12K      12K 166960K        58        0
         rtable   231      7K       7K 166960K       376        0
             pf    35     17K      25K 166960K       106        0
         ifaddr    41      7K       7K 166960K        48        0
        ifgroup    55      2K       2K 166960K        70        0
         sysctl     4      1K       9K 166960K         9        0
       counters    70     37K      37K 166960K        76        0
       ioctlops     0      0K       4K 166960K      1628        0
            iov     0      0K      12K 166960K         9        0
          mount     1      1K       1K 166960K         1        0
            log     0      0K       0K 166960K         4        0
         vnodes  1363     86K      86K 166960K      1626        0
      UFS quota     1     32K      32K 166960K         1        0
      UFS mount     5     36K      36K 166960K         5        0
            shm     2      1K       5K 166960K         5        0
         VM map     2      1K       1K 166960K         2        0
            sem     8      0K       0K 166960K        10        0
        dirhash    12      2K       2K 166960K        12        0
           ACPI  1692    195K     286K 166960K     12470        0
      file desc    17     61K      89K 166960K       399        0
           proc    72    115K     180K 166960K       565        0
        subproc    72      4K       4K 166960K        72        0
    NFS srvsock     1      0K       0K 166960K         1        0
     NFS daemon     1     16K      16K 166960K         1        0
    ip_moptions     0      0K       0K 166960K         8        0
       in_multi    89      6K       7K 166960K       101        0
    ether_multi     1      0K       0K 166960K         1        0
            mrt     0      0K       0K 166960K        12        0
    ISOFS mount     1     32K      32K 166960K         1        0
  MSDOSFS mount     1     16K      16K 166960K         1        0
           ttys    67    307K     307K 166960K        67        0
           exec     0      0K       1K 166960K       392        0
   fusefs mount     1     32K      32K 166960K         1        0
     pfkey data     0      0K       0K 166960K         1        0
            tdb     3      0K       0K 166960K         3        0
        VM swap     8     62K      64K 166960K        10        0
       UVM amap   231    167K     185K 166960K      5354        0
       UVM aobj     9      2K       2K 166960K        10        0
     pinsyscall    42     84K     104K 166960K      1556        0
        memdesc     1      4K       4K 166960K         1        0
    crypto data     1      1K       1K 166960K         1        0
    ip6_options     0      0K       0K 166960K        11        0
            NDP    12      0K       2K 166960K        31        0
           temp    40   9071K    9139K 166960K     23760        0
         kqueue    13     20K      28K 166960K        56        0
      SYN cache     2     16K      16K 166960K         2        0
ddb{0}> show all pools
Name      Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle
plcache    128       26    0        0     1     0     1     1     0     8    0
rtpcb      120       76    0       73     2     0     2     2     0     8    1
rtentry    176      113    0        9     6     0     6     6     0     8    0
unpcb      144      139    0      122     2     0     2     2     0     8    1
syncache   336        5    0        5     1     0     1     1     0     8    1
tcpqe       32        1    0        1     1     0     1     1     0     8    1
tcpcb      736       36    0       31     1     0     1     1     0     8    0
arp        136       18    0        0     1     0     1     1     0     8    0
inpcb      328      183    0      175     2     0     2     2     0     8    0
nd6        152       24    0        3     1     0     1     1     0     8    0
pkpcb       40        2    0        2     1     0     1     1     0     8    1
kcovpl      48        8    0        0     1     0     1     1     0     8    0
ppxss      1192       1    0        1     1     0     1     1     0     8    1
pfstscr     40       10    0        9     1     0     1     1     0     8    0
pffrag     232        2    0        0     1     0     1     1     0   482    0
pffrnode    88        2    0        0     1     0     1     1     0     8    0
pffrent     40        3    0        1     1     0     1     1     0     8    0
pfosfp      40     1428    0     1005     5     0     5     5     0     8    0
pfosfpen   112     1428    0      714    21     0    21    21     0     8    0
pfanchor   1288       3    0        0     1     0     1     1     0     8    0
pfstitem    24       30    0        0     1     0     1     1     0     8    0
pfstkey    128       44    0       14     2     0     2     2     0     8    1
pfstate    448       36    0        7     4     0     4     4     0     8    0
pfrule     1360      43    0       38     2     1     1     2     0     8    0
rttmr      136        1    0        1     1     0     1     1     0     8    1
art_heap8  4096       2    0        0     2     0     2     2     0     8    0
art_heap4  256      453    0       40    29     0    29    29     0     8    0
art_table   40      455    0       40     5     0     5     5     0     8    0
art_node    32      113    0       18     1     0     1     1     0     8    0
sysvmsgpl   40        3    0        0     1     0     1     1     0     8    0
semapl     112        7    0        1     1     0     1     1     0     8    0
shmpl      112        7    0        1     1     0     1     1     0     8    0
dirhash    1024      17    0        0     3     0     3     3     0     8    0
dino2pl    256     2002    0      531    93     0    93    93     0     8    0
ffsino     296     2002    0      531   114     0   114   114     0     8    0
nchpl      144     2571    0      873    64     0    64    64     0     8    0
vnodes     216     2297    0        0   128     0   128   128     0     8    0
namei      1024    7745    0     7744     1     0     1     1     0     8    0
percpumem   16       53    0        3     1     0     1     1     0     8    0
kstatmem   264       32    0        6     2     0     2     2     0     8    0
acpiwqpl    32        1    0        1     1     0     1     1     1     8    1
scsiplug    72        1    0        1     1     0     1     1     0     8    1
scxspl     216     8977    0     8976    11     3     8     8     1     8    7
plimitpl   152       45    0       27     1     0     1     1     0     8    0
sigapl     424      716    0      669     7     0     7     7     0     8    1
knotepl    120      295    0        0     9     0     9     9     0     8    0
kqueuepl   224       72    0       63     1     0     1     1     0     8    0
pipepl     344      136    0      109     3     0     3     3     0     8    0
fdescpl    528      699    0      668     3     0     3     3     0     8    0
filepl     160     2787    0     2573    13     0    13    13     0     8    4
lockfpl    104       99    0       96     1     0     1     1     0     8    0
lockfspl    48       42    0       39     1     0     1     1     0     8    0
sessionpl  144       27    0       18     1     0     1     1     0     8    0
pgrppl      48       35    0       18     1     0     1     1     0     8    0
ucredpl    104      235    0      222     1     0     1     1     0     8    0
zombiepl   144      669    0      669     1     0     1     1     0     8    1
processpl  1232     716    0      669     5     0     5     5     0     8    0
procpl     664     1058    0     1003     6     0     6     6     0     8    1
sosppl     176        1    0        1     1     0     1     1     0     8    1
sockpl     752      409    0      381     7     0     7     7     0     8    3
mcl64k     65536     18    0        0     3     0     3     3     0     8    0
mcl16k     16384     14    0        0     2     0     2     2     0     8    0
mcl12k     12288      2    0        0     1     0     1     1     0     8    0
mcl9k      9216       2    0        0     1     0     1     1     0     8    0
mcl8k      8192       4    0        0     1     0     1     1     0     8    0
mcl4k      4096     122    0        0    16     0    16    16     0     8    0
mcl2k2     2112       2    0        0     1     0     1     1     0     8    0
mcl2k      2048      20    0        0     3     0     3     3     0     8    0
mtagpl      96        3    0        0     1     0     1     1     0     8    0
mbufpl     256      237    0        0    15     0    15    15     0     8    0
bufpl      280     3238    0      106   224     0   224   224     0     8    0
anonpl      32     5795    0        0    47     0    47    47     0   246    0
amapchunkpl 152   16773    0    16304    28     0    28    28     0   158    9
amappl16   200     2334    0     2306     5     3     2     5     0     8    0
amappl15   192        6    0        6     1     1     0     1     0     8    0
amappl14   184      533    0      532     1     0     1     1     0     8    0
amappl13   176      130    0      118     1     0     1     1     0     8    0
amappl12   168      950    0      919     2     0     2     2     0     8    0
amappl11   160        8    0        8     1     1     0     1     0     8    0
amappl10   152       99    0       85     1     0     1     1     0     8    0
amappl9    144      265    0      265     1     1     0     1     0     8    0
amappl8    136      112    0      110     1     0     1     1     0     8    0
amappl7    128      147    0      134     1     0     1     1     0     8    0
amappl6    120      166    0      164     1     0     1     1     0     8    0
amappl5    112      100    0       89     1     0     1     1     0     8    0
amappl4    104      297    0      277     1     0     1     1     0     8    0
amappl3     96     3215    0     3103     4     0     4     4     0     8    0
amappl2     88      544    0      483     2     0     2     2     0     8    0
amappl1     80    11366    0    10772    16     0    16    16     0     8    1
amappl      88     4560    0     4402     5     0     5     5     0    92    1
uvmvnodes   80      121    0        0     3     0     3     3     0     8    0
dma4096    4096       1    0        1     1     1     0     1     0     8    0
dma1024    1024       1    0        0     1     0     1     1     0     8    0
dma256     256        6    0        6     1     1     0     1     0     8    0
dma128     128      254    0      254     2     1     1     1     0     8    1
dma64       64        6    0        6     1     1     0     1     0     8    0
dma32       32        7    0        7     1     1     0     1     0     8    0
dma16       16       18    0       17     1     0     1     1     0     8    0
aobjpl      72        9    0        1     1     0     1     1     0     8    0
uaddrrnd    24      699    0      668     1     0     1     1     0     8    0
uaddrbest   32        2    0        0     1     0     1     1     0     8    0
uaddr       24      699    0      668     1     0     1     1     0     8    0
vmmpekpl   168     7375    0     7341     2     0     2     2     0     8    0
vmmpepl    168    52967    0    51067    97     0    97    97     0   357   11
vmsppl     488      698    0      668     5     0     5     5     0     8    1
rwobjpl     80    17431    0    16391    27     1    26    26     0     8    1
pdppl      4096    1405    0     1336    99    24    75    83     0     8    6
pvpl        32    14313    0        0   116     0   116   116     0   265    0
pmappl     256      698    0      668     3     0     3     3     0     8    0
extentpl    40       45    0       27     1     0     1     1     0     8    0
phpool     112      281    0       27     8     0     8     8     0     8    0
ddb{0}> machine ddbcpu 0
Invalid cpu 0
ddb{0}> trace
db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438
witness_checkorder(ffffffff83a4a1c8,9,0) at witness_checkorder+0x10d1 sys/kern/subr_witness.c:-1
mtx_enter(ffffffff83a4a1b8) at mtx_enter+0x95 sys/kern/kern_lock.c:406
sleep_setup(ffff8000299bee80,220,ffffffff834cb92c) at sleep_setup+0xed sys/kern/kern_synch.c:304
msleep_nsec(ffff8000299bee80,ffff8000299bee20,220,ffffffff834cb92c,ffffffffffffffff) at msleep_nsec+0x10b sys/kern/kern_synch.c:217
dt_ioctl_record_stop(ffff800001669000) at dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:589
dtclose(21e5f,81,2000,ffff80003c3f6558) at dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline]
dtclose(21e5f,81,2000,ffff80003c3f6558) at dtclose+0x109 sys/dev/dt/dt_dev.c:239
spec_close(ffff80003c4017c0) at spec_close+0x466 sys/kern/spec_vnops.c:-1
VOP_CLOSE(fffffd805f7a62d8,81,fffffd80097fd2d8,ffff80003c3f6558) at VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156
vn_closefile(fffffd806c19eea0,ffff80003c3f6558) at vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline]
vn_closefile(fffffd806c19eea0,ffff80003c3f6558) at vn_closefile+0x12b sys/kern/vfs_vnops.c:615
fdrop(fffffd806c19eea0,ffff80003c3f6558) at fdrop+0x121 sys/kern/kern_descrip.c:1281
closef(fffffd806c19eea0,ffff80003c3f6558) at closef+0x192 sys/kern/kern_descrip.c:1265
syscall(ffff80003c401a20) at syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
syscall(ffff80003c401a20) at syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0xefd3ba36cb0, count: -14
ddb{0}> machine ddbcpu 1
Stopped at      x86_ipi_db+0x27:        addq    $0x8,%rsp
ddb{1}> trace
x86_ipi_db(ffff8000299bdff0) at x86_ipi_db+0x27 sys/arch/amd64/amd64/db_interface.c:394
x86_ipi_handler() at x86_ipi_handler+0xd9 sys/arch/amd64/amd64/ipi.c:106
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
mtx_enter(ffff8000299bee20) at mtx_enter+0x377 sys/kern/kern_lock.c:454
clockintr_dispatch(ffff80002eba5180) at clockintr_dispatch+0x401 sys/kern/kern_clockintr.c:215
lapic_clockintr(0,0) at lapic_clockintr+0x51 sys/arch/amd64/amd64/lapic.c:482
Xresume_lapic_ltimer() at Xresume_lapic_ltimer+0x2a
end of kernel
end trace frame: 0x78a86ea8bdd0, count: -7