witness: lock order reversal:
 1st 0xffffffff82cfb918 &sched_lock (&sched_lock)
 2nd 0xffff80002121ef30 &pr->ps_mtx (&pr->ps_mtx)
lock order "&pr->ps_mtx"(mutex) -> "&sched_lock"(sched_lock) first seen at:
#0  __mp_lock+0xa2 read_rflags machine/cpufunc.h:195 [inline]
#0  __mp_lock+0xa2 intr_disable machine/cpufunc.h:216 [inline]
#0  __mp_lock+0xa2 sys/kern/kern_lock.c:142
#1  donice+0x178 sys/kern/kern_resource.c:218
#2  syscall+0x5e2 mi_syscall sys/sys/syscall_mi.h:110 [inline]
#2  syscall+0x5e2 sys/arch/amd64/amd64/trap.c:623
#3  Xsyscall+0x128
lock order "&sched_lock"(sched_lock) -> "&pr->ps_mtx"(mutex) first seen at:
#0  mtx_enter+0x3e sys/kern/kern_lock.c:265
#1  single_thread_set+0x35a single_thread_wait sys/kern/kern_sig.c:2172 [inline]
#1  single_thread_set+0x35a sys/kern/kern_sig.c:2155
#2  sigexit+0x90 sys/kern/kern_sig.c:1562
#3  postsig+0x4a9 sys/kern/kern_sig.c:1494
#4  userret+0x16e sys/kern/kern_sig.c:1987
#5  syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline]
#5  syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644
#6  Xsyscall+0x128
Stopped at      db_enter+0x1c:  addq    $0x8,%rsp
ddb{1}> 
ddb{1}> set $lines = 0
ddb{1}> set $maxwidth = 0
ddb{1}> show panic
the kernel did not panic
ddb{1}> trace
db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437
witness_checkorder(ffff80002121ef30,9,0) at witness_checkorder+0x10b7 witness_debugger sys/kern/subr_witness.c:2510 [inline]
witness_checkorder(ffff80002121ef30,9,0) at witness_checkorder+0x10b7 sys/kern/subr_witness.c:1110
mtx_enter(ffff80002121ef20) at mtx_enter+0x3e sys/kern/kern_lock.c:265
single_thread_set(ffff8000211f5560,0,1) at single_thread_set+0x35a single_thread_wait sys/kern/kern_sig.c:2172 [inline]
single_thread_set(ffff8000211f5560,0,1) at single_thread_set+0x35a sys/kern/kern_sig.c:2155
sigexit(ffff8000211f5560,6) at sigexit+0x90 sys/kern/kern_sig.c:1562
postsig(ffff8000211f5560,6,ffff8000212a76d8) at postsig+0x4a9 sys/kern/kern_sig.c:1494
userret(ffff8000211f5560) at userret+0x16e sys/kern/kern_sig.c:1987
syscall(ffff8000212a7810) at syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline]
syscall(ffff8000212a7810) at syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0xd0b507df190, count: -9
ddb{1}> show registers
rdi                              0x3
rsi               0xffffffff82b8e4a0    __sancov_gen_cov_switch_values.134
rbp               0xffff8000212a73e0
rbx                              0x3
rdx                            0x3fd
rcx                                0
rax                              0x1
r8                0xffff8000212a7350
r9                0x8080808080808080
r10               0xf7fa211f888a2172
r11               0x1a374c6c29b9a75b
r12               0xfffffd8003a0e590
r13                                0
r14               0xfffffd8003a06ea0
r15               0xfffffd80031cbbc0
rip               0xffffffff810eab1c    db_enter+0x1c
cs                               0x8
rflags                         0x246
rsp               0xffff8000212a73d0
ss                              0x10
db_enter+0x1c:  addq    $0x8,%rsp
ddb{1}> show proc
PROC (syz-executor.0) pid=144751 stat=onproc
    flags process=100000<PLEDGE> proc=4002000<WEXIT,THREAD>
    pri=32, usrpri=50, nice=20
    forw=0xffffffffffffffff, list=0xffff8000211f52b8,0xffffffff82cfc210
    process=0xffff80002121ee20 user=0xffff8000212a2000, vmspace=0xfffffd8008a16910
    estcpu=36, cpticks=0, pctcpu=0.0
    user=0, sys=0, intr=0
ddb{1}> ps
   PID     TID   PPID    UID  S       FLAGS  WAIT          COMMAND
 11689  292002  48983      0  4    0x182000                syz-executor.0
*11689  144751  48983      0  7   0x4102000                syz-executor.0
 48983  355345  13863      0  3        0x82  nanoslp       syz-executor.0
 13863  335598  24498      0  3   0x2000082  thrsleep      syz-execprog
 13863   62261  24498      0  3   0x6000082  thrsleep      syz-execprog
 13863  169013  24498      0  3   0x6000082  thrsleep      syz-execprog
 13863  297067  24498      0  3   0x6000082  thrsleep      syz-execprog
 13863  231040  24498      0  3   0x6000082  thrsleep      syz-execprog
 13863  209383  24498      0  3   0x6000082  thrsleep      syz-execprog
 13863  394308  24498      0  3   0x6000082  kqread        syz-execprog
 13863  389240  24498      0  3   0x6000082  thrsleep      syz-execprog
 13863  386442  24498      0  3   0x6000082  wait          syz-execprog
 13863  306752  24498      0  3   0x6000082  thrsleep      syz-execprog
 13863   86716  24498      0  3   0x6000082  thrsleep      syz-execprog
 24498   42504  74284      0  3    0x10008a  sigsusp       ksh
 74284    9292  88596      0  3        0x9a  kqread        sshd
 29938  495664      1      0  3    0x100083  ttyin         getty
 88596  388804      1      0  3        0x88  kqread        sshd
 56969   46580  66911     74  3   0x1100092  bpf           pflogd
 66911  403112      1      0  3        0x80  netio         pflogd
 32972  514910  51375     73  3   0x1100090  kqread        syslogd
 51375  124933      1      0  3    0x100082  netio         syslogd
  2340  503501      1      0  3    0x100080  kqread        resolvd
 71725  153277  82963     77  3    0x100092  kqread        dhcpleased
 14688  183077  82963     77  3    0x100092  kqread        dhcpleased
 82963   88972      1      0  3        0x80  kqread        dhcpleased
 27526  286132      0      0  3     0x14200  bored         smr
 43010  271103      0      0  7     0x14200                zerothread
 65069  421715      0      0  3     0x14200  aiodoned      aiodoned
 97188  362403      0      0  3     0x14200  syncer        update
 21575  282965      0      0  3     0x14200  cleaner       cleaner
 77054   39759      0      0  3     0x14200  reaper        reaper
 42607   25774      0      0  3     0x14200  pgdaemon      pagedaemon
 95552  400143      0      0  3     0x14200  bored         viomb
 75236  493921      0      0  3  0x40014200  acpi0         acpi0
 92234  426578      0      0  3  0x40014200                idle1
 29906  494059      0      0  3     0x14200  bored         softnet3
 96667  285575      0      0  3     0x14200  bored         softnet2
 68867  397023      0      0  3     0x14200  bored         softnet1
 73868   16211      0      0  3     0x14200  bored         softnet0
 22957    6554      0      0  3     0x14200  bored         systqmp
 85660  392094      0      0  3     0x14200  bored         systq
 43346  360227      0      0  3  0x40014200  bored         softclock
 17960  502908      0      0  3  0x40014200                idle0
     1    9937      0      0  3        0x82  wait          init
     0       0     -1      0  3     0x10200  scheduler     swapper
ddb{1}> show all locks
CPU 1:
exclusive sched_lock &sched_lock r = 0 (0xffffffff82cfb918)
#0  witness_lock+0x447
#1  single_thread_set+0x179 sys/kern/kern_sig.c:2113
#2  sigexit+0x90 sys/kern/kern_sig.c:1562
#3  postsig+0x4a9 sys/kern/kern_sig.c:1494
#4  userret+0x16e sys/kern/kern_sig.c:1987
#5  syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline]
#5  syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644
#6  Xsyscall+0x128
Process 11689 (syz-executor.0) thread 0xffff8000211f5560 (144751)
exclusive kernel_lock &kernel_lock r = 0 (0xffffffff82c82c68)
#0  witness_lock+0x447
#1  postsig+0x49e sys/kern/kern_sig.c:1500
#2  userret+0x16e sys/kern/kern_sig.c:1987
#3  syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline]
#3  syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644
#4  Xsyscall+0x128
exclusive sched_lock &sched_lock r = 0 (0xffffffff82cfb918)
#0  witness_lock+0x447
#1  single_thread_set+0x179 sys/kern/kern_sig.c:2113
#2  sigexit+0x90 sys/kern/kern_sig.c:1562
#3  postsig+0x4a9 sys/kern/kern_sig.c:1494
#4  userret+0x16e sys/kern/kern_sig.c:1987
#5  syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline]
#5  syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644
#6  Xsyscall+0x128
ddb{1}> show malloc
           Type InUse  MemUse  HighUse   Limit  Requests Type Lim
         devbuf 10165   6458K    6458K  78643K     11243        0
            pcb    13      8K       8K  78643K        13        0
         rtable    80      2K       2K  78643K       144        0
             pf    18      7K      10K  78643K        29        0
         ifaddr    17     10K      10K  78643K        17        0
        ifgroup    27      1K       1K  78643K        27        0
       counters    46     34K      34K  78643K        46        0
       ioctlops     0      0K       4K  78643K      1476        0
          mount     1      1K       1K  78643K         1        0
            log     0      0K       0K  78643K         4        0
         vnodes  1176     74K      74K  78643K      1192        0
      UFS quota     1     32K      32K  78643K         1        0
      UFS mount     5     36K      36K  78643K         5        0
            shm     2      1K       1K  78643K         2        0
         VM map     2      1K       1K  78643K         2        0
            sem     2      0K       0K  78643K         2        0
        dirhash    12      2K       2K  78643K        12        0
           ACPI  1697    195K     286K  78643K     12548        0
      file desc     4      9K      13K  78643K        25        0
           proc    67     91K     103K  78643K       330        0
    NFS srvsock     1      0K       0K  78643K         1        0
     NFS daemon     1     16K      16K  78643K         1        0
       in_multi    22      1K       1K  78643K        22        0
    ether_multi     1      0K       0K  78643K         1        0
    ISOFS mount     1     32K      32K  78643K         1        0
  MSDOSFS mount     1     16K      16K  78643K         1        0
           ttys    25    122K     122K  78643K        25        0
           exec     0      0K       1K  78643K       299        0
            tdb     3      0K       0K  78643K         3        0
        pagedep     1      8K       8K  78643K         1        0
       inodedep     1     32K      32K  78643K         1        0
         newblk     1      0K       0K  78643K         1        0
        VM swap     8     62K      64K  78643K        10        0
       UVM amap   167     16K      18K  78643K      2903        0
       UVM aobj     3      2K       2K  78643K         3        0
        memdesc     1      4K       4K  78643K         1        0
    crypto data     1      1K       1K  78643K         1        0
            NDP     7      0K       0K  78643K         7        0
           temp     1   5904K    5968K  78643K      3461        0
         kqueue    12     18K      18K  78643K        25        0
      SYN cache     2     16K      16K  78643K         2        0
ddb{1}> show all pools
Name      Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle
plcache    128       22    0        0     1     0     1     1     0     8    0
rtpcb      120       22    0       19     1     0     1     1     0     8    0
rtentry    112       34    0        1     1     0     1     1     0     8    0
unpcb      144       35    0       20     1     0     1     1     0     8    0
syncache   304        5    0        5     1     0     1     1     0     8    1
tcpqe       32      208    0      208     1     1     0     1     0     8    0
tcpcb      808        8    0        5     1     0     1     1     0     8    0
arp        120        4    0        0     1     0     1     1     0     8    0
inpcb      368       37    0       31     1     0     1     1     0     8    0
nd6        136        3    0        0     1     0     1     1     0     8    0
pfosfp      40     1428    0     1005     5     0     5     5     0     8    0
pfosfpen   112     1428    0      714    21     0    21    21     0     8    0
pfstitem    24       11    0        0     1     0     1     1     0     8    0
pfstkey    128       11    0        0     1     0     1     1     0     8    0
pfstate    376       11    0        0     2     0     2     2     0     8    0
pfrule     1344      21    0       16     2     1     1     2     0     8    0
art_heap8  4096       1    0        0     1     0     1     1     0     8    0
art_heap4  256      144    0        0     9     0     9     9     0     8    0
art_table   32      145    0        0     2     0     2     2     0     8    0
art_node    16       33    0        3     1     0     1     1     0     8    0
dirhash    1024      17    0        0     3     0     3     3     0     8    0
dino2pl    256     1465    0       52    89     0    89    89     0     8    0
ffsino     272     1465    0       52    95     0    95    95     0     8    0
nchpl      144     1704    0       63    61     0    61    61     0     8    0
uvmvnodes   80     1479    0        0    31     0    31    31     0     8    0
vnodes     216     1479    0        0    83     0    83    83     0     8    0
namei      1024    5490    0     5490     2     0     2     2     0     8    2
percpumem   16       36    0        0     1     0     1     1     0     8    0
kstatmem   264       10    0        0     1     0     1     1     0     8    0
scxspl     216     6303    0     6303     3     2     1     2     1     8    1
plimitpl   152       28    0       19     1     0     1     1     0     8    0
sigapl     424      368    0      334     7     1     6     6     0     8    2
knotepl    120       70    0        0     3     0     3     3     0     8    0
kqueuepl   216       21    0       13     1     0     1     1     0     8    0
pipepl     320      118    0      111     2     0     2     2     0     8    1
fdescpl    496      351    0      334     4     0     4     4     0     8    1
filepl     152     1538    0     1468     4     0     4     4     0     8    1
lockfpl    104        6    0        4     1     0     1     1     0     8    0
lockfspl    48        4    0        2     1     0     1     1     0     8    0
sessionpl  144       19    0        9     1     0     1     1     0     8    0
pgrppl      48       19    0        9     1     0     1     1     0     8    0
ucredpl    104       71    0       59     1     0     1     1     0     8    0
zombiepl   144      334    0      334     1     0     1     1     0     8    1
processpl  1072     368    0      334     4     0     4     4     0     8    1
procpl     680      379    0      334     6     1     5     5     0     8    1
sockpl     488       94    0       70     4     0     4     4     0     8    0
mcl8k      8192       5    0        0     1     0     1     1     0     8    0
mcl4k      4096       2    0        0     1     0     1     1     0     8    0
mcl2k      2048     257    0        0    33     0    33    33     0     8    0
mtagpl      96        2    0        0     1     0     1     1     0     8    0
mbufpl     256      276    0        0    18     0    18    18     0     8    0
bufpl      288     4117    0      148   284     0   284   284     0     8    0
anonpl      24   174076    0   170293    58     0    58    58     0   186   32
amapchunkpl 152    9668    0     9268    28     0    28    28     0   158   11
amappl16   200     4637    0     4574     5     0     5     5     0     8    0
amappl15   192       12    0       11     1     0     1     1     0     8    0
amappl14   184      139    0      127     2     0     2     2     0     8    1
amappl13   176        8    0        8     1     1     0     1     0     8    0
amappl12   168      928    0      910     3     1     2     2     0     8    1
amappl11   160       60    0       44     1     0     1     1     0     8    0
amappl10   152       28    0       25     1     0     1     1     0     8    0
amappl9    144      184    0      184     1     0     1     1     0     8    1
amappl8    136      104    0       81     2     0     2     2     0     8    1
amappl7    128       61    0       56     1     0     1     1     0     8    0
amappl6    120      191    0      172     3     1     2     3     0     8    1
amappl5    112      140    0      129     1     0     1     1     0     8    0
amappl4    104      551    0      511     3     0     3     3     0     8    1
amappl3     96     2478    0     2432     3     1     2     2     0     8    0
amappl2     88      617    0      560     3     0     3     3     0     8    1
amappl1     80    10268    0     9768    23     0    23    23     0     8   10
amappl      88     2451    0     2349     5     0     5     5     0    92    2
dma4096    4096       1    0        1     1     1     0     1     0     8    0
dma1024    1024       1    0        0     1     0     1     1     0     8    0
dma256     256        6    0        6     1     1     0     1     0     8    0
dma128     128      253    0      253     1     1     0     1     0     8    0
dma64       64        6    0        6     1     1     0     1     0     8    0
dma32       32        7    0        7     1     1     0     1     0     8    0
dma16       16       18    0       17     1     0     1     1     0     8    0
aobjpl      72        2    0        0     1     0     1     1     0     8    0
uaddrrnd    24      351    0      334     1     0     1     1     0     8    0
uaddrbest   32        2    0        0     1     0     1     1     0     8    0
uaddr       24      351    0      334     1     0     1     1     0     8    0
vmmpekpl   168     9592    0     9568     2     0     2     2     0     8    0
vmmpepl    168    41382    0    40049   119     1   118   118     0   357   60
vmsppl     464      350    0      334     4     0     4     4     0     8    1
rwobjpl     56    20005    0    17641    47     0    47    47     0     8    8
pdppl      4096     710    0      668    84    18    66    66     0     8   24
pvpl        32   368959    0   360631   367     0   367   367     0   265  291
pmappl     248      350    0      334     2     0     2     2     0     8    0
extentpl    40       56    0       38     1     0     1     1     0     8    0
phpool     112      631    0       21    18     0    18    18     0     8    0
ddb{1}> machine ddbcpu 0
Stopped at      x86_ipi_db+0x1e:        addq    $0x8,%rsp
ddb{0}> trace
x86_ipi_db(ffffffff82be4ff0) at x86_ipi_db+0x1e sys/arch/amd64/amd64/db_interface.c:393
x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
__mp_lock(ffffffff82cfb710) at __mp_lock+0x122 __mp_lock_spin sys/kern/kern_lock.c:116 [inline]
__mp_lock(ffffffff82cfb710) at __mp_lock+0x122 sys/kern/kern_lock.c:147
yield() at yield+0x37 sys/kern/sched_bsd.c:321
uvm_pagezero_thread(ffff8000211b2800) at uvm_pagezero_thread+0x6d sys/uvm/uvm_pmemrange.c:2170
end trace frame: 0x0, count: -6
ddb{0}> machine ddbcpu 1
Stopped at      db_enter+0x1c:  addq    $0x8,%rsp
ddb{1}> trace
db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437
witness_checkorder(ffff80002121ef30,9,0) at witness_checkorder+0x10b7 witness_debugger sys/kern/subr_witness.c:2510 [inline]
witness_checkorder(ffff80002121ef30,9,0) at witness_checkorder+0x10b7 sys/kern/subr_witness.c:1110
mtx_enter(ffff80002121ef20) at mtx_enter+0x3e sys/kern/kern_lock.c:265
single_thread_set(ffff8000211f5560,0,1) at single_thread_set+0x35a single_thread_wait sys/kern/kern_sig.c:2172 [inline]
single_thread_set(ffff8000211f5560,0,1) at single_thread_set+0x35a sys/kern/kern_sig.c:2155
sigexit(ffff8000211f5560,6) at sigexit+0x90 sys/kern/kern_sig.c:1562
postsig(ffff8000211f5560,6,ffff8000212a76d8) at postsig+0x4a9 sys/kern/kern_sig.c:1494
userret(ffff8000211f5560) at userret+0x16e sys/kern/kern_sig.c:1987
syscall(ffff8000212a7810) at syscall+0x4e7 mi_syscall_return sys/sys/syscall_mi.h:137 [inline]
syscall(ffff8000212a7810) at syscall+0x4e7 sys/arch/amd64/amd64/trap.c:644
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0xd0b507df190, count: -9
ddb{1}>