syzbot


panic: _mtx_lock_sleep: recursed on non-recursive mutex so_snd @ /syzkaller/managers/main/kernel/sys/modules/tcp/rack/..

Status: fixed on 2021/05/22 04:14
Reported-by: syzbot+b279ddc8ab51b2b77a9e@syzkaller.appspotmail.com
Fix commit: 39756885633f rack: honor prior socket buffer lock when doing the upcall
First crash: 1071d, last: 1070d

Sample crash report:
panic: _mtx_lock_sleep: recursed on non-recursive mutex so_snd @ /syzkaller/managers/main/kernel/sys/modules/tcp/rack/../../../netinet/tcp_stacks/rack.c:10632

cpuid = 0
time = 1621643990
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x47/frame 0xfffffe0003779310
vpanic() at vpanic+0x1c7/frame 0xfffffe0003779370
panic() at panic+0x43/frame 0xfffffe00037793d0
__mtx_lock_sleep() at __mtx_lock_sleep+0x6e9/frame 0xfffffe0003779460
__mtx_lock_flags() at __mtx_lock_flags+0x150/frame 0xfffffe00037794c0
rack_do_established() at rack_do_established+0x127b/frame 0xfffffe00037795d0
rack_do_segment_nounlock() at rack_do_segment_nounlock+0x2649/frame 0xfffffe00037797d0
rack_do_segment() at rack_do_segment+0x121/frame 0xfffffe0003779850
tcp_input_with_port() at tcp_input_with_port+0x13d5/frame 0xfffffe00037799b0
tcp_input() at tcp_input+0x1f/frame 0xfffffe00037799d0
ip_input() at ip_input+0x38b/frame 0xfffffe0003779a90
swi_net() at swi_net+0x20d/frame 0xfffffe0003779b10
ithread_loop() at ithread_loop+0x33f/frame 0xfffffe0003779bb0
fork_exit() at fork_exit+0xb3/frame 0xfffffe0003779bf0
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0003779bf0
--- trap 0, rip = 0, rsp = 0, rbp = 0 ---
KDB: enter: panic
[ thread pid 12 tid 100032 ]
Stopped at      kdb_enter+0x67: movq    $0,0x16390de(%rip)
db> 
db> set $lines = 0
db> set $maxwidth = 0
db> show registers
cs                        0x20
ds                        0x3b
es                        0x3b
fs                        0x13
gs                        0x1b
ss                        0x28
rax                       0x12
rcx         0xffffffff81138d00  vprintf+0x140
rdx                        0x1
rbx                          0
rsp         0xfffffe00037792f0
rbp         0xfffffe0003779310
rsi                          0
rdi         0xffffffff81138d36  vprintf+0x176
r8                           0
r9          0x8080808080808080
r10         0xfffffe00037791e0
r11              0x1fffcfff59c
r12         0xffffffff82267b80  ddb_dbbe
r13                          0
r14         0xffffffff81a73b6e
r15         0xffffffff81a73b6e
rip         0xffffffff8112fd27  kdb_enter+0x67
rflags                    0x86
kdb_enter+0x67: movq    $0,0x16390de(%rip)
db> show proc
Process 12 (intr) at 0xfffff80004c5ca70:
 state: NORMAL
 uid: 0  gids: 0
 parent: pid 0 at 0xffffffff8271c330
 ABI: null
 flag: 0x10000284  flag2: 0
 reaper: 0xffffffff8271c330 reapsubtree: 12
 sigparent: 20
 vmspace: 0xffffffff8271cfb0
   (map 0xffffffff8271cfb0)
   (map.pmap 0xffffffff8271d070)
   (pmap 0xffffffff8271d0d0)
 threads: 23
100012                   I                                   [swi5: fast taskq]
100015                   I                                   [swi6: task queue]
100017                   I                                   [swi6: Giant taskq]
100030                   I                                   [swi4: clock (0)]
100031                   I                                   [swi4: clock (1)]
100032                   Run     CPU 0                       [swi1: netisr 0]
100033                   I                                   [swi3: vm]
100046                   I                                   [irq24: virtio_pci0]
100047                   I                                   [irq25: virtio_pci0]
100048                   I                                   [irq26: virtio_pci0]
100049                   I                                   [irq27: virtio_pci0]
100050                   I                                   [irq28: virtio_pci1]
100051                   I                                   [irq29: virtio_pci1]
100052                   I                                   [irq30: virtio_pci1]
100053                   I                                   [irq31: virtio_pci1]
100054                   I                                   [irq32: virtio_pci1]
100059                   I                                   [irq10: virtio_pci2]
100061                   I                                   [irq1: atkbd0]
100062                   I                                   [irq12: psm0]
100063                   I                                   [swi0: uart uart++]
100071                   I                                   [swi1: pf send]
100084                   I                                   [swi1: hpts]
100085                   I                                   [swi1: hpts]
db> ps
  pid  ppid  pgrp   uid  state   wmesg   wchan               cmd
50102   861   861     0  R                                   syz-executor.3
50101   783   783     0  R       (threaded)                  syz-executor.0
154560                   RunQ                                syz-executor.0
160328                   RunQ                                syz-executor.0
50098   784   784     0  R       (threaded)                  syz-executor.1
160322                   RunQ                                syz-executor.1
160329                   S       uwait   0xfffff80061295900  syz-executor.1
50096   804   804     0  R       (threaded)                  syz-executor.2
160296                   RunQ                                syz-executor.2
160325                   RunQ                                syz-executor.2
49953     1   861     0  S       uwait   0xfffff80074a6c700  syz-executor.3
49951     1   861     0  S       uwait   0xfffff8004c165500  syz-executor.3
49708     1   783     0  S       uwait   0xfffff8000bd04b00  syz-executor.0
49706     1   783     0  S       uwait   0xfffff8004c165700  syz-executor.0
49679     1   861     0  S       uwait   0xfffff80061295d00  syz-executor.3
49678     1   861     0  S       uwait   0xfffff8000a178100  syz-executor.3
48445     1   804     0  S       uwait   0xfffff80074a6c300  syz-executor.2
48438     1   804     0  S       uwait   0xfffff8004c165f00  syz-executor.2
48110     1   784     0  S       uwait   0xfffff80074c54100  syz-executor.1
48106     1   784     0  S       uwait   0xfffff8000a70b100  syz-executor.1
47944     1   784     0  S       uwait   0xfffff80033e02d00  syz-executor.1
47940     1   784     0  S       uwait   0xfffff8000a178b00  syz-executor.1
47590     1   804     0  S       uwait   0xfffff80074a17d00  syz-executor.2
47589     1   804     0  S       uwait   0xfffff80033e24b00  syz-executor.2
46285     1   804     0  S       uwait   0xfffff80033e02300  syz-executor.2
46284     1   804     0  S       uwait   0xfffff80020334900  syz-executor.2
46178     1   804     0  S       uwait   0xfffff80033e20b00  syz-executor.2
46177     1   804     0  S       uwait   0xfffff8000bd99b00  syz-executor.2
46160     1   804     0  S       uwait   0xfffff800075cc700  syz-executor.2
46159     1   804     0  S       uwait   0xfffff80074a17700  syz-executor.2
44809     1   784     0  S       uwait   0xfffff80033e20100  syz-executor.1
44805     1   784     0  S       uwait   0xfffff80074c54500  syz-executor.1
44798     1   784     0  S       uwait   0xfffff80074c54d00  syz-executor.1
44794     1   784     0  S       uwait   0xfffff80033e24500  syz-executor.1
44787     1   784     0  S       uwait   0xfffff80074a6c500  syz-executor.1
44781     1   784     0  S       uwait   0xfffff80074a6cb00  syz-executor.1
44775     1   784     0  S       uwait   0xfffff80033e02700  syz-executor.1
44767     1   784     0  S       uwait   0xfffff8000bd99300  syz-executor.1
44761     1   784     0  S       uwait   0xfffff80074a6c900  syz-executor.1
44758     1   784     0  S       uwait   0xfffff8000a722900  syz-executor.1
44749     1   784     0  S       uwait   0xfffff80074c54300  syz-executor.1
44743     1   784     0  S       uwait   0xfffff80074c54b00  syz-executor.1
44738     1   784     0  S       uwait   0xfffff80033e20700  syz-executor.1
44732     1   784     0  S       uwait   0xfffff80074c54900  syz-executor.1
44613     1   783     0  S       uwait   0xfffff80020305900  syz-executor.0
44608     1   783     0  S       uwait   0xfffff80033e20900  syz-executor.0
44102     1   784     0  S       uwait   0xfffff80074c54f00  syz-executor.1
44101     1   784     0  S       uwait   0xfffff80033e02f00  syz-executor.1
42764     1   784     0  S       uwait   0xfffff800075cc100  syz-executor.1
42763     1   784     0  S       umtxn   0xfffff800075cc500  syz-executor.1
42115     1   804     0  S       uwait   0xfffff8000c3ce900  syz-executor.2
42113     1   804     0  S       uwait   0xfffff80033e1e700  syz-executor.2
42112     1   804     0  S       uwait   0xfffff8000bd99500  syz-executor.2
42111     1   804     0  S       uwait   0xfffff80005565900  syz-executor.2
41582     1   804     0  S       uwait   0xfffff80033e1eb00  syz-executor.2
41581     1   804     0  S       uwait   0xfffff80033e24f00  syz-executor.2
40614     1   861     0  S       uwait   0xfffff80033e02900  syz-executor.3
40613     1   861     0  S       uwait   0xfffff8000a70b900  syz-executor.3
40212     1   804     0  S       uwait   0xfffff80033e20500  syz-executor.2
40211     1   804     0  S       uwait   0xfffff8000bd04100  syz-executor.2
39538     1   784     0  S       uwait   0xfffff8000c3ce100  syz-executor.1
39537     1   784     0  S       uwait   0xfffff80033e20d00  syz-executor.1
39221     1   861     0  SV      uwait   0xfffff80074a6cd00  syz-executor.3
39211     1   784     0  SV      uwait   0xfffff80033e20300  syz-executor.1
39207     1   784     0  SV      uwait   0xfffff80033e1e300  syz-executor.1
37206     1   804     0  S       uwait   0xfffff80033e24700  syz-executor.2
37204     1   804     0  S       uwait   0xfffff80008a09300  syz-executor.2
36792     1   784     0  S       uwait   0xfffff80020334300  syz-executor.1
36789     1   784     0  S       uwait   0xfffff8000bd99d00  syz-executor.1
36234 36227   804     0  SV      uwait   0xfffff800075ccb00  syz-executor.2
36227     1   804     0  DV      ppwait  0xfffff8001b3224e8  syz-executor.2
35427 35425   861     0  SV      uwait   0xfffff80033e1ef00  syz-executor.3
35425     1   861     0  DV      ppwait  0xfffff8001b3dba20  syz-executor.3
35024     1   784     0  S       uwait   0xfffff8000c3ce300  syz-executor.1
34989     1   783     0  S       uwait   0xfffff80033e24300  syz-executor.0
34986     1   783     0  S       uwait   0xfffff8000bd04300  syz-executor.0
34979     1   804     0  S       uwait   0xfffff8000bd04900  syz-executor.2
34976     1   804     0  S       uwait   0xfffff8000bd99900  syz-executor.2
34605     1   784     0  S       uwait   0xfffff80033e1ed00  syz-executor.1
34599     1   784     0  S       uwait   0xfffff8000bd04700  syz-executor.1
34555     1   784     0  S       uwait   0xfffff8000bd04500  syz-executor.1
34552     1   784     0  S       uwait   0xfffff800075cc300  syz-executor.1
34260     1   783     0  S       uwait   0xfffff8000bd99700  syz-executor.0
34215     1   804     0  S       uwait   0xfffff800075ccd00  syz-executor.2
34212     1   804     0  S       uwait   0xfffff8000bd99100  syz-executor.2
33687     1   804     0  S       uwait   0xfffff8000bd04f00  syz-executor.2
33684     1   804     0  S       uwait   0xfffff8000a70bf00  syz-executor.2
33683     1   804     0  S       uwait   0xfffff8007c3a2900  syz-executor.2
33680     1   804     0  S       uwait   0xfffff8000a722300  syz-executor.2
33622     1   861     0  S       uwait   0xfffff8000a70bd00  syz-executor.3
33620     1   861     0  S       uwait   0xfffff8000bd04d00  syz-executor.3
33412     1 33404     0  S       uwait   0xfffff80008362b00  syz-executor.2
33411     1 33404     0  S       uwait   0xfffff80008a09b00  syz-executor.2
33409     1   804     0  S       uwait   0xfffff80005565f00  syz-executor.2
33405     1   804     0  S       uwait   0xfffff8000a722d00  syz-executor.2
33385     1   804     0  S       uwait   0xfffff8000bd99f00  syz-executor.2
33382     1   804     0  S       uwait   0xfffff8006c37d500  syz-executor.2
33381     1   804     0  S       uwait   0xfffff80020334100  syz-executor.2
33380     1   804     0  S       uwait   0xfffff8000a70b700  syz-executor.2
32775     1   784     0  S       uwait   0xfffff80005565700  syz-executor.1
32773     1   784     0  S       uwait   0xfffff8000c3cef00  syz-executor.1
32769     1   784     0  S       uwait   0xfffff80008362700  syz-executor.1
32766     1   784     0  S       uwait   0xfffff8000a722500  syz-executor.1
32761     1   784     0  S       uwait   0xfffff80020305700  syz-executor.1
32756     1   784     0  S       uwait   0xfffff80008a09100  syz-executor.1
32753     1   784     0  S       uwait   0xfffff800075ccf00  syz-executor.1
32748     1   784     0  S       uwait   0xfffff80020305d00  syz-executor.1
32739     1   784     0  S       uwait   0xfffff8000c3ced00  syz-executor.1
32735     1   784     0  S       uwait   0xfffff80008362f00  syz-executor.1
32710     1   804     0  S       uwait   0xfffff8000a70bb00  syz-executor.2
32709     1   804     0  S       uwait   0xfffff8000a722100  syz-executor.2
32223     1   784     0  S       uwait   0xfffff8000c3ce700  syz-executor.1
32216     1   784     0  S       uwait   0xfffff8000a722700  syz-executor.1
32181     1   804     0  S       uwait   0xfffff8000c3ceb00  syz-executor.2
32180     1   804     0  S       uwait   0xfffff80008362100  syz-executor.2
32069     1   784     0  S       uwait   0xfffff80008362d00  syz-executor.1
32065     1   784     0  S       uwait   0xfffff80003aee900  syz-executor.1
31973     1   784     0  S       uwait   0xfffff80008a09500  syz-executor.1
31972 31970   784     0  SV      uwait   0xfffff8007c398300  syz-executor.1
31971 31970   784     0  S       uwait   0xfffff8007c398100  syz-executor.1
31970     1   784     0  DV      ppwait  0xfffff800077b84e8  syz-executor.1
31834     1   784     0  S       uwait   0xfffff8000c3ce500  syz-executor.1
31826     1   784     0  S       uwait   0xfffff8000a722b00  syz-executor.1
31797     1   784     0  S       uwait   0xfffff80008362900  syz-executor.1
31796     1   784     0  S       uwait   0xfffff80005565300  syz-executor.1
31427     1   783     0  S       uwait   0xfffff8006c18cb00  syz-executor.0
31424     1   783     0  S       uwait   0xfffff80003fe4100  syz-executor.0
30953     1   861     0  S       uwait   0xfffff80003fe4b00  syz-executor.3
30952     1   861     0  S       uwait   0xfffff80005565100  syz-executor.3
30861     1   784     0  S       uwait   0xfffff8000a722f00  syz-executor.1
30860     1   784     0  S       uwait   0xfffff80008362500  syz-executor.1
30758     1   784     0  S       uwait   0xfffff80020334500  syz-executor.1
30757     1   784     0  S       uwait   0xfffff80008a09900  syz-executor.1
30754     1   784     0  S       uwait   0xfffff8007c3a2700  syz-executor.1
30752     1   784     0  S       uwait   0xfffff80020305100  syz-executor.1
30736     1   861     0  S       uwait   0xfffff8006c199f00  syz-executor.3
30735     1   861     0  S       uwait   0xfffff80008362300  syz-executor.3
29825     1   861     0  S       uwait   0xfffff8006c511b00  syz-executor.3
29823     1   861     0  S       uwait   0xfffff80008a09700  syz-executor.3
29607 29606   861     0  SV      uwait   0xfffff80003aeef00  syz-executor.3
29606     1   861     0  DV      ppwait  0xfffff8007c39ba20  syz-executor.3
28897     1   783     0  S       uwait   0xfffff80020305f00  syz-executor.0
28894     1   783     0  S       uwait   0xfffff80003aee300  syz-executor.0
28893     1   783     0  S       uwait   0xfffff8006c199100  syz-executor.0
28891     1   783     0  S       uwait   0xfffff80003fe4f00  syz-executor.0
28837     1   804     0  S       uwait   0xfffff80008a09f00  syz-executor.2
28836     1   804     0  S       uwait   0xfffff8007c3a2300  syz-executor.2
28835     1   783     0  S       uwait   0xfffff80005565b00  syz-executor.0
28833     1   783     0  S       uwait   0xfffff80020334b00  syz-executor.0
28717     1   783     0  S       uwait   0xfffff80003aee100  syz-executor.0
28714     1   783     0  S       uwait   0xfffff8007c3a2100  syz-executor.0
26606     1   804     0  S       uwait   0xfffff80005565d00  syz-executor.2
26601     1   804     0  S       uwait   0xfffff80020334700  syz-executor.2
25227     1   784     0  SV      uwait   0xfffff80003fe4700  syz-executor.1
25211     1   861     0  SV      uwait   0xfffff8006c511d00  syz-executor.3
25210     1   861     0  SV      uwait   0xfffff80020334d00  syz-executor.3
25109     1   784     0  S       uwait   0xfffff80003aee700  syz-executor.1
25104     1   784     0  S       uwait   0xfffff8006c199300  syz-executor.1
20376     1   783     0  S       uwait   0xfffff80005565500  syz-executor.0
20372     1   783     0  S       uwait   0xfffff80020305b00  syz-executor.0
20257     1   861     0  S       uwait   0xfffff80020305300  syz-executor.3
20254     1   861     0  S       uwait   0xfffff8006c37d100  syz-executor.3
20231     1   783     0  S       uwait   0xfffff8006c37d300  syz-executor.0
20230     1   783     0  S       uwait   0xfffff80020334f00  syz-executor.0
19499     1   783     0  S       uwait   0xfffff8007c3a2d00  syz-executor.0
19498     1   783     0  S       uwait   0xfffff8006c199500  syz-executor.0
19277     1   804     0  S       uwait   0xfffff80003fe4300  syz-executor.2
19274     1   804     0  S       uwait   0xfffff80003fe4500  syz-executor.2
18221     1 18211     0  S       uwait   0xfffff80003aee500  syz-executor.2
18220     1 18211     0  S       uwait   0xfffff80003fe4d00  syz-executor.2
18083     1   804     0  S       uwait   0xfffff8007c398700  syz-executor.2
18082     1   804     0  S       uwait   0xfffff8007c398b00  syz-executor.2
17887 17883   861     0  SV      uwait   0xfffff8006c199d00  syz-executor.3
17883     1   861     0  DV      ppwait  0xfffff80005501a20  syz-executor.3
17125     1   784     0  S       uwait   0xfffff8007c398f00  syz-executor.1
17124     1   784     0  S       uwait   0xfffff8007c3a2f00  syz-executor.1
17019 17018   784     0  SV      uwait   0xfffff8006c199b00  syz-executor.1
17018     1   784     0  DV      ppwait  0xfffff800055014e8  syz-executor.1
16998     1   804     0  S       uwait   0xfffff8006c511f00  syz-executor.2
16997     1   804     0  S       uwait   0xfffff8007c3a2500  syz-executor.2
16297     1   804     0  S       uwait   0xfffff8006c511300  syz-executor.2
16287     1   804     0  S       uwait   0xfffff8006c199700  syz-executor.2
16281     1   804     0  S       uwait   0xfffff8006c11ef00  syz-executor.2
16275     1   804     0  S       uwait   0xfffff8007c3a2b00  syz-executor.2
16269     1   804     0  S       uwait   0xfffff80003aeeb00  syz-executor.2
16263     1   804     0  S       uwait   0xfffff8007c398900  syz-executor.2
16254     1   804     0  S       uwait   0xfffff8006c199900  syz-executor.2
16247     1   804     0  S       uwait   0xfffff8006c18c100  syz-executor.2
15455 15452   804     0  SV      uwait   0xfffff80003aeed00  syz-executor.2
15452     1   804     0  DV      ppwait  0xfffff8006c5494e8  syz-executor.2
15180     1   804     0  S       uwait   0xfffff8006c11e100  syz-executor.2
15179     1   804     0  S       uwait   0xfffff8007c398500  syz-executor.2
13576     1   804     0  S       uwait   0xfffff8006c511900  syz-executor.2
13575     1   804     0  S       uwait   0xfffff8006c511700  syz-executor.2
13456     1   784     0  S       uwait   0xfffff8006c11e700  syz-executor.1
13454     1   784     0  S       uwait   0xfffff8006c37db00  syz-executor.1
13438     1   783     0  S       uwait   0xfffff8006c511500  syz-executor.0
13437     1   783     0  S       uwait   0xfffff8006c511100  syz-executor.0
13435     1   784     0  S       uwait   0xfffff8006c18cf00  syz-executor.1
13432     1   784     0  S       uwait   0xfffff8004de55f00  syz-executor.1
13426     1   784     0  S       uwait   0xfffff8006c11eb00  syz-executor.1
13424     1   784     0  S       umtxn   0xfffff8005cca5700  syz-executor.1
13413     1   784     0  S       uwait   0xfffff8006c11e900  syz-executor.1
13411     1   784     0  S       uwait   0xfffff80072361300  syz-executor.1
13401     1   784     0  S       uwait   0xfffff8005c5c1f00  syz-executor.1
13400     1   784     0  S       uwait   0xfffff8006c18c300  syz-executor.1
13392     1   784     0  S       uwait   0xfffff8005cca5b00  syz-executor.1
13391     1   784     0  S       uwait   0xfffff8006c37d900  syz-executor.1
13389     1   784     0  S       uwait   0xfffff8005cca5300  syz-executor.1
13388     1   784     0  S       uwait   0xfffff8005cca5d00  syz-executor.1
13346     1   804     0  S       uwait   0xfffff8005cba9f00  syz-executor.2
13345     1   804     0  S       uwait   0xfffff8006c11e500  syz-executor.2
13317     1   804     0  S       uwait   0xfffff8006c37d700  syz-executor.2
13312     1   804     0  S       uwait   0xfffff8006c11e300  syz-executor.2
13302     1   804     0  S       uwait   0xfffff8006c18c500  syz-executor.2
13301     1   804     0  S       uwait   0xfffff8006c37dd00  syz-executor.2
13252 13250 13250     0  SV      uwait   0xfffff8005cca5500  syz-executor.2
13250     1 13250     0  DVs     ppwait  0xfffff8006c284a20  syz-executor.2
13189     1   804     0  S       uwait   0xfffff8004de58300  syz-executor.2
13188     1   804     0  S       uwait   0xfffff80015de1300  syz-executor.2
13131 13124   783     0  SV      uwait   0xfffff8005cca5f00  syz-executor.0
13124     1   783     0  RV                                  syz-executor.0
13073     1   783     0  S       uwait   0xfffff8004de55900  syz-executor.0
13069     1   783     0  S       uwait   0xfffff80072361500  syz-executor.0
13033     1   861     0  S       uwait   0xfffff8006c11ed00  syz-executor.3
13031     1   861     0  S       umtxn   0xfffff8004dccdd00  syz-executor.3
13026     1   861     0  S       uwait   0xfffff8004dc12500  syz-executor.3
12972     1   861     0  S       uwait   0xfffff80072361900  syz-executor.3
12970     1   861     0  S       uwait   0xfffff8006c18c700  syz-executor.3
12929     1   783     0  S       uwait   0xfffff80004d60d00  syz-executor.0
12928     1   783     0  S       uwait   0xfffff80072431f00  syz-executor.0
12917     1   861     0  S       uwait   0xfffff8004de61100  syz-executor.3
12914     1   861     0  S       uwait   0xfffff8006c18cd00  syz-executor.3
12842     1   861     0  S       uwait   0xfffff80072300f00  syz-executor.3
12839     1   861     0  S       uwait   0xfffff80072431700  syz-executor.3
12810     1   783     0  S       uwait   0xfffff80072361100  syz-executor.0
12806     1   783     0  S       uwait   0xfffff80072300b00  syz-executor.0
12797     1   804     0  S       uwait   0xfffff8004dc12100  syz-executor.2
12795     1   804     0  S       uwait   0xfffff8004dde8300  syz-executor.2
12794     1   804     0  S       uwait   0xfffff8005c5c1700  syz-executor.2
12791     1   804     0  S       uwait   0xfffff80015d91100  syz-executor.2
12788     1 12779     0  S       uwait   0xfffff8004dccd900  syz-executor.0
12787     1 12779     0  S       uwait   0xfffff8004de58500  syz-executor.0
12768     1   861     0  S       uwait   0xfffff80015d9cf00  syz-executor.3
12767     1   861     0  S       uwait   0xfffff8005c5c1900  syz-executor.3
12370     0     0     0  DL      (threaded)                  [zfskern]
109888                   D       t->zthr 0xfffff8006c088748  [arc_evict]
112730                   D       t->zthr 0xfffff8006c088548  [arc_reap]
112731                   D       -       0xfffff80072315500  [dbu_evict]
112732                   D       dbuf_ev 0xffffffff839c3fd8  [dbuf_evict_thread]
112733                   D       -       0xfffff80072310500  [z_vdev_file_0]
112734                   D       -       0xfffff80072310500  [z_vdev_file_1]
112735                   D       -       0xfffff80072310500  [z_vdev_file_2]
112736                   D       -       0xfffff80072310500  [z_vdev_file_3]
112737                   D       -       0xfffff80072310500  [z_vdev_file_4]
112738                   D       -       0xfffff80072310500  [z_vdev_file_5]
112739                   D       -       0xfffff80072310500  [z_vdev_file_6]
112740                   D       -       0xfffff80072310500  [z_vdev_file_7]
112741                   D       -       0xfffff80072310500  [z_vdev_file_8]
112742                   D       -       0xfffff80072310500  [z_vdev_file_9]
112743                   D       -       0xfffff80072310500  [z_vdev_file_10]
112744                   D       -       0xfffff80072310500  [z_vdev_file_11]
112745                   D       -       0xfffff80072310500  [z_vdev_file_12]
112746                   D       -       0xfffff80072310500  [z_vdev_file_13]
112747                   D       -       0xfffff80072310500  [z_vdev_file_14]
112748                   D       -       0xfffff80072310500  [z_vdev_file_15]
112749                   D       l2arc_f 0xffffffff839829b8  [l2arc_feed_thread]
112750                   D       -       0xfffff80072507500  [zfsvfs]
112751                   S       zevent_ 0xffffffff839c4560  [sysevent]
10767     1 10767    65  Ss      select  0xfffff8005c726a40  dhclient
 7919     1   804     0  S       uwait   0xfffff80072300700  syz-executor.2
 7913     1   804     0  S       uwait   0xfffff80072431900  syz-executor.2
 6542     1   783     0  S       uwait   0xfffff8004de61b00  syz-executor.0
 6531     1   783     0  S       uwait   0xfffff80072300100  syz-executor.0
 6521     1   783     0  S       uwait   0xfffff8004dc12900  syz-executor.0
 6512     1   783     0  S       uwait   0xfffff8004de61900  syz-executor.0
 6503     1   783     0  S       uwait   0xfffff8004de61500  syz-executor.0
 6495     1   783     0  S       uwait   0xfffff8004dde8d00  syz-executor.0
 6485     1   783     0  S       uwait   0xfffff8004de61300  syz-executor.0
 6475     1   783     0  S       uwait   0xfffff8004dde8900  syz-executor.0
 6470     1   783     0  S       uwait   0xfffff8004de55d00  syz-executor.0
 6456     1   783     0  S       uwait   0xfffff80015ddeb00  syz-executor.0
 6446     1   783     0  S       uwait   0xfffff80015de5b00  syz-executor.0
 5490     1   861     0  S       uwait   0xfffff80072300300  syz-executor.3
 5487     1   861     0  S       uwait   0xfffff8004de61700  syz-executor.3
 5453     1  5453     0  Ss      select  0xfffff80074ddc740  dhclient
 5450     1  5450     0  Ss      select  0xfffff80074ddca40  dhclient
 5431     1  5431    65  Ss      select  0xfffff8007423ebc0  dhclient
 4417     1   783     0  S       uwait   0xfffff8004dccd500  syz-executor.0
 4416     1   783     0  S       uwait   0xfffff8004dde8b00  syz-executor.0
 4397     1  4397     0  Ss      select  0xfffff800720ee140  dhclient
 4394     1  4394     0  Ss      select  0xfffff8004df6c2c0  dhclient
 4375     1  4375    65  Ss      select  0xfffff80072583140  dhclient
 3560     1   784     0  S       uwait   0xfffff8004de55500  syz-executor.1
 3559     1   784     0  S       uwait   0xfffff80015de0700  syz-executor.1
 3557     1   784     0  S       uwait   0xfffff8004de55b00  syz-executor.1
 3553     1   784     0  S       uwait   0xfffff8004de55100  syz-executor.1
 2246     1   784     0  S       uwait   0xfffff8004dccdb00  syz-executor.1
 2244     1   784     0  S       uwait   0xfffff8004de58f00  syz-executor.1
 2235     1  2235     0  Ss      select  0xfffff800721eaa40  dhclient
 2226     1  2226     0  Ss      select  0xfffff8004ddeb2c0  dhclient
 2204     1  2204    65  Ss      select  0xfffff800721ea5c0  dhclient
 1892  1891   784     0  SV      uwait   0xfffff8004dccd700  syz-executor.1
 1891     1   784     0  DV      ppwait  0xfffff80072022f58  syz-executor.1
 1120     1  1120     0  Ss      select  0xfffff8004ddeb8c0  dhclient
 1117     1  1117     0  Ss      select  0xfffff8004dd9c5c0  dhclient
  861   780   861     0  Rs                                  syz-executor.3
  841     1   784     0  S       uwait   0xfffff80015de5f00  syz-executor.1
  837     1   784     0  S       uwait   0xfffff80015d9c300  syz-executor.1
  804   780   804     0  Rs                                  syz-executor.2
  784   780   784     0  Rs                                  syz-executor.1
  783   780   783     0  Rs                                  syz-executor.0
  780   778   778     0  R       (threaded)                  syz-fuzzer
100116                   S       uwait   0xfffff80015d9c100  syz-fuzzer
100119                   RunQ                                syz-fuzzer
100120                   S       uwait   0xfffff80004d60500  syz-fuzzer
100121                   S       kqread  0xfffff80015de5100  syz-fuzzer
100122                   S       uwait   0xfffff80015dde700  syz-fuzzer
100123                   S       uwait   0xfffff80015dde900  syz-fuzzer
100124                   RunQ                                syz-fuzzer
100126                   S       uwait   0xfffff80015de1b00  syz-fuzzer
100140                   S       uwait   0xfffff80015dde500  syz-fuzzer
145143                   S       uwait   0xfffff80033e24d00  syz-fuzzer
  778   776   778     0  Ss      pause   0xfffff8004d4bc5e8  csh
  776   694   776     0  Ss      select  0xfffff80015de2bc0  sshd
  760     1   760     0  Ss+     ttyin   0xfffff80015465cb0  getty
  759     1   759     0  Ss+     ttyin   0xfffff80015a8a4b0  getty
  758     1   758     0  Ss+     ttyin   0xfffff80015a8acb0  getty
  757     1   757     0  Ss+     ttyin   0xfffff80015a8e4b0  getty
  756     1   756     0  Ss+     ttyin   0xfffff80015a8ecb0  getty
  755     1   755     0  Ss+     ttyin   0xfffff80015a924b0  getty
  754     1   754     0  Ss+     ttyin   0xfffff80015a92cb0  getty
  753     1   753     0  Ss+     ttyin   0xfffff80015a954b0  getty
  752     1   752     0  Ss+     ttyin   0xfffff80015a95cb0  getty
  698     1   698     0  Ss      nanslp  0xffffffff8273c560  cron
  694     1   694     0  Ss      select  0xfffff80015df2a40  sshd
  507     1   507     0  Ss      select  0xfffff80015eeb740  syslogd
  436     1   436     0  Ss      select  0xfffff80015de4ec0  devd
  435     1   435    65  Ss      select  0xfffff80015ec48c0  dhclient
  350     1   350     0  Ss      select  0xfffff80015f2f140  dhclient
  347     1   347     0  Ss      select  0xfffff8004d316bc0  dhclient
   23     0     0     0  DL      vlruwt  0xfffff80015ca7538  [vnlru]
   22     0     0     0  DL      syncer  0xffffffff8282b9d0  [syncer]
   21     0     0     0  DL      (threaded)                  [bufdaemon]
100081                   D       qsleep  0xffffffff8282aa80  [bufdaemon]
100086                   D       -       0xffffffff8220ae80  [bufspacedaemon-0]
100098                   D       sdflush 0xfffff80004dfc4e8  [/ worker]
   20     0     0     0  DL      psleep  0xffffffff828528c8  [vmdaemon]
   19     0     0     0  DL      (threaded)                  [pagedaemon]
100079                   D       psleep  0xffffffff82846d38  [dom0]
100087                   D       launds  0xffffffff82846d44  [laundry: dom0]
100088                   D       umarcl  0xffffffff815cb6d0  [uma]
   18     0     0     0  DL      -       0xffffffff82570908  [rand_harvestq]
   17     0     0     0  DL      waiting 0xffffffff830da828  [sctp_iterator]
   16     0     0     0  DL      pftm    0xffffffff82d783c0  [pf purge]
   15     0     0     0  DL      -       0xffffffff828280dc  [soaiod4]
    9     0     0     0  DL      -       0xffffffff828280dc  [soaiod3]
    8     0     0     0  DL      -       0xffffffff828280dc  [soaiod2]
    7     0     0     0  DL      -       0xffffffff828280dc  [soaiod1]
    6     0     0     0  DL      (threaded)                  [cam]
100044                   D       -       0xffffffff82447dc0  [doneq0]
100045                   D       -       0xffffffff82447d40  [async]
100078                   D       -       0xffffffff82447c10  [scanner]
   14     0     0     0  DL      seqstat 0xfffff80004dcbc88  [sequencer 00]
    5     0     0     0  DL      crypto_ 0xfffff80004d99d80  [crypto returns 1]
    4     0     0     0  DL      crypto_ 0xfffff80004d99d30  [crypto returns 0]
    3     0     0     0  DL      crypto_ 0xffffffff82844220  [crypto]
   13     0     0     0  DL      (threaded)                  [geom]
100035                   D       -       0xffffffff8271bda0  [g_event]
100036                   D       -       0xffffffff8271bda8  [g_up]
100037                   D       -       0xffffffff8271bdb0  [g_down]
    2     0     0     0  DL      (threaded)                  [KTLS]
100028                   D       -       0xfffff80004c39d00  [thr_0]
100029                   D       -       0xfffff80004c39d80  [thr_1]
   12     0     0     0  RL      (threaded)                  [intr]
100012                   I                                   [swi5: fast taskq]
100015                   I                                   [swi6: task queue]
100017                   I                                   [swi6: Giant taskq]
100030                   I                                   [swi4: clock (0)]
100031                   I                                   [swi4: clock (1)]
100032                   Run     CPU 0                       [swi1: netisr 0]
100033                   I                                   [swi3: vm]
100046                   I                                   [irq24: virtio_pci0]
100047                   I                                   [irq25: virtio_pci0]
100048                   I                                   [irq26: virtio_pci0]
100049                   I                                   [irq27: virtio_pci0]
100050                   I                                   [irq28: virtio_pci1]
100051                   I                                   [irq29: virtio_pci1]
100052                   I                                   [irq30: virtio_pci1]
100053                   I                                   [irq31: virtio_pci1]
100054                   I                                   [irq32: virtio_pci1]
100059                   I                                   [irq10: virtio_pci2]
100061                   I                                   [irq1: atkbd0]
100062                   I                                   [irq12: psm0]
100063                   I                                   [swi0: uart uart++]
100071                   I                                   [swi1: pf send]
100084                   I                                   [swi1: hpts]
100085                   I                                   [swi1: hpts]
   11     0     0     0  RL      (threaded)                  [idle]
100003                   CanRun                              [idle: cpu0]
100004                   CanRun                              [idle: cpu1]
    1     0     1     0  SLs     wait    0xfffff80004bc7538  [init]
   10     0     0     0  DL      audit_w 0xffffffff82844730  [audit]
    0     0     0     0  RLs     (threaded)                  [kernel]
100000                   D       swapin  0xffffffff8271c330  [swapper]
100005                   D       -       0xfffff80004c63d00  [if_config_tqg_0]
100006                   D       -       0xfffff80004c63900  [softirq_0]
100007                   D       -       0xfffff80004c63500  [softirq_1]
100008                   D       -       0xfffff80004c63100  [if_io_tqg_0]
100009                   D       -       0xfffff80004c61d00  [if_io_tqg_1]
100010                   D       -       0xfffff80004c5fd00  [in6m_free taskq]
100011                   D       -       0xfffff80004c5f900  [aiod_kick taskq]
100013                   D       -       0xfffff80004c5f100  [kqueue_ctx taskq]
100014                   D       -       0xfffff80004c58d00  [pci_hp taskq]
100016                   D       -       0xfffff80004c58500  [inm_free taskq]
100018                   D       -       0xfffff80004c53d00  [linuxkpi_irq_wq]
100019                   Run     CPU 1                       [thread taskq]
100020                   D       -       0xfffff80004c53500  [linuxkpi_short_wq_0]
100021                   D       -       0xfffff80004c53500  [linuxkpi_short_wq_1]
100022                   D       -       0xfffff80004c53500  [linuxkpi_short_wq_2]
100023                   D       -       0xfffff80004c53500  [linuxkpi_short_wq_3]
100024                   D       -       0xfffff80004c53100  [linuxkpi_long_wq_0]
100025                   D       -       0xfffff80004c53100  [linuxkpi_long_wq_1]
100026                   D       -       0xfffff80004c53100  [linuxkpi_long_wq_2]
100027                   D       -       0xfffff80004c53100  [linuxkpi_long_wq_3]
100034                   D       -       0xfffff80004c39900  [firmware taskq]
100038                   D       -       0xfffff80004c39500  [crypto_0]
100039                   D       -       0xfffff80004c39500  [crypto_1]
100055                   D       -       0xfffff800153dc900  [vtnet0 rxq 0]
100056                   D       -       0xfffff800153dc500  [vtnet0 txq 0]
100057                   D       -       0xfffff800153dc100  [vtnet0 rxq 1]
100058                   D       -       0xfffff800153c6d00  [vtnet0 txq 1]
100060                   D       vtbslp  0xfffff80015429500  [virtio_balloon]
100064                   D       -       0xfffff800153c6900  [mca taskq]
100069                   D       -       0xffffffff81e20451  [deadlkres]
100074                   D       -       0xfffff80015a9bd00  [acpi_task_0]
100075                   D       -       0xfffff80015a9bd00  [acpi_task_1]
100076                   D       -       0xfffff80015a9bd00  [acpi_task_2]
100077                   D       -       0xfffff80004c39100  [CAM taskq]
112724                   D       -       0xfffff80072543100  [system_taskq_0]
112725                   D       -       0xfffff80072543100  [system_taskq_1]
112726                   D       -       0xfffff80072315100  [system_delay_taskq_]
112727                   D       -       0xfffff80072315100  [system_delay_taskq_]
112728                   D       -       0xfffff80072593100  [arc_prune_0]
112729                   D       -       0xfffff80072593100  [arc_prune_1]
db> show all locks
Process 50096 (syz-executor.2) thread 0xfffffe009f1ce000 (160325)
exclusive rw vm object (vm object) r = 0 (0xfffff80061403630) locked @ /syzkaller/managers/main/kernel/sys/vm/vm_fault.c:1341
shared sx vm map (user) (vm map (user)) r = 0 (0xfffffe009f1c9440) locked @ /syzkaller/managers/main/kernel/sys/vm/vm_map.c:4935
Process 12 (intr) thread 0xfffffe00042bd020 (100032)
exclusive sleep mutex so_snd (so_snd) r = 0 (0xfffff8005cafa610) locked @ /syzkaller/managers/main/kernel/sys/modules/tcp/rack/../../../netinet/tcp_stacks/rack.c:10625
exclusive rw tcpinp (tcpinp) r = 0 (0xfffff800744d47c0) locked @ /syzkaller/managers/main/kernel/sys/netinet/in_pcb.c:2524
Process 0 (kernel) thread 0xfffffe00042b2000 (100019)
exclusive sleep mutex kernel arena domain (kernel arena domain) r = 0 (0xfffff8000400a000) locked @ /syzkaller/managers/main/kernel/sys/kern/subr_vmem.c:789
exclusive sleep mutex vmem list lock (vmem list lock) r = 0 (0xffffffff82202b00) locked @ /syzkaller/managers/main/kernel/sys/kern/subr_vmem.c:784
db> show malloc
              Type        InUse        MemUse     Requests
         sysctloid        46272        17352K        46339
           pf_hash            5        11560K            5
          filedesc          538         8693K        98079
            devbuf         4216         6982K         4245
            linker          388         6144K          587
           solaris          148         4440K          218
         sctp_stro           47         4194K         7351
          tcp_hpts            5         3219K            5
           subproc          660         2942K        50446
              kobj          328         2624K          488
             vtbuf           24         2064K           46
               pcb          539         2027K       138384
          vfscache            3         1035K            3
            newblk            5         1034K        51548
            acpica         1674          649K        55230
          inodedep           95          614K        49974
           kdtrace          766          563K       110433
           callout            2          528K            2
         ufs_quota            1          520K            1
          vfs_hash            1          520K            1
              intr            4          480K            4
              umtx          924          405K          924
               bus          990          378K         3499
               BPF           46          337K           46
            DEVFS1          108          216K          125
         sctp_atcl          196          196K        67168
            module          517          194K          517
         vnet_data            1          176K            1
           tidhash            3          164K            3
           pagedep            9          140K        48805
               UMA          365          137K          365
        tfo_ccache            1          136K            1
            kqueue          328          131K        50297
           pwddesc          324          122K        50650
               sem            4          120K            4
               osd          313          118K        35463
       inpcbpolicy          309          116K        39672
              vmem            3          112K            7
         sctp_atky          243           92K        76585
         sctp_timw          182           91K          182
     audit_evclass          236           89K          294
          mtx_pool            2           80K            2
              temp           35           77K         4537
          syncache            1           76K            1
          routetbl          139           76K         2900
         sctp_athm          196           74K        69069
               msg            4           68K            4
            ifaddr           70           66K           72
          acpitask            1           64K            1
       ddb_capture            1           64K            1
            DEVFS3          127           64K          137
       ether_multi          152           57K          162
        gtaskqueue           18           57K           18
        DEVFS_RULE           56           54K           56
              kenv           95           52K           95
      eventhandler          136           51K          136
         taskqueue           78           47K           78
            DEVFSP          118           45K        22016
            dirrem           86           43K        49798
           lltable           49           43K          430
           ithread           99           43K           99
              rman           84           42K          425
            KTRACE          100           38K          100
          sctp_map           94           36K        14702
        kstat_data           17           34K           17
              proc            3           34K            3
         ufs_mount            5           34K            6
            bus-sc           33           34K         1710
           devstat            4           33K            4
          freefile           86           33K        49643
           pf_rule            8           32K          373
         hostcache            1           32K            1
               tty           16           32K           16
               shm            1           32K           31
              cred           31           31K         1604
              GEOM           60           29K          493
         in6_multi           65           29K           65
            kbdmux            6           28K            6
            plimit           25           25K         1321
             ifnet            7           25K            7
          kcovinfo           64           24K           68
             selfd           59           23K       675736
         CAM queue            5           21K         1528
         pfs_nodes           20           20K           20
         proc-args           52           20K          771
             xform           52           20K        33659
           session           38           19K           88
              UART           12           18K           12
         bmsafemap            2           17K        49941
              ksem            1           16K          474
               rpc            2           16K            2
             shmfd            1           16K          219
       pfs_vncache            1           16K            1
       ufs_dirhash           24           14K           24
             lockf           36           14K          185
            sglist            5           13K            5
            crypto           21           13K         8888
               MCA           32           12K           32
           CAM DEV            3           12K          510
                vt           11           11K           11
           acpisem           28           11K           28
           CAM XPT           22           11K          543
          pf_ifnet           13           10K          440
            Unitno           25           10K           99
           uidinfo            3            9K           37
          pf_table            2            8K          620
           memdesc            1            8K            1
         ipsec-saq            2            8K            2
             evdev            4            8K            4
            select           21            8K          235
           acpidev           20            8K           20
             hhook           15            8K           17
           tcp_fsb           18            7K         7055
             mount           16            7K          108
       fpukern_ctx            3            6K            3
          terminal           11            6K           11
            ip6ndp           12            6K           14
          sctp_ifa           13            5K           14
       ipsecpolicy            2            5K            2
 encap_export_host           12            5K           12
             clone            9            5K            9
        local_apic            1            4K            1
           io_apic            1            4K            1
            sahead            1            4K            1
          secasvar            1            4K            1
             nhops            6            4K            8
          pci_link           10            4K           10
               msi            9            4K            9
             DEVFS            9            4K           10
               tun            7            4K            7
             ipsec            3            3K            3
             mkdir            8            3K        97560
          in_multi            6            3K            8
       vnodemarker            3            3K          309
            prison            8            3K            8
          nexusdev            7            3K            7
            feeder            7            3K            7
             sigio            7            3K           24
          procdesc            5            3K           18
          freework            5            3K        48691
          sctp_ifn            6            3K           14
           pf_osfp            6            3K            6
               mld            6            3K            6
              igmp            6            3K            6
         toponodes            6            3K            6
            isadev            6            3K            6
           softdep            1            2K            1
          freeblks            4            2K        48654
      NFSD session            1            2K            1
        cubic data            5            2K          423
             linux            5            2K            6
        CAM periph            4            2K          271
          filecaps            5            2K          185
          cdg data            4            2K          183
         newdirblk            4            2K        48780
            diradd            4            2K        49840
              pfil            4            2K            4
          CAM path            4            2K         1034
            soname            4            2K        73751
filedesc_to_leader            4            2K            6
           tcpfunc            3            2K            3
        loginclass            3            2K            6
         sctp_stri            1            1K         3773
    chacha20random            1            1K            1
            vnodes            1            1K            7
           CAM SIM            2            1K            2
              ktls            1            1K            1
              cdev            2            1K            2
             taskq            2            1K            2
        aesni_data            2            1K            2
         htcp data            2            1K          429
              cpus            2            1K            2
          atkbddev            2            1K            2
     CAM dev queue            2            1K            2
       ip6_msource            2            1K            2
           entropy            2            1K           49
      NFSD lckfile            1            1K            1
     NFSD V4client            1            1K            1
            ip6opt            1            1K          395
          pmchooks            1            1K            1
          sctp_vrf            1            1K            1
        vegas data            1            1K          286
            apmdev            1            1K            1
 CAM I/O Scheduler            1            1K            1
    vnet_data_free            1            1K            1
              vnet            1            1K            1
           Per-cpu            1            1K            1
          p1003.1b            1            1K            1
          acpiintr            1            1K            1
               pmc            1            1K            1
        ext2_mount            0            0K            0
         ext2_node            0            0K            0
      ext2_extents            0            0K            0
         sfs_nodes            0            0K            0
        zones_data            0            0K            0
        sctp_mcore            0            0K            0
        sctp_socko            0            0K        29987
         sctp_iter            0            0K           15
         sctp_mvrf            0            0K            0
         sctp_cpal            0            0K            4
         sctp_cmsg            0            0K            0
         sctp_stre            0            0K            0
         sctp_athi            0            0K            0
         sctp_a_it            0            0K           11
         sctp_aadr            0            0K            5
          chd data            0            0K          172
        dctcp data            0            0K          115
            mqdata            0            0K            0
            tcp_do            0            0K            0
           pf_altq            0            0K            0
           pf_temp            0            0K            0
       NFSD V4lock            0            0K            0
        madt_table            0            0K            2
          smartpqi            0            0K            0
      NFSD V4state            0            0K            0
     NFSD srvcache            0            0K            0
       msdosfs_fat            0            0K            0
     msdosfs_mount            0            0K            0
      msdosfs_node            0            0K            0
              iavf            0            0K            0
               ixl            0            0K            0
            DEVFS4            0            0K            0
            DEVFS2            0            0K            0
            gntdev            0            0K            0
       privcmd_dev            0            0K            0
        ice-resmgr            0            0K            0
         ice-osdep            0            0K            0
               ice            0            0K            0
             axgbe            0            0K            0
        evtchn_dev            0            0K            0
          xenstore            0            0K            0
         ciss_data            0            0K            0
         BACKLIGHT            0            0K            0
               xnb            0            0K            0
              xbbd            0            0K            0
               xbd            0            0K            0
           Balloon            0            0K            0
          sysmouse            0            0K            0
            vtfont            0            0K            0
          xen_intr            0            0K            0
           xen_hvm            0            0K            0
         legacydrv            0            0K            0
            qpidrv            0            0K            0
           ath_hal            0            0K            0
            athdev            0            0K            0
      dmar_idpgtbl            0            0K            0
          dmar_dom            0            0K            0
          dmar_ctx            0            0K            0
           ata_pci            0            0K            0
           ata_dma            0            0K            0
       ata_generic            0            0K            0
              isci            0            0K            0
      iommu_dmamap            0            0K            0
               amr            0            0K            0
     hyperv_socket            0            0K            0
           bxe_ilt            0            0K            0
            xenbus            0            0K            0
            pvscsi            0            0K            0
           scsi_da            0            0K           69
     vm_fictitious            0            0K            0
            ata_da            0            0K            0
           scsi_ch            0            0K            0
           scsi_cd            0            0K            0
       AHCI driver            0            0K            0
            USBdev            0            0K            0
               USB            0            0K            0
               agp            0            0K            0
           nvme_da            0            0K            0
           UMAHash            0            0K            0
           acpipwr            0            0K            0
         acpi_perf            0            0K            0
         vm_pgdata            0            0K            0
           jblocks            0            0K            0
          savedino            0            0K        40804
          sentinel            0            0K            0
            jfsync            0            0K            0
            jtrunc            0            0K            0
             sbdep            0            0K          154
           jsegdep            0            0K            0
              jseg            0            0K            0
         jfreefrag            0            0K            0
          jfreeblk            0            0K            0
           jnewblk            0            0K            0
            jmvref            0            0K            0
           jremref            0            0K            0
           jaddref            0            0K            0
           freedep            0            0K            0
          freefrag            0            0K            7
        allocindir            0            0K            0
          indirdep            0            0K           19
       allocdirect            0            0K            0
          ufs_trim            0            0K            0
           mactemp            0            0K            0
     audit_trigger            0            0K            0
 audit_pipe_presel            0            0K            0
     audit_pipeent            0            0K            0
        audit_pipe            0            0K            0
      audit_evname            0            0K            0
         audit_bsm            0            0K            0
      audit_gidset            0            0K            0
        audit_text            0            0K            0
        audit_path            0            0K            0
        audit_data            0            0K            0
        audit_cred            0            0K            0
            twsbuf            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
         MLX5E_TLS            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
            MLX5EN            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
          MLX5DUMP            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
          seq_file            0            0K            0
             radix            0            0K            0
               idr            0            0K            0
            lkpifw            0            0K            0
               NLM            0            0K            0
    ipsec-spdcache            0            0K            0
         ipsec-reg            0            0K            0
        ipsec-misc            0            0K            0
      ipsecrequest            0            0K            0
      ip6_moptions            0            0K            4
       in6_mfilter            0            0K            6
             frag6            0            0K           17
            tcplog            0            0K            0
        tcp_hwpace            0            0K            0
      twe_commands            0            0K            0
               LRO            0            0K            0
      newreno data            0            0K           24
        ip_msource            0            0K            0
       ip_moptions            0            0K           28
        in_mfilter            0            0K            2
              ipid            0            0K            0
         80211scan            0            0K            0
      80211ratectl            0            0K            0
        80211power            0            0K            0
       80211nodeie            0            0K            0
         80211node            0            0K            0
      80211mesh_gt            0            0K            0
      80211mesh_rt            0            0K            0
         80211perr            0            0K            0
         80211prep            0            0K            0
         80211preq            0            0K            0
          80211dfs            0            0K            0
       80211crypto            0            0K            0
          80211vap            0            0K            0
             iflib            0            0K            0
              vlan            0            0K            0
               gif            0            0K            0
           ifdescr            0            0K            0
              zlib            0            0K            0
           fadvise            0            0K           17
           VN POLL            0            0K            0
      twa_commands            0            0K            0
            statfs            0            0K        49161
     namei_tracker            0            0K           25
       export_host            0            0K            0
        cl_savebuf            0            0K           75
       tcp_log_dev            0            0K         1477
      midi buffers            0            0K            0
             mixer            0            0K            0
              ac97            0            0K            0
             hdacc            0            0K            0
              hdac            0            0K            0
              hdaa            0            0K            0
         acpicmbat            0            0K            0
       SIIS driver            0            0K            0
           CAM CCB            0            0K        12442
               PUC            0            0K            0
          ppbusdev            0            0K            0
agtiapi_MemAlloc malloc            0            0K            0
    osti_cacheable            0            0K            0
          tempbuff            0            0K            0
            biobuf            0            0K            0
              aios            0            0K            0
               lio            0            0K            0
               acl            0            0K            0
          tempbuff            0            0K            0
          mbuf_tag            0            0K          296
ag_tgt_map_t malloc            0            0K            0
ag_slr_map_t malloc            0            0K            0
lDevFlags * malloc            0            0K            0
tiDeviceHandle_t * malloc            0            0K            0
ag_portal_data_t malloc            0            0K            0
ag_device_t malloc            0            0K            0
     STLock malloc            0            0K            0
          CCB List            0            0K            0
            sr_iov            0            0K            0
               OCS            0            0K            0
               OCS            0            0K            0
              nvme            0            0K            0
               nvd            0            0K            0
            netmap            0            0K            0
            mwldev            0            0K            0
        MVS driver            0            0K            0
     CAM ccb queue            0            0K            0
          mrsasbuf            0            0K            0
          mpt_user            0            0K            0
          mps_user            0            0K            0
              accf            0            0K            0
               pts            0            0K            0
               iov            0            0K        38255
          ioctlops            0            0K         4236
           eventfd            0            0K            0
           Witness            0            0K            0
             stack            0            0K            0
            MPSSAS            0            0K            0
               mps            0            0K            0
          mpr_user            0            0K            0
            MPRSAS            0            0K            0
               mpr            0            0K            0
            mfibuf            0            0K            0
              sbuf            0            0K          294
        md_sectors            0            0K            0
          firmware            0            0K            0
        compressor            0            0K            0
           md_disk            0            0K            0
              SWAP            0            0K            0
           malodev            0            0K            0
               LED            0            0K            0
         sysctltmp            0            0K         1029
            sysctl            0            0K            3
              ekcd            0            0K            0
            dumper            0            0K            0
          sendfile            0            0K            0
              rctl            0            0K            0
          ix_sriov            0            0K            0
        aacraidcam            0            0K            0
       aacraid_buf            0            0K            0
                ix            0            0K            0
            ipsbuf            0            0K            0
             cache            0            0K            0
            iirbuf            0            0K            0
      prison_racct            0            0K            0
       Fail Points            0            0K            0
               pwd            0            0K            0
       tty console            0            0K            0
            aaccam            0            0K            0
            aacbuf            0            0K            0
              zstd            0            0K            0
            XZ_DEC            0            0K            0
            nvlist            0            0K            0
          SCSI ENC            0            0K            0
           SCSI sa            0            0K            0
         scsi_pass            0            0K            0
        isofs_node            0            0K            0
       isofs_mount            0            0K            0
     tr_raid5_data            0            0K            0
    tr_raid1e_data            0            0K            0
     tr_raid1_data            0            0K            0
     tr_raid0_data            0            0K            0
    tr_concat_data            0            0K            0
       md_sii_data            0            0K            0
   md_promise_data            0            0K            0
    md_nvidia_data            0            0K            0
   md_jmicron_data            0            0K            0
     md_intel_data            0            0K            0
       md_ddf_data            0            0K            0
         raid_data            0            0K           72
     geom_flashmap            0            0K            0
         tmpfs dir            0            0K            0
        tmpfs name            0            0K            0
       tmpfs mount            0            0K            0
           NFS FHA            0            0K            0
         newnfsmnt            0            0K            0
  newnfsclient_req            0            0K            0
   NFSCL layrecall            0            0K            0
     NFSCL session            0            0K            0
     NFSCL sockreq            0            0K            0
     NFSCL devinfo            0            0K            0
     NFSCL flayout            0            0K            0
      NFSCL layout            0            0K            0
     NFSD rollback            0            0K            0
      NFSCL diroff            0            0K            0
       NEWdirectio            0            0K            0
        NEWNFSnode            0            0K            0
         NFSCL lck            0            0K            0
      NFSCL lckown            0            0K            0
      NFSCL client            0            0K            0
       NFSCL deleg            0            0K            0
        NFSCL open            0            0K            0
       NFSCL owner            0            0K            0
            NFS fh            0            0K            0
           NFS req            0            0K            0
     NFSD usrgroup            0            0K            0
       NFSD string            0            0K            0
db> show uma
              Zone   Size    Used    Free    Requests  Sleeps  Bucket  Total Mem    XFree
   mbuf_jumbo_page   4096    8350   19815     1537001       0     254  115363840        0
        malloc-384    384   47503     897      244642       0      30   18585600        0
    ertt_txseginfo     40  422106   10982     4680809       0     254   17323520        0
      malloc-32768  32768     271       4       49186       0       1    9011200        0
              mbuf    256    9240   21391     7602909       0     254    7841536        0
           tcp_log    416     131   14953      171417       0     254    6274944        0
       UMA Slabs 0    112   34529    5932       41149       0     126    4531632        0
       malloc-1024   1024    4160      28        6043       0      16    4288512        0
       malloc-1024   1024     170    3326       51358       0      16    3579904        0
        RADIX NODE    144   23979     854     1294649       0      64    3575952        0
       malloc-8192   8192     337       4       50148       0       1    2793472        0
       malloc-8192   8192     151     183       57903       0       1    2736128        0
       malloc-8192   8192     329       2         953       0       1    2711552        0
              pbuf   2624       0     941           0       0       2    2469184        0
       malloc-1024   1024       2    2126       40822       0      16    2179072        0
        malloc-384    384    2434    3086      404009       0      30    2119680        0
      mbuf_cluster   2048     959       1         959       0     254    1966080        0
          BUF TRIE    144     387   13081        9234       0      62    1939392        0
        malloc-512    512     152    3416      151597       0      30    1826816        0
         MAP ENTRY     96   15093     237     2714047       0     126    1471680        0
         VM OBJECT    264    4532      58      714741       0      30    1211760        0
        malloc-384    384    2652     338      167677       0      30    1148160        0
       tcp_bbr_map    128    2204    5484     1348376       0     126     984064        0
       malloc-1024   1024     594     366       72403       0      16     983040        0
            socket    944     441     547      101082       0     254     932672        0
             tcpcb   1064     284     549       35420       0     254     886312        0
            THREAD   1808     439      23       60329       0       8     835296        0
       malloc-4096   4096       4     196       13079       0       2     819200        0
        256 Bucket   2048     351      43       67178       0       8     806912        0
         FFS inode   1160     670      23       50321       0       8     803880        0
           VMSPACE   2544     290      10       50054       0       4     763200        0
         tcp_inpcb    488     289     991       35420       0     254     624640        0
         sctp_asoc   2288      47     208        7351       0     254     583440        0
           sctp_ep   1280     148     290       57900       0     254     560640        0
       malloc-1024   1024     511      33       62535       0      16     557056        0
        malloc-384    384    1072     278       43079       0      30     518400        0
       malloc-2048   2048      55     189        7866       0       8     499712        0
              PROC   1336     325       8       50102       0       8     444888        0
      malloc-16384  16384      26       0          30       0       1     425984        0
       malloc-1024   1024     184     204       59969       0      16     397312        0
      malloc-65536  65536       4       2         750       0       1     393216        0
          lkpicurr    160       2    2348           2       0      62     376000        0
            lkpimm    160       1    2324           1       0      62     372000        0
         filedesc0   1072     324      19       50650       0       8     367696        0
       tcp_bbr_pcb    832      33     399       17413       0      16     359424        0
             VNODE    448     708      75       50361       0      30     350784        0
        malloc-384    384     335     545       35601       0      30     337920        0
        malloc-384    384     310     560       52721       0      30     334080        0
        sctp_raddr    736      52     366        8822       0     254     307648        0
        malloc-512    512     592       8        1587       0      30     307200        0
        malloc-384    384     695      55        1216       0      30     288000        0
         UMA Zones    768     337       2         337       0      16     260352        0
      tcp_rack_pcb    832      18     279        7055       0      16     247104        0
       mbuf_packet    256     158     801       68887       0     254     245504        0
         vmem btag     56    4362      18        4362       0     254     245280        0
             g_bio    408       0     590       35059       0      30     240720        0
      malloc-16384  16384      11       3         277       0       1     229376        0
       malloc-2048   2048     108       2         125       0       8     225280        0
            DEVCTL   1024       0     216         133       0       0     221184        0
      malloc-16384  16384      10       2          16       0       1     196608        0
       FFS2 dinode    256     670      35       50321       0      62     180480        0
       malloc-8192   8192      15       6         234       0       1     172032        0
            clpbuf   2624       0      64        1073       0      16     167936        0
        malloc-512    512     204     116        1970       0      30     163840        0
       S VFS Cache    104    1066     377       53343       0     126     150072        0
        malloc-384    384     308      72      746945       0      30     145920        0
        128 Bucket   1024      97      38       22412       0      16     138240        0
zio_buf_comb_131072 131072       0       1           1       0       1     131072        0
      malloc-65536  65536       2       0           2       0       1     131072        0
      malloc-65536  65536       2       0         179       0       1     131072        0
      malloc-32768  32768       4       0           4       0       1     131072        0
      tcp_rack_map    112      23    1129       13453       0     126     129024        0
          UMA Kegs    384     322       1         322       0      30     124032        0
         32 Bucket    256     350     115      117919       0      62     119040        0
          ksiginfo    112     318     726       11500       0     126     116928        0
        malloc-512    512     213      11         427       0      30     114688        0
       malloc-4096   4096      18       6        1117       0       2      98304        0
        malloc-512    512      98      70       50364       0      30      86016        0
         16 Bucket    144     435     125        9617       0      62      80640        0
        sctp_chunk    152      50     392        5313       0     254      67184        0
         TURNSTILE    136     463      20         463       0      62      65688        0
      malloc-65536  65536       1       0           1       0       1      65536        0
      malloc-65536  65536       1       0           1       0       1      65536        0
      malloc-65536  65536       1       0           1       0       1      65536        0
      malloc-65536  65536       0       1           2       0       1      65536        0
      malloc-32768  32768       2       0           2       0       1      65536        0
      malloc-16384  16384       3       1           5       0       1      65536        0
      malloc-16384  16384       2       2          19       0       1      65536        0
       malloc-8192   8192       2       6       49165       0       1      65536        0
       malloc-2048   2048      26       4          32       0       8      61440        0
              ertt     72     284     556       35420       0     126      60480        0
             Files     80     500     250      199712       0     126      60000        0
       malloc-1024   1024      53       3          57       0      16      57344        0
            pcpu-8      8    5211    1957       17675       0     254      57344        0
         64 Bucket    512      47      57      110076       0      30      53248        0
zio_data_buf_16384  16384       0       3           9       0       1      49152        0
      malloc-16384  16384       1       2          11       0       1      49152        0
       malloc-8192   8192       4       2         534       0       1      49152        0
       malloc-4096   4096       5       7         737       0       2      49152        0
       malloc-4096   4096      10       2         389       0       2      49152        0
       malloc-2048   2048       4      20        1355       0       8      49152        0
        malloc-512    512      20      76       49284       0      30      49152        0
        malloc-512    512      34      62       55600       0      30      49152        0
              pipe    744      33      32        2612       0      16      48360        0
       malloc-2048   2048       9      13        1717       0       8      45056        0
       malloc-2048   2048      18       4          22       0       8      45056        0
        SLEEPQUEUE     88     463      17         463       0     126      42240        0
           DIRHASH   1024      34       6          34       0      16      40960        0
       malloc-4096   4096       8       2         206       0       2      40960        0
             NAMEI   1024       0      36      247792       0      16      36864        0
     zio_buf_16384  16384       0       2          10       0       1      32768        0
           pcpu-64     64     494      18         494       0     254      32768        0
               PWD     32     167     841       48881       0     254      32256        0
       malloc-4096   4096       1       6         282       0       2      28672        0
       malloc-4096   4096       7       0           7       0       2      28672        0
        malloc-512    512      32      24        2495       0      30      28672        0
             KNOTE    160      28     147      866594       0      62      28000        0
             ripcb    488      10      46        2391       0     254      27328        0
     udplite_inpcb    488       4      52        1016       0     254      27328        0
          2 Bucket     32     270     486       18729       0     254      24192        0
            ttyinq    160     135      15         300       0      62      24000        0
          8 Bucket     80      68     232        9470       0     126      24000        0
           ttyoutq    256      72      18         160       0      62      23040        0
       malloc-2048   2048       9       1           9       0       8      20480        0
         udp_inpcb    488      10      30         845       0     254      19520        0
       Mountpoints   2752       2       5           2       0       4      19264        0
      malloc-16384  16384       1       0           1       0       1      16384        0
      malloc-16384  16384       1       0           1       0       1      16384        0
       malloc-8192   8192       0       2          66       0       1      16384        0
      vtnet_tx_hdr     24       0     668     5170819       0     254      16032        0
             unpcb    256      24      36        3428       0     254      15360        0
sctp_stream_msg_out    112      10      98        2750       0     254      12096        0
        sctp_readq    152       0      78        1853       0     254      11856        0
              kenv    258      15      30        1062       0      30      11610        0
       malloc-4096   4096       0       2        3033       0       2       8192        0
       malloc-2048   2048       0       4           9       0       8       8192        0
       malloc-1024   1024       0       8         487       0      16       8192        0
           pf tags    104       3      75           3       0     126       8112        0
           rtentry    176      30      16          34       0      62       8096        0
              PGRP     88      39      53         101       0     126       8096        0
          rl_entry     40     153      49         153       0     254       8080        0
        sctp_laddr     48      23     145        2398       0     254       8064        0
             udpcb     32      14     238        1861       0     254       8064        0
          4 Bucket     48       7     161        1009       0     254       8064        0
           cryptop    280       0      28         812       0      30       7840        0
            itimer    352       0      22          17       0      30       7744        0
     routing nhops    256      27       3          34       0      62       7680        0
     FPU_save_area    832       1       8           1       0      16       7488        0
            cpuset    104       7      55         119       0     126       6448        0
tfo_ccache_entries     80       9      55           9       0     126       5120        0
         domainset     40       0     126          89       0     254       5040        0
 epoch_record pcpu    256       4      12           4       0      62       4096        0
           pcpu-16     16       7     249           7       0     254       4096        0
    tcp_log_bucket    176       0      23           5       0      62       4048        0
             tcptw     88       1      45           1       0     254       4048        0
   sctp_asconf_ack     48       0      84           2       0     254       4032        0
          tcpreass     48       0      84           2       0     254       4032        0
          sackhole     32       0     126           1       0     254       4032        0
         hostcache     64       1      62           1       0     254       4032        0
          syncache    168       0      24           4       0     254       4032        0
      tcp_log_node    120       0      33          10       0     126       3960        0
       UMA Slabs 1    176      13       9          13       0      62       3872        0
            mqnode    416       3       6           3       0      30       3744        0
        KMAP ENTRY     96      19      20          23       0       0       3744        0
              vmem   1856       1       1           1       0       8       3712        0
           SMR CPU     32       3      60           3       0     254       2016        0
        SMR SHARED     24       3      60           3       0     254       1512        0
   zfs_znode_cache    472       0       0           0       0      30          0        0
       sio_cache_2    168       0       0           0       0      62          0        0
       sio_cache_1    152       0       0           0       0      62          0        0
       sio_cache_0    136       0       0           0       0      62          0        0
     zil_zcw_cache     80       0       0           0       0     126          0        0
     zil_lwb_cache    360       0       0           0       0      30          0        0
    dmu_buf_impl_t    408       0       0           0       0      30          0        0
         arc_buf_t     64       0       0           0       0     254          0        0
arc_buf_hdr_t_l2only     96       0       0           0       0     126          0        0
arc_buf_hdr_t_full_crypt    424       0       0           0       0      30          0        0
arc_buf_hdr_t_full    360       0       0           0       0      30          0        0
           dnode_t   1040       0       0           0       0       8          0        0
          sa_cache    296       0       0           0       0      30          0        0
         abd_chunk   4096       0       0           0       0       2          0        0
zio_buf_comb_16777216 16777216       0       0           0       0       1          0        0
zio_buf_comb_14680064 14680064       0       0           0       0       1          0        0
zio_buf_comb_12582912 12582912       0       0           0       0       1          0        0
zio_buf_comb_10485760 10485760       0       0           0       0       1          0        0
zio_buf_comb_8388608 8388608       0       0           0       0       1          0        0
zio_buf_comb_7340032 7340032       0       0           0       0       1          0        0
zio_buf_comb_6291456 6291456       0       0           0       0       1          0        0
zio_buf_comb_5242880 5242880       0       0           0       0       1          0        0
zio_buf_comb_4194304 4194304       0       0           0       0       1          0        0
zio_buf_comb_3670016 3670016       0       0           0       0       1          0        0
zio_buf_comb_3145728 3145728       0       0           0       0       1          0        0
zio_buf_comb_2621440 2621440       0       0           0       0       1          0        0
zio_buf_comb_2097152 2097152       0       0           0       0       1          0        0
zio_buf_comb_1835008 1835008       0       0           0       0       1          0        0
zio_buf_comb_1572864 1572864       0       0           0       0       1          0        0
zio_buf_comb_1310720 1310720       0       0           0       0       1          0        0
zio_buf_comb_1048576 1048576       0       0           0       0       1          0        0
zio_buf_comb_917504 917504       0       0           0       0       1          0        0
zio_buf_comb_786432 786432       0       0           0       0       1          0        0
zio_buf_comb_655360 655360       0       0           0       0       1          0        0
zio_buf_comb_524288 524288       0       0           0       0       1          0        0
zio_buf_comb_458752 458752       0       0           0       0       1          0        0
zio_buf_comb_393216 393216       0       0           0       0       1          0        0
zio_buf_comb_327680 327680       0       0           0       0       1          0        0
zio_buf_comb_262144 262144       0       0           0       0       1          0        0
zio_buf_comb_229376 229376       0       0           0       0       1          0        0
zio_buf_comb_196608 196608       0       0           0       0       1          0        0
zio_buf_comb_163840 163840       0       0           0       0       1          0        0
zio_buf_comb_114688 114688       0       0           0       0       1          0        0
zio_buf_comb_98304  98304       0       0           0       0       1          0        0
zio_buf_comb_81920  81920       0       0           0       0       1          0        0
zio_buf_comb_65536  65536       0       0           0       0       1          0        0
zio_buf_comb_57344  57344       0       0           0       0       1          0        0
zio_buf_comb_49152  49152       0       0           0       0       1          0        0
zio_buf_comb_40960  40960       0       0           0       0       1          0        0
zio_buf_comb_32768  32768       0       0           0       0       1          0        0
zio_buf_comb_28672  28672       0       0           0       0       1          0        0
zio_buf_comb_24576  24576       0       0           0       0       1          0        0
zio_buf_comb_20480  20480       0       0           0       0       1          0        0
zio_data_buf_14336  14336       0       0           0       0       1          0        0
     zio_buf_14336  14336       0       0           0       0       1          0        0
zio_data_buf_12288  12288       0       0           0       0       1          0        0
     zio_buf_12288  12288       0       0           0       0       1          0        0
zio_data_buf_10240  10240       0       0           0       0       1          0        0
     zio_buf_10240  10240       0       0           0       0       1          0        0
 zio_data_buf_8192   8192       0       0           0       0       1          0        0
      zio_buf_8192   8192       0       0           0       0       1          0        0
 zio_data_buf_7168   7168       0       0           0       0       1          0        0
      zio_buf_7168   7168       0       0           0       0       1          0        0
 zio_data_buf_6144   6144       0       0           0       0       1          0        0
      zio_buf_6144   6144       0       0           0       0       1          0        0
 zio_data_buf_5120   5120       0       0           0       0       1          0        0
      zio_buf_5120   5120       0       0           0       0       1          0        0
 zio_data_buf_4096   4096       0       0           0       0       2          0        0
      zio_buf_4096   4096       0       0           0       0       2          0        0
 zio_data_buf_3584   3584       0       0           0       0       2          0        0
      zio_buf_3584   3584       0       0           0       0       2          0        0
 zio_data_buf_3072   3072       0       0           0       0       4          0        0
      zio_buf_3072   3072       0       0           0       0       4          0        0
 zio_data_buf_2560   2560       0       0           0       0       4          0        0
      zio_buf_2560   2560       0       0           0       0       4          0        0
 zio_data_buf_2048   2048       0       0           0       0       8          0        0
      zio_buf_2048   2048       0       0           0       0       8          0        0
 zio_data_buf_1536   1536       0       0           0       0       8          0        0
      zio_buf_1536   1536       0       0           0       0       8          0        0
 zio_data_buf_1024   1024       0       0           0       0      16          0        0
      zio_buf_1024   1024       0       0           0       0      16          0        0
  zio_data_buf_512    512       0       0           0       0      30          0        0
       zio_buf_512    512       0       0           0       0      30          0        0
    zio_link_cache     48       0       0           0       0     254          0        0
         zio_cache   1232       0       0           0       0       8          0        0
   ddt_entry_cache    392       0       0           0       0      30          0        0
         ddt_cache  24840       0       0           0       0       1          0        0
metaslab_alloc_trace_cache     72       0       0           0       0     126          0        0
zfs_btree_leaf_cache   4096       0       0           0       0       2          0        0
reference_history_cache      8       0       0           0       0     254          0        0
   reference_cache     40       0       0           0       0     254          0        0
        taskq_zone    192       0       0           0       0      62          0        0
       FFS1 dinode    128       0       0           0       0     126          0        0
            da_ccb    544       0       0           0       0      16          0        0
           ada_ccb    272       0       0           0       0      30          0        0
             swblk    136       0       0           0       0      62          0        0
          swpctrie    144       0       0           0       0      62          0        0
   cdg_qdiffsample     16       0       0           0       0     254          0        0
       sctp_asconf     40       0       0           0       0     254          0        0
   pf state scrubs     40       0       0           0       0     254          0        0
   pf frag entries     40       0       0           0       0     254          0        0
          pf frags    248       0       0           0       0      62          0        0
  pf table entries    160       0       0           0       0      62          0        0
pf table entry counters     64       0       0           0       0     254          0        0
   pf source nodes    136       0       0           0       0     254          0        0
     pf state keys     88       0       0           0       0     126          0        0
         pf states    304       0       0           0       0     254          0        0
          pf mtags     48       0       0           0       0     254          0        0
               tfo      4       0       0           0       0     254          0        0
               ipq     56       0       0           0       0     254          0        0
    IPsec SA lft_c     16       0       0           0       0     254          0        0
            AIOLIO    272       0       0           0       0      30          0        0
             AIOCB    552       0       0           0       0      16          0        0
              AIOP     32       0       0           0       0     254          0        0
               AIO    208       0       0           0       0      62          0        0
        TMPFS node    224       0       0           0       0      62          0        0
           NCLNODE    584       0       0           0       0      16          0        0
        mqnotifier    216       0       0           0       0      62          0        0
            mvdata     64       0       0           0       0     254          0        0
            mqueue    248       0       0           0       0      62          0        0
     LTS VFS Cache    360       0       0           0       0      30          0        0
       L VFS Cache    320       0       0           0       0      30          0        0
     STS VFS Cache    144       0       0           0       0      62          0        0
  linux_dma_object     24       0       0           0       0     254          0        0
  linux_dma_pctrie    144       0       0           0       0      62          0        0
   IOMMU_MAP_ENTRY    120       0       0           0       0     126          0        0
      ktls_session    192       0       0           0       0      62          0        0
    mbuf_jumbo_16k  16384       0       0           0       0     254          0        0
     mbuf_jumbo_9k   9216       0       0           0       0     254          0        0
      audit_record   1280       0       0           0       0       8          0        0
        MAC labels     40       0       0           0       0     254          0        0
            vnpbuf   2624       0       0           0       0      64          0        0
            mdpbuf   2624       0       0           0       0       3          0        0
           nfspbuf   2624       0       0           0       0      16          0        0
            swwbuf   2624       0       0           0       0       8          0        0
            swrbuf   2624       0       0           0       0      16          0        0
          umtx_shm     88       0       0           0       0     126          0        0
           umtx pi     96       0       0           0       0     126          0        0
rangeset pctrie nodes    144       0       0           0       0      62          0        0
      malloc-65536  65536       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
       malloc-8192   8192       0       0           0       0       1          0        0
        malloc-256    256       0       0           0       0      62          0        0
        malloc-256    256       0       0           0       0      62          0        0
        malloc-256    256       0       0           0       0      62          0        0
        malloc-256    256       0       0           0       0      62          0        0
        malloc-256    256       0       0           0       0      62          0        0
        malloc-256    256       0       0           0       0      62          0        0
        malloc-256    256       0       0           0       0      62          0        0
        malloc-256    256       0       0           0       0      62          0        0
        malloc-128    128       0       0           0       0     126          0        0
        malloc-128    128       0       0           0       0     126          0        0
        malloc-128    128       0       0           0       0     126          0        0
        malloc-128    128       0       0           0       0     126          0        0
        malloc-128    128       0       0           0       0     126          0        0
        malloc-128    128       0       0           0       0     126          0        0
        malloc-128    128       0       0           0       0     126          0        0
        malloc-128    128       0       0           0       0     126          0        0
         malloc-64     64       0       0           0       0     254          0        0
         malloc-64     64       0       0           0       0     254          0        0
         malloc-64     64       0       0           0       0     254          0        0
         malloc-64     64       0       0           0       0     254          0        0
         malloc-64     64       0       0           0       0     254          0        0
         malloc-64     64       0       0           0       0     254          0        0
         malloc-64     64       0       0           0       0     254          0        0
         malloc-64     64       0       0           0       0     254          0        0
         malloc-32     32       0       0           0       0     254          0        0
         malloc-32     32       0       0           0       0     254          0        0
         malloc-32     32       0       0           0       0     254          0        0
         malloc-32     32       0       0           0       0     254          0        0
         malloc-32     32       0       0           0       0     254          0        0
         malloc-32     32       0       0           0       0     254          0        0
         malloc-32     32       0       0           0       0     254          0        0
         malloc-32     32       0       0           0       0     254          0        0
         malloc-16     16       0       0           0       0     254          0        0
         malloc-16     16       0       0           0       0     254          0        0
         malloc-16     16       0       0           0       0     254          0        0
         malloc-16     16       0       0           0       0     254          0        0
         malloc-16     16       0       0           0       0     254          0        0
         malloc-16     16       0       0           0       0     254          0        0
         malloc-16     16       0       0           0       0     254          0        0
         malloc-16     16       0       0           0       0     254          0        0
           pcpu-32     32       0       0           0       0     254          0        0
            pcpu-4      4       0       0           0       0     254          0        0
            fakepg    104       0       0           0       0     126          0        0
          UMA Hash    256       0       0           0       0      62          0        0

Crashes (2):
Time Kernel Commit Syzkaller Config Log Report Syz repro C repro VM info Assets (help?) Manager Title
2021/05/22 00:36 freebsd-src 4eac63af23dd 3c7fef33 console log report ci-freebsd-main panic: _mtx_lock_sleep: recursed on non-recursive mutex so_snd @ /syzkaller/managers/main/kernel/sys/modules/tcp/rack/..
2021/05/21 14:44 freebsd-src 4eac63af23dd 3c7fef33 console log report ci-freebsd-main panic: _mtx_lock_sleep: recursed on non-recursive mutex so_snd @ /syzkaller/managers/main/kernel/sys/modules/tcp/rack/..
* Struck through repros no longer work on HEAD.