syzbot


Fatal trap NUM: page fault in sctp_ss_default_add (3)

Status: upstream: reported on 2024/03/03 07:15
Reported-by: syzbot+50e1ff0378b2eaf3d3dc@syzkaller.appspotmail.com
First crash: 78d, last: 78d
Similar bugs (3)
Kernel Title Repro Cause bisect Fix bisect Count Last Reported Patched Status
freebsd Fatal trap NUM: page fault in sctp_ss_default_add (2) 1 397d 397d 0/2 auto-obsoleted due to no activity on 2023/07/18 21:54
freebsd Fatal trap 12: page fault in sctp_ss_default_add (2) 1 1050d 1050d 2/2 fixed on 2021/09/20 20:59
freebsd Fatal trap NUM: page fault in sctp_ss_default_add 1 833d 833d 0/2 closed as invalid on 2022/04/17 13:29

Sample crash report:
Fatal trap 12: page fault while in kernel mode
cpuid = 1; apic id = 01
fault virtual address	= 0xfffffe007699b010
fault code		= supervisor read data, page not present
instruction pointer	= 0x20:0xffffffff8401290f
stack pointer	        = 0x0:0xfffffe006a5cd240
frame pointer	        = 0x0:0xfffffe006a5cd270
code segment		= base 0x0, limit 0xfffff, type 0x1b
			= DPL 0, pres 1, long 1, def32 0, gran 1
processor eflags	= interrupt enabled, resume, IOPL = 0
current process		= 28506 (syz-executor.0)
07:14:29 executing program 3:
r0 = socket(0x10, 0x20000000, 0x1)
sendto(r0, 0x0, 0x0, 0x0, &(0x7f0000000000)=@in6={0x1c, 0x1c, 0x0, 0x0, @remote={0xfe, 0x80, '\x00', 0x0}}, 0x1c)
bind$inet6(r0, &(0x7f0000000040)={0x1c, 0x1c, 0x0, 0x0, @remote={0xfe, 0x80, '\x00', 0x0}}, 0x1c)
r1 = openat$md(0xffffff9c, &(0x7f00000001c0), 0x0, 0x0)
r2 = openat$filemon(0xffffff9c, &(0x7f0000000000), 0x0, 0x0)
ioctl$FILEMON_SET_FD(r2, 0xc0045301, &(0x7f0000000040)=r1)
ioctl$MDIOCATTACH(r1, 0xc1b46d00, &(0x7f0000000280)={0x2, 0xe2, 0x4, &(0x7f0000000200)='./file0\x00', 0x0, 0x7, 0x7, 0xffffffff7fffffff, 0x3e28, 0x6, &(0x7f0000000240)='\x00', [0x400, 0xfff, 0x4, 0x7fff, 0x9, 0xfffffffc, 0x400, 0x4, 0x6, 0x7f, 0x9, 0x7ff, 0x5, 0xff, 0x8, 0x5, 0x80000000, 0xbb, 0x9, 0x4ceb, 0x6, 0x401, 0x608, 0x7, 0xfffffffd, 0xffffff86, 0x8001, 0x2, 0x7f, 0x4, 0x0, 0x4, 0x81, 0x10001, 0x100, 0x10000, 0x7, 0x4, 0xffff8001, 0x2, 0x7, 0x4, 0x200, 0xfffff800, 0x3, 0xfffffffe, 0x9, 0x7ff, 0x2e, 0x7, 0x885d, 0x800, 0x1, 0x1f, 0x3, 0x0, 0xff, 0x80, 0xffff, 0xa59, 0x5, 0x11c2, 0x8a6, 0x10001, 0xc0000000, 0x7, 0x6, 0x3, 0x9, 0xaa, 0x0, 0x5, 0x1, 0x200, 0x2, 0x885, 0x6, 0x4, 0x2353421c, 0x80, 0x8, 0x800, 0x0, 0x9, 0x0, 0x7, 0xe700000, 0x8, 0x3, 0x59f358e9, 0xa7, 0xfffffff7, 0x0, 0x7, 0x400, 0x8001]})
accept4(r0, &(0x7f00000000c0)=@un=@file={0xf9, 0x0, ""/247}, &(0x7f00000001c0)=0xf9, 0x10000000)
sendto(r0, &(0x7f0000000080)="7869e88b25b2981e246cf38bea4771fb24ee1498490f4d45fb6197883a30eb3aaf001d09dd38795d8df291c79fc08b453c1fc7652c8e10df231886", 0x3b, 0x0, 0x0, 0x0)

rdi: fffffe007699b010 rsi: 0000000000000000 rdx: 0000000000000000
rcx: 0000000000000000  r8: 0000000000000000  r9: 0000000000000001
rax: 000000000ed33602 rbx: fffffe0076df9000 rbp: fffffe006a5cd270
r10: f8ff3013798974ef r11: fffffe0076903c60 r12: fffffe00743705d8
r13: fffffe007699b010 r14: fffffe00743701f8 
r15: fffffe0076df9028
trap number		= 12
panic: page fault
cpuid = 1
time = 1709450069
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0xc6/frame 0xfffffe006a5cc970
kdb_backtrace() at kdb_backtrace+0xd0/frame 0xfffffe006a5ccad0
vpanic() at vpanic+0x26a/frame 0xfffffe006a5ccc90
panic() at panic+0xb5/frame 0xfffffe006a5ccd50
trap_fatal() at trap_fatal+0x7f1/frame 0xfffffe006a5cce70
trap_pfault() at trap_pfault+0x179/frame 0xfffffe006a5ccf90
trap() at trap+0x5f3/frame 0xfffffe006a5cd170
calltrap() at calltrap+0x8/frame 0xfffffe006a5cd170
--- trap 0xc, rip = 0xffffffff8401290f, rsp = 0xfffffe006a5cd240, rbp = 0xfffffe006a5cd270 ---
sctp_ss_default_add() at sctp_ss_default_add+0x9f/frame 0xfffffe006a5cd270
sctp_lower_sosend() at sctp_lower_sosend+0x5272/frame 0xfffffe006a5cd6c0
sctp_sosend() at sctp_sosend+0x701/frame 0xfffffe006a5cd960
sousrsend() at sousrsend+0x116/frame 0xfffffe006a5cd9f0
kern_sendit() at kern_sendit+0x5a2/frame 0xfffffe006a5cdb50
freebsd32_sendmsg() at freebsd32_sendmsg+0x515/frame 0xfffffe006a5cdd10
ia32_syscall() at ia32_syscall+0x466/frame 0xfffffe006a5cdf30
int0x80_syscall_common() at int0x80_syscall_common+0x9c/frame 0xdfffcf9c
KDB: enter: panic
[ thread pid 28506 tid 255220 ]
Stopped at      kdb_enter+0x6e: movq    $0,0x2192727(%rip)
db> set $lines = 0
db> set $maxwidth = 0
db> show registers
cs                        0x20
ds                        0x3b
es                        0x3b
fs                        0x13
gs                        0x1b
ss                           0
rax                       0x12
rcx         0xfffffe00033eee30
rdx                          0
rbx         0xffffffff82701340  .str.27
rsp         0xfffffe006a5ccab0
rbp         0xfffffe006a5ccad0
rsi                          0
rdi         0xffffffff815e1df9  printf+0x149
r8                           0
r9                  0xffffffff
r10                        0x1
r11                        0x1
r12         0xfffffe0076903740
r13         0xfffffffffffffffd
r14         0xffffffff82701340  .str.27
r15                          0
rip         0xffffffff815ce8ee  kdb_enter+0x6e
rflags                    0x46
kdb_enter+0x6e: movq    $0,0x2192727(%rip)
db> show proc
Process 28506 (syz-executor.0) at 0xfffffe0074325060:
 state: NORMAL
 uid: 0  gids: 0, 0, 5
 parent: pid 770 at 0xfffffe006cd13ae0
 ABI: FreeBSD ELF32
 flag: 0x10000080  flag2: 0
 arguments: /root/syz-executor.0 exec
 reaper: 0xfffffe00541d7040 reapsubtree: 1
 sigparent: 20
 vmspace: 0xfffffe0073b0ac08
   (map 0xfffffe0073b0ac08)
   (map.pmap 0xfffffe0073b0acc8)
   (pmap 0xfffffe0073b0ad38)
 threads: 3
252554                   RunQ                                syz-executor.0
255220                   Run     CPU 1                       syz-executor.0
255221                   S       uwait   0xfffffe0073af9b00  syz-executor.0
db> ps
  pid  ppid  pgrp   uid  state   wmesg   wchan               cmd
28525   774   774     0  R                                   syz-executor.2
28506   770   770     0  R       (threaded)                  syz-executor.0
252554                   RunQ                                syz-executor.0
255220                   Run     CPU 1                       syz-executor.0
255221                   S       uwait   0xfffffe0073af9b00  syz-executor.0
28502   771   771     0  R       (threaded)                  syz-executor.1
254206                   RunQ                                syz-executor.1
255216                   S       uwait   0xfffffe0073cd7180  syz-executor.1
52401     0     0     0  DL      -       0xffffffff83852240  [soaiod4]
52400     0     0     0  DL      -       0xffffffff83852240  [soaiod3]
52399     0     0     0  DL      -       0xffffffff83852240  [soaiod2]
52398     0     0     0  DL      -       0xffffffff83852240  [soaiod1]
25636     1 25636    65  Ss      select  0xfffffe0073bc48c0  dhclient
18698     1 18698     0  Ss      select  0xfffffe0073de51c0  dhclient
18694     1 18694     0  Ss      select  0xfffffe0073af7c40  dhclient
18675     1 18675    65  Ss      select  0xfffffe0073b508c0  dhclient
13743     1 13743     0  Ss      select  0xfffffe0073b51840  dhclient
13734     1 13734     0  Ss      select  0xfffffe0073b51a40  dhclient
13681     1 13681    65  Ss      select  0xfffffe0073cd9c40  dhclient
 8990     0     0     0  DL      aiordy  0xfffffe0057bce060  [aiod4]
 8989     0     0     0  DL      aiordy  0xfffffe006cdbdae0  [aiod3]
 8988     0     0     0  DL      aiordy  0xfffffe006cdbcac0  [aiod2]
 8987     0     0     0  DL      aiordy  0xfffffe0073b94ae0  [aiod1]
 8192     1  8192     0  Ss      select  0xfffffe0074002b40  dhclient
 8183     1  8183     0  Ss      select  0xfffffe007403da40  dhclient
 8133     1  8133    65  Ss      select  0xfffffe0073b8f840  dhclient
 2223     1  2223     0  Ss      select  0xfffffe0073ca42c0  dhclient
 2216     1  2216     0  Ss      select  0xfffffe0073ca43c0  dhclient
  777   768   777     0  Rs                                  syz-executor.3
  774   768   774     0  Rs                                  syz-executor.2
  771   768   771     0  Rs                                  syz-executor.1
  770   768   770     0  Rs                                  syz-executor.0
  768   766   766     0  R       (threaded)                  syz-fuzzer
100119                   S       uwait   0xfffffe0057a95200  syz-fuzzer
100120                   RunQ                                syz-fuzzer
100121                   S       uwait   0xfffffe005765f580  syz-fuzzer
100122                   S       uwait   0xfffffe0057a94680  syz-fuzzer
100123                   S       wait    0xfffffe006cd14040  syz-fuzzer
100124                   S       uwait   0xfffffe005765e400  syz-fuzzer
100125                   S       uwait   0xfffffe005765e500  syz-fuzzer
100127                   S       wait    0xfffffe006cd14040  syz-fuzzer
100128                   S       wait    0xfffffe006cd14040  syz-fuzzer
100131                   S       wait    0xfffffe006cd14040  syz-fuzzer
100136                   S       uwait   0xfffffe005765e800  syz-fuzzer
100188                   S       kqread  0xfffffe005407b500  syz-fuzzer
108395                   S       uwait   0xfffffe005765e900  syz-fuzzer
  766   764   766     0  Ss      pause   0xfffffe006cd15670  csh
  764   682   764     0  Ss      select  0xfffffe005765e0c0  sshd
  748     1   748     0  Rs+     CPU 0                       getty
  747     1   747     0  Ss+     ttyin   0xfffffe0057b738b0  getty
  746     1   746     0  Ss+     ttyin   0xfffffe0057b73cb0  getty
  745     1   745     0  Ss+     ttyin   0xfffffe0057b740b0  getty
  744     1   744     0  Ss+     ttyin   0xfffffe0057b744b0  getty
  743     1   743     0  Ss+     ttyin   0xfffffe0057b748b0  getty
  742     1   742     0  Ss+     ttyin   0xfffffe0057b74cb0  getty
  741     1   741     0  Ss+     ttyin   0xfffffe0057b750b0  getty
  740     1   740     0  Ss+     ttyin   0xfffffe0057b754b0  getty
  686     1   686     0  Ss      nanslp  0xffffffff83742400  cron
  682     1   682     0  Ss      select  0xfffffe005765eac0  sshd
  495     1   495     0  Ds      bo_wwai 0xfffffe006cd2f168  syslogd
  424     1   424     0  Ss      select  0xfffffe005765ebc0  devd
  423     1   423    65  Ss      select  0xfffffe005765ec40  dhclient
  338     1   338     0  Ss      select  0xfffffe005765f540  dhclient
  335     1   335     0  Ss      select  0xfffffe0057a95540  dhclient
   17     0     0     0  DL      syncer  0xffffffff8385fb20  [syncer]
   16     0     0     0  DL      vlruwt  0xfffffe00571a0060  [vnlru]
   15     0     0     0  DL      (threaded)                  [bufdaemon]
100079                   D       psleep  0xffffffff8385e100  [bufdaemon]
100082                   D       -       0xffffffff82c0a140  [bufspacedaemon-0]
100095                   D       sdflush 0xfffffe0057bf40e8  [/ worker]
    9     0     0     0  DL      psleep  0xffffffff838a75c0  [vmdaemon]
    8     0     0     0  DL      (threaded)                  [pagedaemon]
100077                   D       psleep  0xffffffff8388f438  [dom0]
100080                   D       launds  0xffffffff8388f444  [laundry: dom0]
100081                   D       umarcl  0xffffffff81d67750  [uma]
    7     0     0     0  RL                                  [rand_harvestq]
    6     0     0     0  DL      pftm    0xffffffff842c1ea0  [pf purge]
    5     0     0     0  DL      waiting 0xffffffff840985c0  [sctp_iterator]
    4     0     0     0  DL      (threaded)                  [cam]
100044                   D       -       0xffffffff8348a340  [doneq0]
100045                   D       -       0xffffffff8348a2c0  [async]
100076                   D       -       0xffffffff8348a140  [scanner]
    3     0     0     0  DL      (threaded)                  [crypto]
100041                   D       crypto_ 0xffffffff8388ac60  [crypto]
100042                   D       crypto_ 0xfffffe0054089e30  [crypto returns 0]
100043                   D       crypto_ 0xfffffe0054089e80  [crypto returns 1]
   14     0     0     0  DL      seqstat 0xfffffe00543c2c88  [sequencer 00]
   13     0     0     0  DL      (threaded)                  [geom]
100035                   D       -       0xffffffff836ead80  [g_event]
100036                   D       -       0xffffffff836eada0  [g_up]
100037                   D       -       0xffffffff836eadc0  [g_down]
    2     0     0     0  WL      (threaded)                  [clock]
100029                   I                                   [clock (0)]
100030                   I                                   [clock (1)]
   12     0     0     0  WL      (threaded)                  [intr]
100012                   I                                   [swi6: task queue]
100013                   I                                   [swi6: Giant taskq]
100015                   I                                   [swi5: fast taskq]
100031                   I                                   [swi1: netisr 0]
100032                   I                                   [swi1: hpts]
100033                   I                                   [swi1: hpts]
100046                   I                                   [irq24: virtio_pci0]
100047                   I                                   [irq25: virtio_pci0]
100048                   I                                   [irq26: virtio_pci0]
100049                   I                                   [irq27: virtio_pci0]
100050                   I                                   [irq28: virtio_pci1]
100051                   I                                   [irq29: virtio_pci1]
100052                   I                                   [irq30: virtio_pci1]
100053                   I                                   [irq31: virtio_pci1]
100054                   I                                   [irq32: virtio_pci1]
100059                   I                                   [irq33: virtio_pci2]
100060                   I                                   [irq34: virtio_pci2]
100061                   I                                   [irq35: virtio_pci2]
100063                   I                                   [irq1: atkbd0]
100064                   I                                   [irq12: psm0]
100065                   I                                   [swi0: uart uart++]
100069                   I                                   [swi1: pf send]
   11     0     0     0  RL      (threaded)                  [idle]
100003                   CanRun                              [idle: cpu0]
100004                   CanRun                              [idle: cpu1]
    1     0     1     0  SLs     wait    0xfffffe00541d7040  [init]
   10     0     0     0  DL      audit_w 0xffffffff8388b6c0  [audit]
    0     0     0     0  DLs     (threaded)                  [kernel]
100000                   D       swapin  0xffffffff836eb760  [swapper]
100005                   D       -       0xfffffe00079cda00  [softirq_0]
100006                   D       -       0xfffffe00079cd800  [softirq_1]
100007                   D       -       0xfffffe00079cd600  [if_io_tqg_0]
100008                   D       -       0xfffffe00079cd400  [if_io_tqg_1]
100009                   D       -       0xfffffe00079cd200  [if_config_tqg_0]
100010                   D       -       0xfffffe00079cd000  [pci_hp taskq]
100011                   D       -       0xfffffe00079ccd00  [kqueue_ctx taskq]
100014                   D       -       0xfffffe00079cc700  [thread taskq]
100016                   D       -       0xfffffe00079cc300  [aiod_kick taskq]
100017                   D       -       0xfffffe00079cc100  [deferred_unmount ta]
100018                   D       -       0xfffffe00079cbe00  [inm_free taskq]
100019                   D       -       0xfffffe00079cbc00  [in6m_free taskq]
100020                   D       -       0xfffffe00079cba00  [linuxkpi_irq_wq]
100021                   D       -       0xfffffe00079cb800  [linuxkpi_short_wq_0]
100022                   D       -       0xfffffe00079cb800  [linuxkpi_short_wq_1]
100023                   D       -       0xfffffe00079cb800  [linuxkpi_short_wq_2]
100024                   D       -       0xfffffe00079cb800  [linuxkpi_short_wq_3]
100025                   D       -       0xfffffe00079cb300  [linuxkpi_long_wq_0]
100026                   D       -       0xfffffe00079cb300  [linuxkpi_long_wq_1]
100027                   D       -       0xfffffe00079cb300  [linuxkpi_long_wq_2]
100028                   D       -       0xfffffe00079cb300  [linuxkpi_long_wq_3]
100034                   D       -       0xfffffe00079ca700  [firmware taskq]
100039                   D       -       0xfffffe00079ca000  [crypto_0]
100040                   D       -       0xfffffe00079ca000  [crypto_1]
100055                   D       -       0xfffffe00079ce400  [vtnet0 rxq 0]
100056                   D       -       0xfffffe00079ce300  [vtnet0 txq 0]
100057                   D       -       0xfffffe00079ce200  [vtnet0 rxq 1]
100058                   D       -       0xfffffe0057558500  [vtnet0 txq 1]
100062                   D       vtbslp  0xfffffe005755bd80  [virtio_balloon]
100066                   D       -       0xffffffff82706581  [deadlkres]
100070                   D       -       0xfffffe0057a9d000  [acpi_task_0]
100071                   D       -       0xfffffe0057a9d000  [acpi_task_1]
100072                   D       -       0xfffffe0057a9d000  [acpi_task_2]
100074                   D       -       0xfffffe00079ce100  [mca taskq]
100075                   D       -       0xfffffe0057139c00  [CAM taskq]
126741                   D       -       0xfffffe0073be0e00  [system_taskq_0]
126742                   D       -       0xfffffe0073be0e00  [system_taskq_1]
126743                   D       -       0xfffffe0073be0c00  [system_delay_taskq_]
126744                   D       -       0xfffffe0073be0c00  [system_delay_taskq_]
126745                   D       -       0xfffffe0057a9ae00  [arc_prune]
126746                   D       -       0xfffffe0073bdf800  [dbu_evict]
126747                   D       -       0xfffffe0057558d00  [z_vdev_file_0]
126748                   D       -       0xfffffe0057558d00  [z_vdev_file_1]
126749                   D       -       0xfffffe0057558d00  [z_vdev_file_2]
126750                   D       -       0xfffffe0057558d00  [z_vdev_file_3]
126751                   D       -       0xfffffe0057558d00  [z_vdev_file_4]
126752                   D       -       0xfffffe0057558d00  [z_vdev_file_5]
126753                   D       -       0xfffffe0057558d00  [z_vdev_file_6]
126754                   D       -       0xfffffe0057558d00  [z_vdev_file_7]
126755                   D       -       0xfffffe0057558d00  [z_vdev_file_8]
126756                   D       -       0xfffffe0057558d00  [z_vdev_file_9]
126757                   D       -       0xfffffe0057558d00  [z_vdev_file_10]
126758                   D       -       0xfffffe0057558d00  [z_vdev_file_11]
126759                   D       -       0xfffffe0057558d00  [z_vdev_file_12]
126760                   D       -       0xfffffe0057558d00  [z_vdev_file_13]
126761                   D       -       0xfffffe0057558d00  [z_vdev_file_14]
126762                   D       -       0xfffffe0057558d00  [z_vdev_file_15]
126763                   D       -       0xfffffe0057a9ab00  [zfsvfs]
db> show all locks
Process 28506 (syz-executor.0) thread 0xfffffe0076903740 (255220)
exclusive sleep mutex sctp-tcb (tcb) r = 0 (0xfffffe0074370a50) locked @ /syzkaller/managers/i386/kernel/sys/netinet/sctp_output.c:13192
Process 748 (getty) thread 0xfffffe006cdc3740 (100114)
exclusive sleep mutex ttymtx (ttymtx) r = 0 (0xfffffe005768bc08) locked @ /syzkaller/managers/i386/kernel/sys/kern/tty.c:217
Process 495 (syslogd) thread 0xfffffe0058b0f740 (100103)
exclusive lockmgr ufs (ufs) r = 0 (0xfffffe006cd2f070) locked @ /syzkaller/managers/i386/kernel/sys/kern/vfs_syscalls.c:3581
db> show malloc
              Type        InUse        MemUse     Requests
           pf_hash            5        11524K            5
            linker          397         9838K         5980
         sctp_stro          598         5796K        17181
          tcp_hpts            7         4801K            7
            devbuf         4192         4324K         4221
           solaris         2236         3596K         4462
         sysctloid        43540         2558K        43646
               pcb         1225         2017K       163776
             vtbuf           24         1968K           46
              kobj          326         1304K          488
          inodedep         1658         1134K       151187
            newblk           31         1032K       133596
          vfscache            3         1025K            3
         ufs_quota            1          512K            1
          vfs_hash            1          512K            1
           callout            2          512K            2
              intr            4          472K            4
         sctp_atcl         1197          449K        91764
            dirrem         1640          410K       144391
           subproc          153          290K       128743
              vmem            3          266K            6
         vnet_data            2          224K            2
          acpitask            1          224K            1
          freefile         1608          201K       144326
            acpica         1674          184K        60830
           tidhash            3          141K            3
           pagedep            9          130K       129580
        tfo_ccache            1          128K            1
          IP reass            1          128K            1
          filedesc           16          121K       252657
            DEVFS1          110          110K          127
               sem            4          106K            4
        gtaskqueue           18           98K           18
               BPF           46           88K           46
               bus          985           81K         5155
           filemon           10           80K        11206
         sctp_timw          316           79K          316
         sctp_atky         1795           75K       107529
          mtx_pool            2           72K            2
          syncache            1           68K            1
     NFSD srvcache            3           68K            3
            module          518           65K          518
       ddb_capture            1           64K            1
              umtx          448           56K          448
           kdtrace          259           53K       283719
              temp           35           37K        77004
            DEVFS3          129           33K          139
         hostcache            1           32K            1
               shm            1           32K         1483
             mount          211           32K         8388
               msg            4           30K            4
            kbdmux            6           28K            6
               LRO           24           25K           24
            ifaddr           70           20K           72
        DEVFS_RULE           56           20K           56
         sctp_athm         1197           19K        91764
          sctp_map         1196           19K        31530
        kstat_data           18           18K           18
              ksem           25           18K         5617
         ufs_mount            4           17K            5
              proc            3           17K            3
               tty           16           16K           16
          routetbl          130           16K          410
           lltable           50           16K          278
           ithread           97           16K           97
            bus-sc           34           15K         1687
      eventhandler          164           14K          164
            KTRACE          102           13K         7155
             ifnet            7           13K            7
       ether_multi          152           13K          162
              kenv           95           12K           95
              GEOM           61           11K          485
              rman           86           11K          451
         CAM queue            5           11K         1528
         in6_multi           65            9K           65
         taskqueue           81            9K           81
         bmsafemap            2            9K       146809
               rpc            4            9K            4
           devstat            4            9K            4
              UART           12            9K           12
             shmfd            1            8K         4545
       pfs_vncache            1            8K            1
     audit_evclass          238            8K          300
              cred           26            7K         6207
          freework           26            7K       132543
          freeblks           25            7K       132445
            kqueue           66            7K       128475
            sglist            6            7K            6
            plimit           24            6K          529
           CAM DEV            3            6K          510
               UMA          330            6K          330
         pfs_nodes           22            6K           22
             hhook           15            5K           17
            DEVFSP           78            5K        11278
       ufs_dirhash           24            5K           24
           session           35            5K        13089
          pf_ifnet           10            5K           19
                vt           11            5K           11
           memdesc            1            4K            1
               MCA           32            4K           32
           pwddesc           64            4K       128469
             evdev            4            4K            4
           acpisem           28            4K           28
             lockf           32            4K          106
         proc-args           91            4K       129239
          kcovinfo           52            4K           52
             selfd           50            4K      1673709
          terminal           11            3K           11
            select           19            3K           56
             clone            9            3K            9
           uidinfo            3            3K           15
        local_apic            1            2K            1
           io_apic            1            2K            1
         ipsec-saq            2            2K            2
            ip6ndp           12            2K           14
            Unitno           28            2K         6777
          sctp_ifa           13            2K           14
               tun            7            2K            7
           CAM XPT           22            2K          543
               msi           12            2K           12
          in_multi            6            2K            8
       vnodemarker            3            2K          121
         toponodes            6            2K            6
       ipsecpolicy            2            2K            2
            CC Mem            5            2K        14385
           acpidev           20            2K           20
           softdep            1            1K            1
             mkdir            8            1K       259116
            sahead            1            1K            1
          secasvar            1            1K            1
             nhops            6            1K            8
      NFSD session            1            1K            1
               osd           33            1K        14428
        CAM periph            4            1K          271
             ipsec            3            1K            3
          sctp_ifn            6            1K           14
               mld            6            1K            6
              igmp            6            1K            6
              pfil            6            1K            6
            isadev            6            1K            6
          pci_link           10            1K           10
            crypto            4            1K            4
 encap_export_host           12            1K           12
          procdesc            5            1K           18
         newdirblk            4            1K       129558
            diradd            4            1K       144435
           netlink            2            1K          257
       inpcbpolicy           16            1K        57233
              cdev            2            1K            2
       lkpikmalloc            8            1K            9
    chacha20random            1            1K            1
            biobuf            1            1K            1
            vnodes            1            1K            1
      NFSD lckfile            1            1K            1
     NFSD V4client            1            1K            1
             DEVFS            9            1K           10
           CAM SIM            2            1K            2
            prison            8            1K            8
            feeder            7            1K            7
             taskq            2            1K            2
           tcpfunc            3            1K            3
        loginclass            3            1K            6
         cryptodev            2            1K           42
          nexusdev            8            1K            8
            apmdev            1            1K            1
          atkbddev            2            1K            2
          freefrag            1            1K           55
               aio            4            1K            4
            soname            5            1K       141369
          pmchooks            1            1K            1
          CAM path            4            1K         1034
     CAM dev queue            2            1K            2
 CAM I/O Scheduler            1            1K            1
          filecaps            5            1K         3853
          sctp_vrf            1            1K            1
              vnet            1            1K            1
               pmc            1            1K            1
           entropy            2            1K           50
          acpiintr            1            1K            1
              cpus            2            1K            2
    vnet_data_free            1            1K            1
           Per-cpu            1            1K            1
               iov            1            1K       114155
          p1003.1b            1            1K            1
        ext2_mount            0            0K            0
         ext2_node            0            0K            0
      ext2_extents            0            0K            0
         sfs_nodes            0            0K            0
        zones_data            0            0K            0
            ipcomp            0            0K            0
               esp            0            0K            0
                ah            0            0K            0
            mqdata            0            0K            0
            tcp_do            0            0K            0
           tcp_fsb            0            0K            0
          pf_table            0            0K            0
           pf_rule            0            0K            0
           pf_altq            0            0K            0
           pf_osfp            0            0K            0
     pf_krule_item            0            0K            0
           pf_temp            0            0K            0
        sctp_mcore            0            0K            0
        sctp_socko            0            0K        72554
         sctp_iter            0            0K         2250
         sctp_mvrf            0            0K            0
         sctp_cpal            0            0K         2259
         sctp_cmsg            0            0K            0
         sctp_stre            0            0K            0
         sctp_athi            0            0K            0
         sctp_a_it            0            0K            9
         sctp_aadr            0            0K            0
         sctp_stri            0            0K            0
        madt_table            0            0K            2
          smartpqi            0            0K            0
               ixl            0            0K            0
        ice-resmgr            0            0K            0
         ice-osdep            0            0K            0
               ice            0            0K            0
              iavf            0            0K            0
             axgbe            0            0K            0
       fpukern_ctx            0            0K            0
          xen_intr            0            0K            0
           xen_hvm            0            0K            0
         legacydrv            0            0K            0
            bounce            0            0K            0
            busdma            0            0K            0
            qpidrv            0            0K            0
      dmar_idpgtbl            0            0K            0
          dmar_dom            0            0K            0
          dmar_ctx            0            0K            0
              isci            0            0K            0
      iommu_dmamap            0            0K            0
     hyperv_socket            0            0K            0
           bxe_ilt            0            0K            0
        aesni_data            0            0K            0
            xenbus            0            0K            0
     vm_fictitious            0            0K            0
           UMAHash            0            0K            0
         vm_pgdata            0            0K            0
           jblocks            0            0K            0
          savedino            0            0K        56390
          sentinel            0            0K            0
            jfsync            0            0K            0
            jtrunc            0            0K            0
             sbdep            0            0K           58
           jsegdep            0            0K            0
              jseg            0            0K            0
         jfreefrag            0            0K            0
          jfreeblk            0            0K            0
           jnewblk            0            0K            0
            jmvref            0            0K            0
           jremref            0            0K            0
           jaddref            0            0K            0
           freedep            0            0K            0
        allocindir            0            0K            0
          indirdep            0            0K           10
       allocdirect            0            0K            0
          ufs_trim            0            0K            0
           mactemp            0            0K            0
     audit_trigger            0            0K            0
 audit_pipe_presel            0            0K            0
     audit_pipeent            0            0K            0
        audit_pipe            0            0K            0
      audit_evname            0            0K            0
         audit_bsm            0            0K            0
      audit_gidset            0            0K            0
        audit_text            0            0K            0
        audit_path            0            0K            0
        audit_data            0            0K            0
        audit_cred            0            0K            0
          ktls_ocf            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
      MLX5E_TLS_RX            0            0K            0
        MLX5EEPROM            0            0K            0
         MLX5E_TLS            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
            MLX5EN            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
          MLX5DUMP            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
       simple_attr            0            0K            0
          seq_file            0            0K            0
           lkpiskb            0            0K            0
             radix            0            0K            0
               idr            0            0K            0
          lkpindev            0            0K            0
           lkpimhi            0            0K            0
            lkpifw            0            0K            0
         lkpi80211            0            0K            0
               NLM            0            0K            0
    ipsec-spdcache            0            0K            0
         ipsec-reg            0            0K            0
        ipsec-misc            0            0K            0
      ipsecrequest            0            0K            0
            ip6opt            0            0K         2274
       ip6_msource            0            0K            0
      ip6_moptions            0            0K            0
       in6_mfilter            0            0K            0
             frag6            0            0K            0
            tcplog            0            0K            0
        tcp_hwpace            0            0K            0
        ip_msource            0            0K            0
       ip_moptions            0            0K            0
        in_mfilter            0            0K            0
              ipid            0            0K            0
         80211scan            0            0K            0
      80211ratectl            0            0K            0
        80211power            0            0K            0
       80211nodeie            0            0K            0
         80211node            0            0K            0
      80211mesh_gt            0            0K            0
      80211mesh_rt            0            0K            0
         80211perr            0            0K            0
         80211prep            0            0K            0
         80211preq            0            0K            0
          80211dfs            0            0K            0
       80211crypto            0            0K            0
          80211vap            0            0K            0
             iflib            0            0K            0
              vlan            0            0K            0
               gif            0            0K            0
           ifdescr            0            0K            0
              zlib            0            0K            0
           fadvise            0            0K            0
           VN POLL            0            0K            0
            statfs            0            0K         5852
     namei_tracker            0            0K          984
       export_host            0            0K            0
        cl_savebuf            0            0K            7
               lio            0            0K            0
               acl            0            0K            0
          mbuf_tag            0            0K            0
              ktls            0            0K            0
              accf            0            0K            0
               pts            0            0K            0
           timerfd            0            0K            0
          ioctlops            0            0K         6527
           eventfd            0            0K            0
           Witness            0            0K            0
             stack            0            0K            0
              sbuf            0            0K          882
          firmware            0            0K            0
        compressor            0            0K            0
              SWAP            0            0K            0
         sysctltmp            0            0K         1030
            sysctl            0            0K            3
              ekcd            0            0K            0
            dumper            0            0K            0
          sendfile            0            0K            0
              rctl            0            0K            0
             cache            0            0K            0
      prison_racct            0            0K            0
       Fail Points            0            0K            0
             sigio            0            0K            1
filedesc_to_leader            0            0K            0
               pwd            0            0K            0
       tty console            0            0K            0
         boottrace            0            0K            0
        isofs_node            0            0K            0
       isofs_mount            0            0K            0
     tr_raid5_data            0            0K            0
    tr_raid1e_data            0            0K            0
     tr_raid1_data            0            0K            0
     tr_raid0_data            0            0K            0
    tr_concat_data            0            0K            0
       md_sii_data            0            0K            0
   md_promise_data            0            0K            0
    md_nvidia_data            0            0K            0
   md_jmicron_data            0            0K            0
     md_intel_data            0            0K            0
       md_ddf_data            0            0K            0
         raid_data            0            0K           72
     geom_flashmap            0            0K            0
         tmpfs dir            0            0K            0
        tmpfs name            0            0K            0
       tmpfs mount            0            0K            0
     tmpfs extattr            0            0K            0
           NFS FHA            0            0K            0
         newnfsmnt            0            0K            0
  newnfsclient_req            0            0K            0
   NFSCL layrecall            0            0K            0
     NFSCL session            0            0K            0
     NFSCL sockreq            0            0K            0
     NFSCL devinfo            0            0K            0
     NFSCL flayout            0            0K            0
      NFSCL layout            0            0K            0
     NFSD rollback            0            0K            0
      NFSCL diroff            0            0K            0
       NEWdirectio            0            0K            0
        NEWNFSnode            0            0K            0
         NFSCL lck            0            0K            0
      NFSCL lckown            0            0K            0
      NFSCL client            0            0K            0
       NFSCL deleg            0            0K            0
        NFSCL open            0            0K            0
       NFSCL owner            0            0K            0
            NFS fh            0            0K            0
           NFS req            0            0K            0
     NFSD usrgroup            0            0K            0
       NFSD string            0            0K            0
       NFSD V4lock            0            0K            0
      NFSD V4state            0            0K            0
       msdosfs_fat            0            0K            0
     msdosfs_mount            0            0K            0
      msdosfs_node            0            0K            0
            DEVFS4            0            0K            0
            DEVFS2            0            0K            0
            gntdev            0            0K            0
       privcmd_dev            0            0K            0
        evtchn_dev            0            0K            0
          xenstore            0            0K            0
               xnb            0            0K            0
          xen_acpi            0            0K            0
              xbbd            0            0K            0
               xbd            0            0K            0
           Balloon            0            0K            0
          sysmouse            0            0K            0
            vtfont            0            0K            0
            pvscsi            0            0K            0
            USBdev            0            0K            0
               USB            0            0K            0
            twsbuf            0            0K            0
       tcp_log_dev            0            0K            0
      midi buffers            0            0K            0
             mixer            0            0K            0
              ac97            0            0K            0
             hdacc            0            0K            0
              hdac            0            0K            0
              hdaa            0            0K            0
       SIIS driver            0            0K            0
               PUC            0            0K            0
          ppbusdev            0            0K            0
            sr_iov            0            0K            0
               OCS            0            0K            0
               OCS            0            0K            0
              nvme            0            0K            0
               nvd            0            0K            0
            netmap            0            0K            0
            mwldev            0            0K            0
        MVS driver            0            0K            0
          mrsasbuf            0            0K            0
          mpt_user            0            0K            0
          mps_user            0            0K            0
            MPSSAS            0            0K            0
               mps            0            0K            0
          mpr_user            0            0K            0
            MPRSAS            0            0K            0
               mpr            0            0K            0
            mfibuf            0            0K            0
        md_sectors            0            0K            0
           md_disk            0            0K         1105
           malodev            0            0K            0
               LED            0            0K            0
          ix_sriov            0            0K            0
                ix            0            0K            0
            ipsbuf            0            0K            0
         ciss_data            0            0K            0
         BACKLIGHT            0            0K            0
           ath_hal            0            0K            0
            athdev            0            0K            0
           ata_pci            0            0K            0
           ata_dma            0            0K            0
       ata_generic            0            0K            0
       AHCI driver            0            0K            0
               agp            0            0K            0
           acpipwr            0            0K            0
         acpi_perf            0            0K            0
         acpicmbat            0            0K            0
        aacraidcam            0            0K            0
       aacraid_buf            0            0K            0
            aaccam            0            0K            0
            aacbuf            0            0K            0
              zstd            0            0K            0
            XZ_DEC            0            0K            0
            nvlist            0            0K            0
          SCSI ENC            0            0K            0
           SCSI sa            0            0K            0
         scsi_pass            0            0K            0
           scsi_da            0            0K           69
            ata_da            0            0K            0
           scsi_ch            0            0K            0
           scsi_cd            0            0K            0
           nvme_da            0            0K            0
           CAM CCB            0            0K          523
     CAM ccb queue            0            0K            0
db> show uma
              Zone   Size    Used    Free    Requests  Sleeps  Bucket  Total Mem    XFree
   mbuf_jumbo_page   4096    8510    1650      166485       0     254   41615360        0
              mbuf    256   14884    3150      768012       0     254    4616704        0
         sctp_asoc   2256     598     927       15765       0     254    3440400        0
       malloc-2048   2048     602     798       76026       0       8    2867200        0
      mbuf_cluster   2048    1016     254        1037       0     254    2600960        0
           sctp_ep   1176     599    1181       75999       0     254    2093280        0
        malloc-128    128   14489     143       14583       0     126    1872896        0
          BUF TRIE    144     246   11570       12545       0      62    1701504        0
        malloc-384    384    4166      34        4168       0      30    1612800        0
         FFS inode   1168     565     814      146684       0       9    1610672        0
        malloc-384    384    1657    2363      153215       0      30    1543680        0
       UMA Slabs 0    112   13267      11       13267       0     126    1487136        0
       malloc-1024   1024     591     785       20206       0      16    1409024        0
       malloc-4096   4096     327       3         491       0       2    1351680        0
         vmem btag     56   22179     108       22179       0     254    1248072        0
        RADIX NODE    144    7745     678     2739697       0      64    1212912        0
        sctp_raddr    736     625     904       17415       0     254    1125344        0
        malloc-384    384    1225    1565       92116       0      30    1071360        0
        malloc-256    256    2268    1887      359194       0      62    1063680        0
zio_buf_comb_1048576 1048576       0       1          15       0       1    1048576        0
        malloc-256    256     194    3286       60672       0      62     890880        0
            socket    960      43     721      231382       0     254     733440        0
       malloc-8192   8192      14      70       11327       0       1     688128        0
             VNODE    448     604     872      146725       0      30     661248        0
        malloc-128    128    2653    2400      148373       0     126     646784        0
         VM OBJECT    264    1536     834     1440171       0      30     625680        0
        malloc-256    256    2232      63        2824       0      62     587520        0
        256 Bucket   2048     234      38        3056       0       8     557056        0
          lkpicurr    168       2    3094           2       0      62     520128        0
              pbuf   2624       0     198           0       0       2     519552        0
      malloc-65536  65536       2       5         428       0       1     458752        0
            THREAD   1824     194      30      155249       0       8     408576        0
         malloc-64     64    5208     651      262227       0     254     374976        0
       FFS2 dinode    256     565     875      146684       0      62     368640        0
      malloc-65536  65536       5       0           5       0       1     327680        0
        sctp_chunk    152     679    1375        3058       0     254     312208        0
         malloc-16     16   18001     499       21030       0     254     296000        0
       malloc-4096   4096      65       7      128471       0       2     294912        0
       mbuf_packet    256     376     640      197167       0     254     260096        0
         malloc-32     32    7040     646      135113       0     254     245952        0
         UMA Zones    768     302       2         302       0      16     233472        0
            DEVCTL   1024       0     220         152       0       0     225280        0
      malloc-16384  16384       8       5      126487       0       1     212992        0
         tcp_inpcb   1312      12     150       14385       0       8     212544        0
        malloc-256    256      55     755      141685       0      62     207360        0
     FPU_save_area    832     196      47      169255       0      16     202176        0
      malloc-65536  65536       1       2         477       0       1     196608        0
      malloc-32768  32768       4       2         124       0       1     196608        0
             unpcb    256      20     745       98089       0     254     195840        0
       malloc-4096   4096      43       3         147       0       2     188416        0
         MAP ENTRY     96    1517     373     3780156       0     126     181440        0
        malloc-128    128    1132     263       29213       0     126     178560        0
            lkpimm     56       1    3095           1       0     254     173376        0
       S VFS Cache    104    1125     396      169338       0     126     158184        0
         64 Bucket    512     196     100       20614       0      30     151552        0
       malloc-1024   1024     127      17         220       0      16     147456        0
        128 Bucket   1024      89      42       25194       0      16     134144        0
zio_buf_comb_131072 131072       0       1           1       0       1     131072        0
      malloc-65536  65536       0       2         110       0       1     131072        0
        malloc-256    256     345     165      435740       0      62     130560        0
         32 Bucket    256     281     229       18695       0      62     130560        0
              PROC   1376      63      25      128468       0       8     121088        0
              pipe    728      23     142       13827       0      16     120120        0
sctp_stream_msg_out    112     221     823        7336       0     254     116928        0
          ksiginfo    112      79     965       14284       0     126     116928        0
         malloc-64     64     656    1171     1691430       0     254     116928        0
      malloc-16384  16384       5       2          12       0       1     114688        0
          UMA Kegs    384     289       4         289       0      30     112512        0
        malloc-256    256     358      77        3041       0      62     111360        0
         malloc-32     32    1567    1835       95398       0     254     108864        0
      malloc-32768  32768       0       3           3       0       1      98304        0
             g_bio    408       4     236       84946       0      30      97920        0
         filedesc0   1072      64      27      128469       0       8      97552        0
     udplite_inpcb    424       0     216       22666       0      30      91584        0
       malloc-8192   8192       7       4         101       0       1      90112        0
       malloc-2048   2048       7      33        2542       0       8      81920        0
        malloc-256    256      18     297       15755       0      62      80640        0
             Files     80     233     667      493992       0     126      72000        0
             ripcb    392       5     175       19806       0      30      70560        0
         malloc-64     64     377     694       37591       0     254      68544        0
         malloc-64     64     514     557        3949       0     254      68544        0
        malloc-128    128     129     398      403784       0     126      67456        0
      malloc-65536  65536       1       0           1       0       1      65536        0
      malloc-65536  65536       1       0           1       0       1      65536        0
      malloc-65536  65536       0       1           8       0       1      65536        0
      malloc-32768  32768       2       0           2       0       1      65536        0
      malloc-32768  32768       2       0           2       0       1      65536        0
       malloc-2048   2048      20      12         845       0       8      65536        0
        malloc-256    256      56     199      147201       0      62      65280        0
       malloc-8192   8192       6       1           8       0       1      57344        0
       malloc-8192   8192       2       5           9       0       1      57344        0
       malloc-4096   4096       3      11        1246       0       2      57344        0
         16 Bucket    144     174     218        7553       0      62      56448        0
             AIOCB    552       0      98        8177       0      16      54096        0
         udp_inpcb    424       6     120         376       0      30      53424        0
         malloc-64     64     213     606      133569       0     254      52416        0
         malloc-16     16    1240    2010      108209       0     254      52000        0
         malloc-16     16    1499    1751       31863       0     254      52000        0
           VMSPACE    616      39      45      127345       0      16      51744        0
        malloc-128    128      57     346      145963       0     126      51584        0
        malloc-128    128      96     307       14434       0     126      51584        0
        malloc-128    128     259     144         694       0     126      51584        0
            cpuset    200       7     249        6342       0      62      51200        0
zio_data_buf_16384  16384       0       3           9       0       1      49152        0
           DIRHASH   1024      35      13          35       0      16      49152        0
             NAMEI   1024       0      48      834571       0      16      49152        0
       malloc-8192   8192       4       2         105       0       1      49152        0
       malloc-2048   2048       2      22         565       0       8      49152        0
       malloc-1024   1024      11      37        7506       0      16      49152        0
        malloc-512    512       9      87        6423       0      30      49152        0
           pcpu-64     64     495     273         495       0     254      49152        0
              PGRP    120      35     361       13669       0     126      47520        0
        malloc-384    384      91      29         110       0      30      46080        0
        malloc-384    384      65      55         408       0      30      46080        0
            da_ccb    544       1      83       21403       0      16      45696        0
            pcpu-8      8    5232     400        5388       0     254      45056        0
               PWD     40      20    1091      126470       0     254      44440        0
          syncache    168       0     264           4       0     254      44352        0
          8 Bucket     80     165     385       15853       0     126      44000        0
         TURNSTILE    136     225      90         225       0      62      42840        0
       malloc-4096   4096       5       5        1668       0       2      40960        0
        sctp_laddr     48     504     336        1567       0     254      40320        0
         malloc-64     64      29     538         865       0     254      36288        0
         malloc-64     64       9     558         425       0     254      36288        0
         malloc-64     64     158     409         188       0     254      36288        0
         malloc-32     32     142     992       64154       0     254      36288        0
        malloc-128    128      91     188       10719       0     126      35712        0
     routing nhops    256      27     108          34       0      62      34560        0
           ttyoutq    256      72      63         160       0      62      34560        0
        SLEEPQUEUE     88     225     159         225       0     126      33792        0
     zio_buf_16384  16384       0       2          10       0       1      32768        0
      malloc-32768  32768       1       0           1       0       1      32768        0
       malloc-4096   4096       4       4        5860       0       2      32768        0
       malloc-2048   2048      13       3          25       0       8      32768        0
       malloc-2048   2048       9       7          58       0       8      32768        0
       malloc-2048   2048       5      11          10       0       8      32768        0
       malloc-1024   1024       2      30          42       0      16      32768        0
       malloc-1024   1024       6      26         187       0      16      32768        0
       malloc-1024   1024      18      14          22       0      16      32768        0
       malloc-1024   1024      18      14        2153       0      16      32768        0
       malloc-1024   1024       1      31          10       0      16      32768        0
        malloc-512    512      16      48          90       0      30      32768        0
        malloc-512    512       0      64        2401       0      30      32768        0
        malloc-512    512       0      64          91       0      30      32768        0
        malloc-512    512       3      61          54       0      30      32768        0
        malloc-512    512       5      59         144       0      30      32768        0
        malloc-512    512       2      62           3       0      30      32768        0
    ertt_txseginfo     40       0     808      171997       0     254      32320        0
             KNOTE    160      30     170      872322       0      62      32000        0
            ttyinq    160     135      65         300       0      62      32000        0
   tcp_log_id_node    120       7     257          83       0     126      31680        0
            clpbuf   2624       0      12         137       0       4      31488        0
udplite_inpcb ports     32       0     882        4322       0     254      28224        0
   tcp_inpcb ports     32       3     879        1125       0     254      28224        0
              ertt     72       5     387       14385       0     126      28224        0
         malloc-32     32     103     779       12287       0     254      28224        0
          4 Bucket     48       5     583          22       0     254      28224        0
          2 Bucket     32     110     772       14603       0     254      28224        0
               AIO    208       0     133        3944       0      62      27664        0
 tcp_log_id_bucket    176       1     137          17       0      62      24288        0
          rl_entry     40     105     501         105       0     254      24240        0
           rtentry    168      30     114          34       0      62      24192        0
            itimer    352       0      66         191       0      30      23232        0
        malloc-384    384       1      59           2       0      30      23040        0
        malloc-384    384      23      37          25       0      30      23040        0
        malloc-384    384       1      59         178       0      30      23040        0
         domainset     40       0     567        5181       0     254      22680        0
         hostcache     64       1     314           1       0     254      20160        0
   udp_inpcb ports     32       3     627          33       0     254      20160        0
         malloc-32     32      33     597        3049       0     254      20160        0
         malloc-32     32     166     464       49183       0     254      20160        0
         malloc-32     32      10     620       15172       0     254      20160        0
         malloc-32     32      43     587        3049       0     254      20160        0
       L VFS Cache    320       0      60          37       0      30      19200        0
 epoch_record pcpu    256       4      60           4       0      62      16384        0
      malloc-16384  16384       1       0           1       0       1      16384        0
      malloc-16384  16384       1       0           1       0       1      16384        0
      malloc-16384  16384       1       0           1       0       1      16384        0
       malloc-8192   8192       2       0           2       0       1      16384        0
       malloc-4096   4096       0       4           6       0       2      16384        0
       malloc-2048   2048       1       7         194       0       8      16384        0
        malloc-512    512       1      31         123       0      30      16384        0
           SMR CPU     32       7     504           7       0     254      16352        0
      vtnet_tx_hdr     24       0     668      179631       0     254      16032        0
         malloc-16     16      62     938      186850       0     254      16000        0
              kenv    258      17      43        1093       0      30      15480        0
            mqnode    416       3      33           3       0      30      14976        0
              vmem   1856       1       7           1       0       8      14848        0
        SMR SHARED     24       7     504           7       0     254      12264        0
arc_buf_hdr_t_full    368       4      29           4       0      30      12144        0
         malloc-16     16      54     696      173681       0     254      12000        0
         malloc-16     16      36     714       24296       0     254      12000        0
         malloc-16     16       8     742          66       0     254      12000        0
         malloc-16     16      26     724       27765       0     254      12000        0
       Mountpoints   2816       2       2           2       0       4      11264        0
       malloc-8192   8192       1       0           1       0       1       8192        0
       malloc-8192   8192       1       0           1       0       1       8192        0
           pcpu-16     16       4     252           4       0     254       4096        0
       UMA Slabs 1    176      10      12          10       0      62       3872        0
        KMAP ENTRY     96      18      21          28       0       0       3744        0
   zfs_znode_cache    464       0       0           0       0      30          0        0
       sio_cache_2    168       0       0           0       0      62          0        0
       sio_cache_1    152       0       0           0       0      62          0        0
       sio_cache_0    136       0       0           0       0      62          0        0
     zil_zcw_cache     80       0       0           0       0     126          0        0
     zil_lwb_cache    376       0       0           0       0      30          0        0
    dmu_buf_impl_t    392       0       0           0       0      30          0        0
         arc_buf_t     32       0       0           0       0     254          0        0
arc_buf_hdr_t_l2only     96       0       0           0       0     126          0        0
           dnode_t    960       0       0           0       0      16          0        0
          sa_cache    272       0       0           0       0      30          0        0
         abd_chunk   4096       0       0           0       0       2          0        0
zio_buf_comb_16777216 16777216       0       0           0       0       1          0        0
zio_buf_comb_12582912 12582912       0       0           0       0       1          0        0
zio_buf_comb_8388608 8388608       0       0           0       0       1          0        0
zio_buf_comb_6291456 6291456       0       0           0       0       1          0        0
zio_buf_comb_4194304 4194304       0       0           0       0       1          0        0
zio_buf_comb_3145728 3145728       0       0           0       0       1          0        0
zio_buf_comb_2097152 2097152       0       0           0       0       1          0        0
zio_buf_comb_1572864 1572864       0       0           0       0       1          0        0
zio_buf_comb_786432 786432       0       0           0       0       1          0        0
zio_buf_comb_524288 524288       0       0           0       0       1          0        0
zio_buf_comb_393216 393216       0       0           0       0       1          0        0
zio_buf_comb_262144 262144       0       0           0       0       1          0        0
zio_buf_comb_196608 196608       0       0           0       0       1          0        0
zio_buf_comb_98304  98304       0       0           0       0       1          0        0
zio_buf_comb_65536  65536       0       0           0       0       1          0        0
zio_buf_comb_49152  49152       0       0           0       0       1          0        0
zio_buf_comb_32768  32768       0       0           0       0       1          0        0
zio_buf_comb_24576  24576       0       0           0       0       1          0        0
zio_data_buf_12288  12288       0       0           0       0       1          0        0
     zio_buf_12288  12288       0       0           0       0       1          0        0
 zio_data_buf_8192   8192       0       0           0       0       1          0        0
      zio_buf_8192   8192       0       0           0       0       1          0        0
 zio_data_buf_6144   6144       0       0           0       0       1          0        0
      zio_buf_6144   6144       0       0           0       0       1          0        0
 zio_data_buf_4096   4096       0       0           0       0       2          0        0
      zio_buf_4096   4096       0       0           0       0       2          0        0
 zio_data_buf_3072   3072       0       0           0       0       4          0        0
      zio_buf_3072   3072       0       0           0       0       4          0        0
 zio_data_buf_2048   2048       0       0           0       0       8          0        0
      zio_buf_2048   2048       0       0           0       0       8          0        0
 zio_data_buf_1536   1536       0       0           0       0       8          0        0
      zio_buf_1536   1536       0       0           0       0       8          0        0
 zio_data_buf_1024   1024       0       0           0       0      16          0        0
      zio_buf_1024   1024       0       0           0       0      16          0        0
  zio_data_buf_512    512       0       0           0       0      30          0        0
       zio_buf_512    512       0       0           0       0      30          0        0
    zio_link_cache     48       0       0           0       0     254          0        0
         zio_cache   1216       0       0           0       0       8          0        0
   ddt_entry_cache    392       0       0           0       0      30          0        0
         ddt_cache  24824       0       0           0       0       1          0        0
brt_pending_entry_cache    160       0       0           0       0      62          0        0
   brt_entry_cache     40       0       0           0       0     254          0        0
metaslab_alloc_trace_cache     72       0       0           0       0     126          0        0
zfs_btree_leaf_cache   4096       0       0           0       0       2          0        0
   reference_cache     48       0       0           0       0     254          0        0
        taskq_zone    160       0       0           0       0      62          0        0
       FFS1 dinode    128       0       0           0       0     126          0        0
           ada_ccb    272       0       0           0       0      30          0        0
             swblk    136       0       0           0       0      62          0        0
          swpctrie    144       0       0           0       0      62          0        0
   cdg_qdiffsample     16       0       0           0       0     254          0        0
   pf state scrubs     40       0       0           0       0     254          0        0
   pf frag entries     40       0       0           0       0     254          0        0
          pf frags    248       0       0           0       0      62          0        0
  pf table entries    160       0       0           0       0     254          0        0
pf table entry counters     64       0       0           0       0     254          0        0
   pf source nodes    152       0       0           0       0     254          0        0
     pf state keys     88       0       0           0       0     126          0        0
         pf states    360       0       0           0       0     254          0        0
           pf tags    104       0       0           0       0     126          0        0
          pf mtags    184       0       0           0       0      62          0        0
       tcp_bbr_pcb    832       0       0           0       0      16          0        0
       tcp_bbr_map    128       0       0           0       0     126          0        0
      tcp_rack_pcb   1024       0       0           0       0      16          0        0
      tcp_rack_map    128       0       0           0       0     126          0        0
tfo_ccache_entries     80       0       0           0       0     126          0        0
               tfo      4       0       0           0       0     254          0        0
          sackhole     32       0       0           0       0     254          0        0
               ipq     56       0       0           0       0     254          0        0
   sctp_asconf_ack     48       0       0           0       0     254          0        0
       sctp_asconf     40       0       0           0       0     254          0        0
        sctp_readq    152       0       0           0       0     254          0        0
           tcp_log    416       0       0           0       0     254          0        0
          tcpreass     48       0       0           0       0     254          0        0
       ripcb ports     32       0       0           0       0     254          0        0
    IPsec SA lft_c     16       0       0           0       0     254          0        0
            AIOLIO    272       0       0           0       0      30          0        0
        mqnotifier    216       0       0           0       0      62          0        0
            mvdata     64       0       0           0       0     254          0        0
            mqueue    248       0       0           0       0      62          0        0
        TMPFS node    232       0       0           0       0      62          0        0
           NCLNODE    608       0       0           0       0      16          0        0
     LTS VFS Cache    360       0       0           0       0      30          0        0
     STS VFS Cache    144       0       0           0       0      62          0        0
           cryptop    280       0       0           0       0      30          0        0
  linux_dma_object     32       0       0           0       0     254          0        0
  linux_dma_pctrie    144       0       0           0       0      62          0        0
   IOMMU_MAP_ENTRY    104       0       0           0       0     126          0        0
    mbuf_jumbo_16k  16384       0       0           0       0     254          0        0
     mbuf_jumbo_9k   9216       0       0           0       0     254          0        0
      audit_record   1280       0       0           0       0       8          0        0
        MAC labels     40       0       0           0       0     254          0        0
            vnpbuf   2624       0       0           0       0      16          0        0
           nfspbuf   2624       0       0           0       0       4          0        0
            swwbuf   2624       0       0           0       0       2          0        0
            swrbuf   2624       0       0           0       0       4          0        0
          umtx_shm     88       0       0           0       0     126          0        0
           umtx pi     96       0       0           0       0     126          0        0
rangeset pctrie nodes    144       0       0           0       0      62          0        0
      malloc-65536  65536       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-16384  16384       0       0           0       0       1          0        0
      malloc-16384  16384       0       0           0       0       1          0        0
      malloc-16384  16384       0       0           0       0       1          0        0
       malloc-4096   4096       0       0           0       0       2          0        0
           pcpu-32     32       0       0           0       0     254          0        0
            pcpu-4      4       0       0           0       0     254          0        0
            fakepg    104       0       0           0       0     126          0        0
          UMA Hash    256       0       0           0       0      62          0        0

Crashes (1):
Time Kernel Commit Syzkaller Config Log Report Syz repro C repro VM info Assets (help?) Manager Title
2024/03/03 07:14 freebsd-src 42b117a4bd10 25905f5d console log report ci-freebsd-i386 Fatal trap NUM: page fault in sctp_ss_default_add
* Struck through repros no longer work on HEAD.