syzbot


panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE

Status: upstream: reported C repro on 2023/07/04 17:53
Reported-by: syzbot+dce5858451a2329877ff@syzkaller.appspotmail.com
First crash: 303d, last: 242d

Sample crash report:
panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:1409
cpuid = 1
time = 1688500590
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0xc6/frame 0xfffffe0057d4d9d0
kdb_backtrace() at kdb_backtrace+0xd0/frame 0xfffffe0057d4db30
vpanic() at vpanic+0x252/frame 0xfffffe0057d4dc10
panic() at panic+0xb5/frame 0xfffffe0057d4dcd0
__mtx_assert() at __mtx_assert+0x241/frame 0xfffffe0057d4dd20
thread_suspend_check() at thread_suspend_check+0x4d/frame 0xfffffe0057d4ddb0
sig_intr() at sig_intr+0xeb/frame 0xfffffe0057d4ddf0
fork1() at fork1+0x47f/frame 0xfffffe0057d4df10
kproc_create() at kproc_create+0x168/frame 0xfffffe0057d4e0c0
kproc_kthread_add() at kproc_kthread_add+0x197/frame 0xfffffe0057d4e290
zthr_create_timer() at zthr_create_timer+0x1cb/frame 0xfffffe0057d4e380
arc_init() at arc_init+0x24bd/frame 0xfffffe0057d4e3d0
dmu_init() at dmu_init+0x36/frame 0xfffffe0057d4e3e0
spa_init() at spa_init+0xf3/frame 0xfffffe0057d4e400
zfs_kmod_init() at zfs_kmod_init+0x34/frame 0xfffffe0057d4e420
zfs_modevent() at zfs_modevent+0x4e/frame 0xfffffe0057d4e440
module_register_init() at module_register_init+0xf4/frame 0xfffffe0057d4e490
linker_load_module() at linker_load_module+0x1ca7/frame 0xfffffe0057d4ead0
kern_kldload() at kern_kldload+0x2ce/frame 0xfffffe0057d4ebb0
vfs_byname_kld() at vfs_byname_kld+0xa2/frame 0xfffffe0057d4ec70
sys_mount() at sys_mount+0x1cf/frame 0xfffffe0057d4ed30
amd64_syscall() at amd64_syscall+0x423/frame 0xfffffe0057d4ef30
fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0057d4ef30
--- syscall (0, FreeBSD ELF64, syscall), rip = 0x2b3f7a, rsp = 0x822966f78, rbp = 0x822966fb0 ---
KDB: enter: panic
[ thread pid 780 tid 100109 ]
Stopped at      kdb_enter+0x6e: movq    $0,0x2133267(%rip)
db> 
db> set $lines = 0
db> set $maxwidth = 0
db> show registers
cs                        0x20
ds                        0x3b
es                        0x3b
fs                        0x13
gs                        0x1b
ss                        0x28
rax                       0x12
rcx         0xfffffe00033eee30
rdx         0xdffff7c000000000
rbx         0xffffffff826bb8a0  .str.26
rsp         0xfffffe0057d4db10
rbp         0xfffffe0057d4db30
rsi                        0x1
rdi                          0
r8                           0
r9                  0xffffffff
r10                          0
r11                        0x6
r12         0xfffffe0058c4a020
r13         0xfffffffffffffffd
r14         0xffffffff826bb8a0  .str.26
r15                          0
rip         0xffffffff815bde2e  kdb_enter+0x6e
rflags                    0x46
kdb_enter+0x6e: movq    $0,0x2133267(%rip)
db> show proc
Process 780 (syz-executor2438980) at 0xfffffe0058ef95c0:
 state: NORMAL
 uid: 0  gids: 0, 0, 5
 parent: pid 778 at 0xfffffe006cd2bac0
 ABI: FreeBSD ELF64
 flag: 0x10084480  flag2: 0x40000
 arguments: ./syz-executor2438980472
 reaper: 0xfffffe00541df040 reapsubtree: 1
 sigparent: 20
 vmspace: 0xfffffe0058bdb820
   (map 0xfffffe0058bdb820)
   (map.pmap 0xfffffe0058bdb8e0)
   (pmap 0xfffffe0058bdb950)
 threads: 2
100108                   s                                   syz-executor2438980
100109                   Run     CPU 1                       syz-executor2438980
db> ps
  pid  ppid  pgrp   uid  state   wmesg   wchan               cmd
  780   778   778     0  T       (threaded)                  syz-executor2438980
100108                   s                                   syz-executor2438980
100109                   Run     CPU 1                       syz-executor2438980
  778   776   778     0  Ss      pause   0xfffffe006cd2bb70  csh
  776   688   776     0  Ss      select  0xfffffe005719bcc0  sshd
  754     1   754     0  Ss+     ttyin   0xfffffe0057106cb0  getty
  753     1   753     0  Ss+     ttyin   0xfffffe00571050b0  getty
  752     1   752     0  Ss+     ttyin   0xfffffe00571054b0  getty
  751     1   751     0  Ss+     ttyin   0xfffffe00571058b0  getty
  750     1   750     0  Ss+     ttyin   0xfffffe0007c900b0  getty
  749     1   749     0  Ss+     ttyin   0xfffffe0057105cb0  getty
  748     1   748     0  Ss+     ttyin   0xfffffe0007c904b0  getty
  747     1   747     0  Ss+     ttyin   0xfffffe0007c908b0  getty
  746     1   746     0  Ss+     ttyin   0xfffffe0007c90cb0  getty
  692     1   692     0  Ss      nanslp  0xffffffff836d2840  cron
  688     1   688     0  Ss      select  0xfffffe005719bf40  sshd
  501     1   501     0  Ss      select  0xfffffe00571a01c0  syslogd
  430     1   430     0  Ss      select  0xfffffe0057a6d9c0  devd
  429     1   429    65  Ss      select  0xfffffe0057a6d8c0  dhclient
  344     1   344     0  Ss      select  0xfffffe0057a6db40  dhclient
  341     1   341     0  Ss      select  0xfffffe0057a6dac0  dhclient
   17     0     0     0  DL      vlruwt  0xfffffe005715fae0  [vnlru]
   16     0     0     0  DL      syncer  0xffffffff837d4d20  [syncer]
   15     0     0     0  DL      (threaded)                  [bufdaemon]
100079                   D       psleep  0xffffffff837d3340  [bufdaemon]
100084                   D       -       0xffffffff82c0a140  [bufspacedaemon-0]
100094                   D       sdflush 0xfffffe0058b428e8  [/ worker]
    9     0     0     0  DL      psleep  0xffffffff8380ac00  [vmdaemon]
    8     0     0     0  DL      (threaded)                  [pagedaemon]
100077                   D       psleep  0xffffffff837feab8  [dom0]
100080                   D       launds  0xffffffff837feac4  [laundry: dom0]
100081                   D       umarcl  0xffffffff81d3e580  [uma]
    7     0     0     0  DL      -       0xffffffff83496e48  [rand_harvestq]
    6     0     0     0  DL      pftm    0xffffffff83f693d0  [pf purge]
    5     0     0     0  DL      waiting 0xffffffff8457a1c0  [sctp_iterator]
    4     0     0     0  DL      (threaded)                  [cam]
100044                   D       -       0xffffffff83479340  [doneq0]
100045                   D       -       0xffffffff834792c0  [async]
100076                   D       -       0xffffffff83479140  [scanner]
   14     0     0     0  DL      seqstat 0xfffffe000795d488  [sequencer 00]
    3     0     0     0  DL      (threaded)                  [crypto]
100040                   D       crypto_ 0xffffffff837fa360  [crypto]
100041                   D       crypto_ 0xfffffe00542c4830  [crypto returns 0]
100042                   D       crypto_ 0xfffffe00542c4880  [crypto returns 1]
   13     0     0     0  DL      (threaded)                  [geom]
100035                   D       -       0xffffffff836a8640  [g_event]
100036                   D       -       0xffffffff836a8660  [g_up]
100037                   D       -       0xffffffff836a8680  [g_down]
    2     0     0     0  WL      (threaded)                  [clock]
100029                   I                                   [clock (0)]
100030                   I                                   [clock (1)]
   12     0     0     0  WL      (threaded)                  [intr]
100016                   I                                   [swi5: fast taskq]
100019                   I                                   [swi6: task queue]
100020                   I                                   [swi6: Giant taskq]
100031                   I                                   [swi1: netisr 0]
100032                   I                                   [swi1: hpts]
100033                   I                                   [swi1: hpts]
100046                   I                                   [irq24: virtio_pci0]
100047                   I                                   [irq25: virtio_pci0]
100048                   I                                   [irq26: virtio_pci0]
100049                   I                                   [irq27: virtio_pci0]
100050                   I                                   [irq28: virtio_pci1]
100051                   I                                   [irq29: virtio_pci1]
100052                   I                                   [irq30: virtio_pci1]
100053                   I                                   [irq31: virtio_pci1]
100054                   I                                   [irq32: virtio_pci1]
100059                   I                                   [irq33: virtio_pci2]
100060                   I                                   [irq34: virtio_pci2]
100061                   I                                   [irq35: virtio_pci2]
100063                   I                                   [irq1: atkbd0]
100064                   I                                   [irq12: psm0]
100065                   I                                   [swi0: uart uart++]
100069                   I                                   [swi1: pf send]
   11     0     0     0  RL      (threaded)                  [idle]
100003                   Run     CPU 0                       [idle: cpu0]
100004                   CanRun                              [idle: cpu1]
    1     0     1     0  SLs     wait    0xfffffe00541df040  [init]
   10     0     0     0  DL      audit_w 0xffffffff837fada0  [audit]
    0     0     0     0  DLs     (threaded)                  [kernel]
100000                   D       swapin  0xffffffff836a9060  [swapper]
100005                   D       -       0xfffffe005408c000  [if_io_tqg_0]
100006                   D       -       0xfffffe005408be00  [if_io_tqg_1]
100007                   D       -       0xfffffe005408bd00  [if_config_tqg_0]
100008                   D       -       0xfffffe005408bc00  [softirq_0]
100009                   D       -       0xfffffe005408bb00  [softirq_1]
100010                   D       -       0xfffffe0007966500  [linuxkpi_irq_wq]
100011                   D       -       0xfffffe0007966400  [thread taskq]
100012                   D       -       0xfffffe0007966300  [inm_free taskq]
100013                   D       -       0xfffffe0007966200  [aiod_kick taskq]
100014                   D       -       0xfffffe0007966100  [deferred_unmount ta]
100015                   D       -       0xfffffe0007966000  [in6m_free taskq]
100017                   D       -       0xfffffe0007965d00  [kqueue_ctx taskq]
100018                   D       -       0xfffffe0007965c00  [pci_hp taskq]
100021                   D       -       0xfffffe0007965900  [linuxkpi_short_wq_0]
100022                   D       -       0xfffffe0007965900  [linuxkpi_short_wq_1]
100023                   D       -       0xfffffe0007965900  [linuxkpi_short_wq_2]
100024                   D       -       0xfffffe0007965900  [linuxkpi_short_wq_3]
100025                   D       -       0xfffffe0007965800  [linuxkpi_long_wq_0]
100026                   D       -       0xfffffe0007965800  [linuxkpi_long_wq_1]
100027                   D       -       0xfffffe0007965800  [linuxkpi_long_wq_2]
100028                   D       -       0xfffffe0007965800  [linuxkpi_long_wq_3]
100034                   D       -       0xfffffe0007965700  [firmware taskq]
100038                   D       -       0xfffffe0007965500  [crypto_0]
100039                   D       -       0xfffffe0007965500  [crypto_1]
100055                   D       -       0xfffffe00571cc700  [vtnet0 rxq 0]
100056                   D       -       0xfffffe00571cc600  [vtnet0 txq 0]
100057                   D       -       0xfffffe00571cc500  [vtnet0 rxq 1]
100058                   D       -       0xfffffe00571cc400  [vtnet0 txq 1]
100062                   D       vtbslp  0xfffffe00571a0380  [virtio_balloon]
100066                   D       -       0xffffffff826c0840  [deadlkres]
100070                   D       -       0xfffffe0007c8c200  [acpi_task_0]
100071                   D       -       0xfffffe0007c8c200  [acpi_task_1]
100072                   D       -       0xfffffe0007c8c200  [acpi_task_2]
100073                   D       -       0xfffffe0007967000  [mca taskq]
100075                   D       -       0xfffffe0007965300  [CAM taskq]
100110                   D       -       0xfffffe0007967400  [system_taskq_0]
100111                   D       -       0xfffffe0007967400  [system_taskq_1]
100112                   D       -       0xfffffe0007c8c000  [system_delay_taskq_]
100113                   D       -       0xfffffe0007c8c000  [system_delay_taskq_]
100114                   D       -       0xfffffe0007967e00  [arc_prune]
db> show all locks
Process 780 (syz-executor2438980) thread 0xfffffe0058c4a020 (100109)
exclusive sleep mutex process lock (process lock) r = 0 (0xfffffe0058ef96e8) locked @ /syzkaller/managers/main/kernel/sys/kern/kern_sig.c:3561
shared sx killpg racer (killpg racer) r = 0 (0xffffffff836aa318) locked @ /syzkaller/managers/main/kernel/sys/kern/kern_fork.c:957
exclusive sleep mutex Giant (Giant) r = 1 (0xffffffff82c09bc0) locked @ /syzkaller/managers/main/kernel/sys/kern/kern_module.c:116
db> show malloc
              Type        InUse        MemUse     Requests
           pf_hash            5        11524K            5
            linker          348         9249K          384
          tcp_hpts            7         4801K            7
            devbuf         4219         4324K         4247
         sysctloid        46176         2718K        46264
           solaris          111         2130K          111
             vtbuf           24         1968K           46
              kobj          326         1304K          488
            newblk          762         1215K          817
          vfscache            3         1025K            3
               pcb           23          669K           42
          inodedep            9          515K           71
         ufs_quota            1          512K            1
          vfs_hash            1          512K            1
           callout            2          512K            2
              intr            4          472K            4
            acpica         1674          184K        57877
           subproc           93          169K          837
           tidhash            3          141K            3
              vmem            3          134K            4
           pagedep            3          129K           18
        tfo_ccache            1          128K            1
          IP reass            1          128K            1
         vnet_data            1          112K            1
               sem            4          106K            4
            DEVFS1          105          105K          114
               bus          988           81K         5135
          mtx_pool            2           72K            2
     NFSD srvcache            3           68K            3
          syncache            1           68K            1
            module          512           64K          512
          acpitask            1           64K            1
       ddb_capture            1           64K            1
              temp           20           53K         1618
         hostcache            1           32K            1
               shm            1           32K            1
           kdtrace          152           32K          896
            DEVFS3          124           31K          134
              umtx          242           31K          242
               msg            4           30K            4
            kbdmux            6           28K            6
        gtaskqueue           18           26K           18
        DEVFS_RULE           56           20K           56
               BPF           10           18K           10
         ufs_mount            4           17K            5
              proc            3           17K            3
               tty           16           16K           16
           ithread           97           16K           97
            bus-sc           34           15K         1648
      eventhandler          159           13K          159
            KTRACE          100           13K          100
              kenv           95           12K           95
            ifaddr           30           12K           32
              rman           88           11K          431
              GEOM           61           11K          485
          routetbl           50           11K          176
         CAM queue            5           11K         1528
        kstat_data            9            9K            9
               rpc            4            9K            4
              UART           12            9K           12
           devstat            4            9K            4
              ksem            1            8K            1
       pfs_vncache            1            8K            1
         bmsafemap            1            8K           41
             shmfd            1            8K            1
         taskqueue           72            8K           72
     audit_evclass          237            8K          297
               UMA          356            7K          356
            sglist            5            7K            5
           CAM DEV            3            6K          510
              cred           22            6K          234
         pfs_nodes           20            5K           20
       ufs_dirhash           24            5K           24
                vt           11            5K           11
             ifnet            3            5K            3
           memdesc            1            4K            1
               MCA           32            4K           32
            plimit           16            4K          323
          filedesc            1            4K            1
             evdev            4            4K            4
           acpisem           28            4K           28
             hhook           15            4K           17
       ether_multi           40            4K           50
           lltable           11            4K           11
          pf_ifnet            5            3K            6
         in6_multi           25            3K           25
          terminal           11            3K           11
           session           19            3K           31
            kqueue           37            3K          783
           pwddesc           37            3K          781
             clone            9            3K            9
           uidinfo            3            3K            8
        local_apic            1            2K            1
           io_apic            1            2K            1
       fpukern_ctx            2            2K            2
         ipsec-saq            2            2K            2
         proc-args           56            2K         1727
             selfd           27            2K        12435
            Unitno           27            2K           41
           CAM XPT           22            2K          543
             lockf           15            2K           22
               msi           12            2K           12
       vnodemarker            3            2K           15
       ipsecpolicy            2            2K            2
           acpidev           20            2K           20
      NFSD session            1            1K            1
           softdep            1            1K            1
            sahead            1            1K            1
          secasvar            1            1K            1
        CAM periph            4            1K          271
            select            7            1K           29
             ipsec            3            1K            3
          indirdep            3            1K            3
            CC Mem            3            1K            7
             nhops            6            1K            6
              pfil            6            1K            6
         toponodes            6            1K            6
            isadev            6            1K            6
             mount           16            1K           89
          pci_link           10            1K           10
          sctp_ifa            5            1K            6
            diradd            5            1K           36
            crypto            4            1K            4
            ip6ndp            4            1K            5
 encap_export_host           12            1K           12
               osd           13            1K           29
          in_multi            2            1K            4
              cdev            2            1K            2
    chacha20random            1            1K            1
            biobuf            1            1K            1
       inpcbpolicy           10            1K          139
          sctp_ifn            2            1K            6
      NFSD lckfile            1            1K            1
     NFSD V4client            1            1K            1
            DEVFSP            4            1K            9
             DEVFS            9            1K           10
               mld            2            1K            2
              igmp            2            1K            2
            vnodes            1            1K            1
           CAM SIM            2            1K            2
            prison            8            1K            8
            feeder            7            1K            7
             taskq            2            1K            2
           tcpfunc            3            1K            3
        loginclass            3            1K            7
       lkpikmalloc            5            1K            6
        aesni_data            2            1K            2
         cryptodev            2            1K           49
          nexusdev            8            1K            8
            apmdev            1            1K            1
          atkbddev            2            1K            2
           netlink            1            1K            1
     CAM dev queue            2            1K            2
 CAM I/O Scheduler            1            1K            1
          CAM path            4            1K         1034
          procdesc            1            1K            6
          pmchooks            1            1K            1
            soname            4            1K         3423
               tun            3            1K            3
          sctp_vrf            1            1K            1
              vnet            1            1K            1
           entropy            2            1K           35
               pmc            1            1K            1
          acpiintr            1            1K            1
              cpus            2            1K            2
          freework            1            1K           26
    vnet_data_free            1            1K            1
           Per-cpu            1            1K            1
          p1003.1b            1            1K            1
          filecaps            1            1K           66
         sfs_nodes            0            0K            0
        zones_data            0            0K            0
        sctp_mcore            0            0K            0
        sctp_socko            0            0K            0
         sctp_iter            0            0K            4
         sctp_mvrf            0            0K            0
         sctp_timw            0            0K            0
         sctp_cpal            0            0K            0
         sctp_cmsg            0            0K            0
         sctp_stre            0            0K            0
         sctp_athi            0            0K            0
         sctp_athm            0            0K            0
         sctp_atky            0            0K            0
         sctp_atcl            0            0K            0
         sctp_a_it            0            0K            4
         sctp_aadr            0            0K            0
         sctp_stro            0            0K            0
         sctp_stri            0            0K            0
          sctp_map            0            0K            0
            tcp_do            0            0K            0
           tcp_fsb            0            0K            0
            ipcomp            0            0K            0
               esp            0            0K            0
                ah            0            0K            0
            mqdata            0            0K            0
           filemon            0            0K            0
          pf_table            0            0K            0
           pf_rule            0            0K            0
           pf_altq            0            0K            0
           pf_osfp            0            0K            0
     pf_krule_item            0            0K            0
           pf_temp            0            0K            0
     md_intel_data            0            0K            0
       md_ddf_data            0            0K            0
        madt_table            0            0K            2
          smartpqi            0            0K            0
         raid_data            0            0K           72
     geom_flashmap            0            0K            0
               ixl            0            0K            0
         tmpfs dir            0            0K            0
        tmpfs name            0            0K            0
       tmpfs mount            0            0K            0
     tmpfs extattr            0            0K            0
           NFS FHA            0            0K            0
        ice-resmgr            0            0K            0
         ice-osdep            0            0K            0
               ice            0            0K            0
              iavf            0            0K            0
             axgbe            0            0K            0
         newnfsmnt            0            0K            0
  newnfsclient_req            0            0K            0
   NFSCL layrecall            0            0K            0
     NFSCL session            0            0K            0
     NFSCL sockreq            0            0K            0
     NFSCL devinfo            0            0K            0
     NFSCL flayout            0            0K            0
      NFSCL layout            0            0K            0
     NFSD rollback            0            0K            0
          xen_intr            0            0K            0
      NFSCL diroff            0            0K            0
       NEWdirectio            0            0K            0
           xen_hvm            0            0K            0
         legacydrv            0            0K            0
            bounce            0            0K            0
            busdma            0            0K            0
            qpidrv            0            0K            0
        NEWNFSnode            0            0K            0
         NFSCL lck            0            0K            0
      dmar_idpgtbl            0            0K            0
          dmar_dom            0            0K            0
          dmar_ctx            0            0K            0
      NFSCL lckown            0            0K            0
      NFSCL client            0            0K            0
       NFSCL deleg            0            0K            0
              isci            0            0K            0
      iommu_dmamap            0            0K            0
        NFSCL open            0            0K            0
     hyperv_socket            0            0K            0
           bxe_ilt            0            0K            0
       NFSCL owner            0            0K            0
            xenbus            0            0K            0
            NFS fh            0            0K            0
           NFS req            0            0K            0
     NFSD usrgroup            0            0K            0
     vm_fictitious            0            0K            0
       NFSD string            0            0K            0
       NFSD V4lock            0            0K            0
      NFSD V4state            0            0K            0
       msdosfs_fat            0            0K            0
     msdosfs_mount            0            0K            0
      msdosfs_node            0            0K            0
           UMAHash            0            0K            0
            DEVFS4            0            0K            0
         vm_pgdata            0            0K            0
           jblocks            0            0K            0
          savedino            0            0K           18
          sentinel            0            0K            0
            jfsync            0            0K            0
            jtrunc            0            0K            0
             sbdep            0            0K            4
           jsegdep            0            0K            0
              jseg            0            0K            0
         jfreefrag            0            0K            0
          jfreeblk            0            0K            0
           jnewblk            0            0K            0
            jmvref            0            0K            0
           jremref            0            0K            0
           jaddref            0            0K            0
           freedep            0            0K            0
         newdirblk            0            0K            8
            dirrem            0            0K           28
             mkdir            0            0K           16
          freefile            0            0K           26
          freeblks            0            0K           25
          freefrag            0            0K            1
        allocindir            0            0K            0
       allocdirect            0            0K            0
          ufs_trim            0            0K            0
           mactemp            0            0K            0
     audit_trigger            0            0K            0
 audit_pipe_presel            0            0K            0
     audit_pipeent            0            0K            0
        audit_pipe            0            0K            0
      audit_evname            0            0K            0
         audit_bsm            0            0K            0
      audit_gidset            0            0K            0
        audit_text            0            0K            0
        audit_path            0            0K            0
        audit_data            0            0K            0
        audit_cred            0            0K            0
            DEVFS2            0            0K            0
            gntdev            0            0K            0
       privcmd_dev            0            0K            0
        evtchn_dev            0            0K            0
          xenstore            0            0K            0
         scsi_pass            0            0K            0
         ciss_data            0            0K            0
               xnb            0            0K            0
          xen_acpi            0            0K            0
              xbbd            0            0K            0
               xbd            0            0K            0
           Balloon            0            0K            0
          sysmouse            0            0K            0
            vtfont            0            0K            0
          ktls_ocf            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
      MLX5E_TLS_RX            0            0K            0
        MLX5EEPROM            0            0K            0
         MLX5E_TLS            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
            MLX5EN            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
          MLX5DUMP            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
       simple_attr            0            0K            0
          seq_file            0            0K            0
           lkpiskb            0            0K            0
             radix            0            0K            0
               idr            0            0K            0
          lkpindev            0            0K            0
           lkpimhi            0            0K            0
            lkpifw            0            0K            0
         lkpi80211            0            0K            0
               NLM            0            0K            0
    ipsec-spdcache            0            0K            0
         ipsec-reg            0            0K            0
        ipsec-misc            0            0K            0
      ipsecrequest            0            0K            0
            ip6opt            0            0K            3
       ip6_msource            0            0K            0
      ip6_moptions            0            0K            0
       in6_mfilter            0            0K            0
             frag6            0            0K            0
            tcplog            0            0K            0
        tcp_hwpace            0            0K            0
         BACKLIGHT            0            0K            0
               LRO            0            0K            0
        ip_msource            0            0K            0
       ip_moptions            0            0K            0
        in_mfilter            0            0K            0
              ipid            0            0K            0
         80211scan            0            0K            0
      80211ratectl            0            0K            0
        80211power            0            0K            0
       80211nodeie            0            0K            0
         80211node            0            0K            0
      80211mesh_gt            0            0K            0
      80211mesh_rt            0            0K            0
         80211perr            0            0K            0
         80211prep            0            0K            0
         80211preq            0            0K            0
          80211dfs            0            0K            0
       80211crypto            0            0K            0
          80211vap            0            0K            0
             iflib            0            0K            0
              vlan            0            0K            0
               gif            0            0K            0
           ifdescr            0            0K            0
              zlib            0            0K            0
           fadvise            0            0K            0
           VN POLL            0            0K            0
           ath_hal            0            0K            0
            statfs            0            0K          197
     namei_tracker            0            0K            0
       export_host            0            0K            0
        cl_savebuf            0            0K            3
            athdev            0            0K            0
           ata_pci            0            0K            0
           ata_dma            0            0K            0
       ata_generic            0            0K            0
            pvscsi            0            0K            0
           scsi_da            0            0K           69
            ata_da            0            0K            0
           scsi_ch            0            0K            0
           scsi_cd            0            0K            0
       AHCI driver            0            0K            0
            USBdev            0            0K            0
               USB            0            0K            0
               agp            0            0K            0
           nvme_da            0            0K            0
           acpipwr            0            0K            0
         acpi_perf            0            0K            0
            twsbuf            0            0K            0
       tcp_log_dev            0            0K            0
               aio            0            0K            0
               lio            0            0K            0
               acl            0            0K            0
      midi buffers            0            0K            0
          mbuf_tag            0            0K            0
              ktls            0            0K            0
             mixer            0            0K            0
              ac97            0            0K            0
             hdacc            0            0K            0
              hdac            0            0K            0
              hdaa            0            0K            0
         acpicmbat            0            0K            0
       SIIS driver            0            0K            0
           CAM CCB            0            0K          523
               PUC            0            0K            0
          ppbusdev            0            0K            0
            sr_iov            0            0K            0
               OCS            0            0K            0
               OCS            0            0K            0
              nvme            0            0K            0
               nvd            0            0K            0
            netmap            0            0K            0
            mwldev            0            0K            0
        MVS driver            0            0K            0
     CAM ccb queue            0            0K            0
              accf            0            0K            0
               pts            0            0K            0
               iov            0            0K        13741
          ioctlops            0            0K           86
           eventfd            0            0K            0
           Witness            0            0K            0
             stack            0            0K            0
          mrsasbuf            0            0K            0
          mpt_user            0            0K            0
          mps_user            0            0K            0
            MPSSAS            0            0K            0
               mps            0            0K            0
              sbuf            0            0K          288
          mpr_user            0            0K            0
          firmware            0            0K            0
        compressor            0            0K            0
            MPRSAS            0            0K            0
              SWAP            0            0K            0
               mpr            0            0K            0
            mfibuf            0            0K            0
         sysctltmp            0            0K          650
            sysctl            0            0K            3
        md_sectors            0            0K            0
              ekcd            0            0K            0
            dumper            0            0K            0
          sendfile            0            0K            0
              rctl            0            0K            0
           md_disk            0            0K            0
           malodev            0            0K            0
               LED            0            0K            0
          ix_sriov            0            0K            0
             cache            0            0K            0
        aacraidcam            0            0K            0
          kcovinfo            0            0K            0
      prison_racct            0            0K            0
       Fail Points            0            0K            0
             sigio            0            0K            1
filedesc_to_leader            0            0K            0
               pwd            0            0K            0
       tty console            0            0K            0
                ix            0            0K            0
            ipsbuf            0            0K            0
       aacraid_buf            0            0K            0
            aaccam            0            0K            0
         boottrace            0            0K            0
            aacbuf            0            0K            0
              zstd            0            0K            0
            XZ_DEC            0            0K            0
            nvlist            0            0K            0
          SCSI ENC            0            0K            0
           SCSI sa            0            0K            0
        isofs_node            0            0K            0
       isofs_mount            0            0K            0
     tr_raid5_data            0            0K            0
    tr_raid1e_data            0            0K            0
     tr_raid1_data            0            0K            0
     tr_raid0_data            0            0K            0
    tr_concat_data            0            0K            0
       md_sii_data            0            0K            0
   md_promise_data            0            0K            0
    md_nvidia_data            0            0K            0
   md_jmicron_data            0            0K            0
db> show uma
              Zone   Size    Used    Free    Requests  Sleeps  Bucket  Total Mem    XFree
   mbuf_jumbo_page   4096    8320    1078       15527       0     254   38494208        0
              mbuf    256    8577    1085       18801       0     254    2473472        0
        malloc-128    128   15119     257       15384       0     126    1968128        0
          BUF TRIE    144     222   11594         626       0      62    1701504        0
        malloc-384    384    4210      80        4283       0      30    1647360        0
       malloc-4096   4096     328       4         565       0       2    1359872        0
       UMA Slabs 0    112   10561      17       10561       0     126    1184736        0
      mbuf_cluster   2048     508       0         508       0     254    1040384        0
         vmem btag     56   15481     110       15481       0     254     873096        0
         FFS inode   1160     484      34         510       0       8     600880        0
        RADIX NODE    144    3810     189       24721       0      62     575856        0
              pbuf   2624       0     198           0       0       2     519552        0
            socket    960      18     490        1319       0     254     487680        0
            lkpimm    168       1    2327           1       0      62     391104        0
          lkpicurr    168       2    2326           2       0      62     391104        0
         malloc-64     64    4951     404        4952       0     254     342720        0
         malloc-16     16   18981     519       19057       0     254     312000        0
      malloc-65536  65536       4       0           4       0       1     262144        0
      malloc-65536  65536       2       2          48       0       1     262144        0
        256 Bucket   2048     114      14         870       0       8     262144        0
             VNODE    448     514      62         542       0      30     258048        0
        malloc-256    256     947      58        1177       0      62     257280        0
         VM OBJECT    264     876      84       14330       0      30     253440        0
         UMA Zones    768     328       1         328       0      16     252672        0
         malloc-32     32    7163     271        7173       0     254     237888        0
            DEVCTL   1024       0     220         124       0       0     225280        0
       malloc-4096   4096      51       3        1335       0       2     221184        0
            THREAD   1824     114       7         114       0       8     220704        0
      malloc-65536  65536       2       1          10       0       1     196608        0
        malloc-128    128    1324     195       27350       0     126     194432        0
       FFS2 dinode    256     484      86         510       0      62     145920        0
      malloc-65536  65536       0       2         126       0       1     131072        0
      malloc-32768  32768       2       2         122       0       1     131072        0
       malloc-2048   2048       4      60        1036       0       8     131072        0
       malloc-1024   1024     116      12         272       0      16     131072        0
             unpcb    256       7     503        1163       0     254     130560        0
       mbuf_packet    256       0     508          97       0     254     130048        0
          UMA Kegs    384     315       8         315       0      30     124032        0
       S VFS Cache    104     972     198        1012       0     126     121680        0
         MAP ENTRY     96     811     449       40825       0     126     120960        0
          ksiginfo    112      31    1013          48       0     126     116928        0
        malloc-128    128     632     267        1222       0     126     115072        0
     FPU_save_area    832     116      19         130       0      16     112320        0
        malloc-128    128     589     186        3908       0     126      99200        0
        128 Bucket   1024      47      36         277       0      16      84992        0
      malloc-16384  16384       5       0           5       0       1      81920        0
              PROC   1376      36      19         780       0       8      75680        0
       malloc-8192   8192       9       0           9       0       1      73728        0
       malloc-8192   8192       7       2         110       0       1      73728        0
             g_bio    408       0     180        5019       0      30      73440        0
         malloc-64     64     753     318        2441       0     254      68544        0
         malloc-64     64     556     515        1698       0     254      68544        0
      malloc-65536  65536       1       0           1       0       1      65536        0
      malloc-65536  65536       1       0           1       0       1      65536        0
      malloc-32768  32768       2       0           2       0       1      65536        0
       malloc-1024   1024      25      39         557       0      16      65536        0
        malloc-256    256     132     123        1087       0      62      65280        0
        malloc-256    256     161      94         526       0      62      65280        0
         filedesc0   1072      37      19         781       0       8      60032        0
       malloc-4096   4096      12       2          23       0       2      57344        0
         64 Bucket    512      62      42        1131       0      30      53248        0
         32 Bucket    256      58     137        1095       0      62      49920        0
           DIRHASH   1024      35      13          35       0      16      49152        0
             NAMEI   1024       0      48       12223       0      16      49152        0
      malloc-16384  16384       2       1          12       0       1      49152        0
      malloc-16384  16384       1       2         161       0       1      49152        0
       malloc-1024   1024      29      19         907       0      16      49152        0
           pcpu-64     64     491     277         491       0     254      49152        0
        malloc-384    384      63      57          63       0      30      46080        0
          syncache    168       0     264           5       0     254      44352        0
            pcpu-8      8    4774     346        4802       0     254      40960        0
              pipe    728       6      49         284       0      16      40040        0
         udp_inpcb    424       6      84         128       0      30      38160        0
            da_ccb    544       0      70        1437       0      16      38080        0
         malloc-64     64     148     419       14552       0     254      36288        0
         malloc-64     64      32     535          50       0     254      36288        0
         malloc-64     64     197     370       13372       0     254      36288        0
        malloc-128    128      36     243         164       0     126      35712        0
        malloc-128    128       4     275          31       0     126      35712        0
        malloc-128    128      10     269          10       0     126      35712        0
     routing nhops    256      10     125          17       0      62      34560        0
           ttyoutq    256      72      63         160       0      62      34560        0
        malloc-256    256      57      78         506       0      62      34560        0
        malloc-256    256       3     132          28       0      62      34560        0
        malloc-256    256      37      98         654       0      62      34560        0
        malloc-256    256       5     130           5       0      62      34560        0
      malloc-32768  32768       1       0           1       0       1      32768        0
       malloc-8192   8192       4       0           4       0       1      32768        0
       malloc-4096   4096       6       2           6       0       2      32768        0
       malloc-2048   2048       6      10           6       0       8      32768        0
       malloc-2048   2048      10       6          20       0       8      32768        0
       malloc-2048   2048       1      15          25       0       8      32768        0
       malloc-2048   2048       8       8         249       0       8      32768        0
       malloc-1024   1024       3      29          43       0      16      32768        0
       malloc-1024   1024       4      28           4       0      16      32768        0
       malloc-1024   1024       4      28           4       0      16      32768        0
        malloc-512    512       0      64         118       0      30      32768        0
        malloc-512    512      12      52          24       0      30      32768        0
        malloc-512    512       0      64           9       0      30      32768        0
        malloc-512    512       2      62          51       0      30      32768        0
        malloc-512    512       3      61           3       0      30      32768        0
            ttyinq    160     135      65         300       0      62      32000        0
              PGRP    120      19     245          31       0     126      31680        0
            clpbuf   2624       0      12          50       0       4      31488        0
           VMSPACE    520      20      40         765       0      16      31200        0
            cpuset    104       7     272           7       0     126      29016        0
        sctp_laddr     48       0     588           4       0     254      28224        0
         malloc-32     32     274     608         497       0     254      28224        0
         malloc-32     32     214     668        1289       0     254      28224        0
         16 Bucket    144      48     148         306       0      62      28224        0
          4 Bucket     48       7     581          14       0     254      28224        0
         TURNSTILE    136     122      67         122       0      62      25704        0
       malloc-8192   8192       3       0           3       0       1      24576        0
       malloc-8192   8192       3       0           3       0       1      24576        0
    ertt_txseginfo     40       0     606         272       0     254      24240        0
          rl_entry     40      24     582          24       0     254      24240        0
               PWD     40      10     596         100       0     254      24240        0
           rtentry    168      13     131          17       0      62      24192        0
             Files     80      67     233        6580       0     126      24000        0
          8 Bucket     80      41     259         534       0     126      24000        0
         tcp_inpcb   1304       3      15           7       0       8      23472        0
        malloc-384    384      10      50          32       0      30      23040        0
        malloc-384    384       2      58         344       0      30      23040        0
       Mountpoints   2816       2       6           2       0       4      22528        0
        SLEEPQUEUE     88     122     134         122       0     126      22528        0
         hostcache     64       1     314           1       0     254      20160        0
   udp_inpcb ports     32       3     627          40       0     254      20160        0
              ertt     72       3     277           7       0     126      20160        0
         malloc-32     32      20     610         175       0     254      20160        0
         malloc-32     32      51     579          77       0     254      20160        0
         malloc-32     32     105     525        3058       0     254      20160        0
         malloc-32     32       4     626        1037       0     254      20160        0
          2 Bucket     32      49     581         295       0     254      20160        0
        malloc-128    128      81      74         423       0     126      19840        0
        malloc-256    256       5      70           6       0      62      19200        0
 epoch_record pcpu    256       4      60           4       0      62      16384        0
      malloc-16384  16384       1       0           1       0       1      16384        0
       malloc-8192   8192       1       1          27       0       1      16384        0
       malloc-8192   8192       2       0           2       0       1      16384        0
       malloc-4096   4096       1       3         202       0       2      16384        0
       malloc-4096   4096       3       1           4       0       2      16384        0
       malloc-2048   2048       1       7          13       0       8      16384        0
       malloc-1024   1024       0      16           1       0      16      16384        0
        malloc-512    512       5      27           6       0      30      16384        0
        malloc-512    512       1      31           2       0      30      16384        0
           SMR CPU     32       7     504           7       0     254      16352        0
         malloc-16     16     524     476        3505       0     254      16000        0
              kenv    258      16      44        1040       0      30      15480        0
            mqnode    416       3      33           3       0      30      14976        0
              vmem   1856       1       7           1       0       8      14848        0
             ripcb    392       1      35           4       0      30      14112        0
        SMR SHARED     24       7     504           7       0     254      12264        0
   tcp_inpcb ports     32       1     377           1       0     254      12096        0
         malloc-32     32       7     371           7       0     254      12096        0
             KNOTE    160       0      75           8       0      62      12000        0
         malloc-16     16      34     716        1313       0     254      12000        0
         malloc-16     16      18     732          68       0     254      12000        0
         malloc-16     16      61     689       26653       0     254      12000        0
         malloc-16     16       4     746           4       0     254      12000        0
        malloc-384    384       0      30           1       0      30      11520        0
        malloc-384    384       1      29           1       0      30      11520        0
arc_buf_hdr_t_full    344       4      29           4       0      30      11352        0
           pcpu-16     16       8     504           8       0     254       8192        0
      vtnet_tx_hdr     24       0     334        2376       0     254       8016        0
         malloc-16     16      15     485          16       0     254       8000        0
         malloc-16     16       0     500           1       0     254       8000        0
       UMA Slabs 1    176       9      13           9       0      62       3872        0
        KMAP ENTRY     96      15      24          23       0       0       3744        0
         arc_buf_t     32       0       0           0       0     254          0        0
arc_buf_hdr_t_l2only     96       0       0           0       0     126          0        0
arc_buf_hdr_t_full_crypt    408       0       0           0       0      30          0        0
           dnode_t   1016       0       0           0       0      16          0        0
          sa_cache    296       0       0           0       0      30          0        0
         abd_chunk   4096       0       0           0       0       2          0        0
zio_buf_comb_16777216 16777216       0       0           0       0       1          0        0
zio_buf_comb_14680064 14680064       0       0           0       0       1          0        0
zio_buf_comb_12582912 12582912       0       0           0       0       1          0        0
zio_buf_comb_10485760 10485760       0       0           0       0       1          0        0
zio_buf_comb_8388608 8388608       0       0           0       0       1          0        0
zio_buf_comb_7340032 7340032       0       0           0       0       1          0        0
zio_buf_comb_6291456 6291456       0       0           0       0       1          0        0
zio_buf_comb_5242880 5242880       0       0           0       0       1          0        0
zio_buf_comb_4194304 4194304       0       0           0       0       1          0        0
zio_buf_comb_3670016 3670016       0       0           0       0       1          0        0
zio_buf_comb_3145728 3145728       0       0           0       0       1          0        0
zio_buf_comb_2621440 2621440       0       0           0       0       1          0        0
zio_buf_comb_2097152 2097152       0       0           0       0       1          0        0
zio_buf_comb_1835008 1835008       0       0           0       0       1          0        0
zio_buf_comb_1572864 1572864       0       0           0       0       1          0        0
zio_buf_comb_1310720 1310720       0       0           0       0       1          0        0
zio_buf_comb_1048576 1048576       0       0           0       0       1          0        0
zio_buf_comb_917504 917504       0       0           0       0       1          0        0
zio_buf_comb_786432 786432       0       0           0       0       1          0        0
zio_buf_comb_655360 655360       0       0           0       0       1          0        0
zio_buf_comb_524288 524288       0       0           0       0       1          0        0
zio_buf_comb_458752 458752       0       0           0       0       1          0        0
zio_buf_comb_393216 393216       0       0           0       0       1          0        0
zio_buf_comb_327680 327680       0       0           0       0       1          0        0
zio_buf_comb_262144 262144       0       0           0       0       1          0        0
zio_buf_comb_229376 229376       0       0           0       0       1          0        0
zio_buf_comb_196608 196608       0       0           0       0       1          0        0
zio_buf_comb_163840 163840       0       0           0       0       1          0        0
zio_buf_comb_131072 131072       0       0           0       0       1          0        0
zio_buf_comb_114688 114688       0       0           0       0       1          0        0
zio_buf_comb_98304  98304       0       0           0       0       1          0        0
zio_buf_comb_81920  81920       0       0           0       0       1          0        0
zio_buf_comb_65536  65536       0       0           0       0       1          0        0
zio_buf_comb_57344  57344       0       0           0       0       1          0        0
zio_buf_comb_49152  49152       0       0           0       0       1          0        0
zio_buf_comb_40960  40960       0       0           0       0       1          0        0
zio_buf_comb_32768  32768       0       0           0       0       1          0        0
zio_buf_comb_28672  28672       0       0           0       0       1          0        0
zio_buf_comb_24576  24576       0       0           0       0       1          0        0
zio_buf_comb_20480  20480       0       0           0       0       1          0        0
zio_data_buf_16384  16384       0       0           0       0       1          0        0
     zio_buf_16384  16384       0       0           0       0       1          0        0
zio_data_buf_14336  14336       0       0           0       0       1          0        0
     zio_buf_14336  14336       0       0           0       0       1          0        0
zio_data_buf_12288  12288       0       0           0       0       1          0        0
     zio_buf_12288  12288       0       0           0       0       1          0        0
zio_data_buf_10240  10240       0       0           0       0       1          0        0
     zio_buf_10240  10240       0       0           0       0       1          0        0
 zio_data_buf_8192   8192       0       0           0       0       1          0        0
      zio_buf_8192   8192       0       0           0       0       1          0        0
 zio_data_buf_7168   7168       0       0           0       0       1          0        0
      zio_buf_7168   7168       0       0           0       0       1          0        0
 zio_data_buf_6144   6144       0       0           0       0       1          0        0
      zio_buf_6144   6144       0       0           0       0       1          0        0
 zio_data_buf_5120   5120       0       0           0       0       1          0        0
      zio_buf_5120   5120       0       0           0       0       1          0        0
 zio_data_buf_4096   4096       0       0           0       0       2          0        0
      zio_buf_4096   4096       0       0           0       0       2          0        0
 zio_data_buf_3584   3584       0       0           0       0       2          0        0
      zio_buf_3584   3584       0       0           0       0       2          0        0
 zio_data_buf_3072   3072       0       0           0       0       4          0        0
      zio_buf_3072   3072       0       0           0       0       4          0        0
 zio_data_buf_2560   2560       0       0           0       0       4          0        0
      zio_buf_2560   2560       0       0           0       0       4          0        0
 zio_data_buf_2048   2048       0       0           0       0       8          0        0
      zio_buf_2048   2048       0       0           0       0       8          0        0
 zio_data_buf_1536   1536       0       0           0       0       8          0        0
      zio_buf_1536   1536       0       0           0       0       8          0        0
 zio_data_buf_1024   1024       0       0           0       0      16          0        0
      zio_buf_1024   1024       0       0           0       0      16          0        0
  zio_data_buf_512    512       0       0           0       0      30          0        0
       zio_buf_512    512       0       0           0       0      30          0        0
    zio_link_cache     48       0       0           0       0     254          0        0
         zio_cache   1216       0       0           0       0       8          0        0
   ddt_entry_cache    392       0       0           0       0      30          0        0
         ddt_cache  24824       0       0           0       0       1          0        0
brt_pending_entry_cache    160       0       0           0       0      62          0        0
   brt_entry_cache     40       0       0           0       0     254          0        0
metaslab_alloc_trace_cache     72       0       0           0       0     126          0        0
zfs_btree_leaf_cache   4096       0       0           0       0       2          0        0
   reference_cache     48       0       0           0       0     254          0        0
        taskq_zone    192       0       0           0       0      62          0        0
       FFS1 dinode    128       0       0           0       0     126          0        0
           ada_ccb    272       0       0           0       0      30          0        0
             swblk    136       0       0           0       0      62          0        0
          swpctrie    144       0       0           0       0      62          0        0
   pf state scrubs     40       0       0           0       0     254          0        0
   pf frag entries     40       0       0           0       0     254          0        0
          pf frags    248       0       0           0       0      62          0        0
  pf table entries    160       0       0           0       0     254          0        0
pf table entry counters     64       0       0           0       0     254          0        0
   pf source nodes    152       0       0           0       0     254          0        0
     pf state keys     88       0       0           0       0     126          0        0
         pf states    344       0       0           0       0     254          0        0
           pf tags    104       0       0           0       0     126          0        0
          pf mtags    184       0       0           0       0      62          0        0
      tcp_rack_pcb   1024       0       0           0       0      16          0        0
      tcp_rack_map    128       0       0           0       0     126          0        0
       tcp_bbr_pcb    832       0       0           0       0      16          0        0
       tcp_bbr_map    128       0       0           0       0     126          0        0
tfo_ccache_entries     80       0       0           0       0     126          0        0
               tfo      4       0       0           0       0     254          0        0
          sackhole     32       0       0           0       0     254          0        0
               ipq     56       0       0           0       0     254          0        0
   sctp_asconf_ack     48       0       0           0       0     254          0        0
       sctp_asconf     40       0       0           0       0     254          0        0
sctp_stream_msg_out    112       0       0           0       0     254          0        0
        sctp_readq    152       0       0           0       0     254          0        0
        sctp_chunk    152       0       0           0       0     254          0        0
        sctp_raddr    736       0       0           0       0     254          0        0
         sctp_asoc   2264       0       0           0       0     254          0        0
           sctp_ep   1176       0       0           0       0     254          0        0
   tcp_log_id_node    120       0       0           0       0     126          0        0
 tcp_log_id_bucket    176       0       0           0       0      62          0        0
           tcp_log    416       0       0           0       0     254          0        0
          tcpreass     48       0       0           0       0     254          0        0
       ripcb ports     32       0       0           0       0     254          0        0
udplite_inpcb ports     32       0       0           0       0     254          0        0
     udplite_inpcb    424       0       0           0       0      30          0        0
    IPsec SA lft_c     16       0       0           0       0     254          0        0
           netlink   2048       0       0           0       0       8          0        0
            itimer    352       0       0           0       0      30          0        0
            AIOLIO    272       0       0           0       0      30          0        0
             AIOCB    552       0       0           0       0      16          0        0
               AIO    208       0       0           0       0      62          0        0
        mqnotifier    216       0       0           0       0      62          0        0
            mvdata     64       0       0           0       0     254          0        0
            mqueue    248       0       0           0       0      62          0        0
        TMPFS node    232       0       0           0       0      62          0        0
           NCLNODE    608       0       0           0       0      16          0        0
     LTS VFS Cache    360       0       0           0       0      30          0        0
       L VFS Cache    320       0       0           0       0      30          0        0
     STS VFS Cache    144       0       0           0       0      62          0        0
           cryptop    280       0       0           0       0      30          0        0
  linux_dma_object     32       0       0           0       0     254          0        0
  linux_dma_pctrie    144       0       0           0       0      62          0        0
   IOMMU_MAP_ENTRY    104       0       0           0       0     126          0        0
    mbuf_jumbo_16k  16384       0       0           0       0     254          0        0
     mbuf_jumbo_9k   9216       0       0           0       0     254          0        0
      audit_record   1280       0       0           0       0       8          0        0
         domainset     40       0       0           0       0     254          0        0
        MAC labels     40       0       0           0       0     254          0        0
            vnpbuf   2624       0       0           0       0      16          0        0
           nfspbuf   2624       0       0           0       0       4          0        0
            swwbuf   2624       0       0           0       0       2          0        0
            swrbuf   2624       0       0           0       0       4          0        0
          umtx_shm     88       0       0           0       0     126          0        0
           umtx pi     96       0       0           0       0     126          0        0
rangeset pctrie nodes    144       0       0           0       0      62          0        0
      malloc-65536  65536       0       0           0       0       1          0        0
      malloc-65536  65536       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-16384  16384       0       0           0       0       1          0        0
      malloc-16384  16384       0       0           0       0       1          0        0
      malloc-16384  16384       0       0           0       0       1          0        0
      malloc-16384  16384       0       0           0       0       1          0        0
       malloc-8192   8192       0       0           0       0       1          0        0
       malloc-4096   4096       0       0           0       0       2          0        0
       malloc-4096   4096       0       0           0       0       2          0        0
       malloc-2048   2048       0       0           0       0       8          0        0
       malloc-2048   2048       0       0           0       0       8          0        0
       malloc-1024   1024       0       0           0       0      16          0        0
        malloc-512    512       0       0           0       0      30          0        0
        malloc-384    384       0       0           0       0      30          0        0
        malloc-384    384       0       0           0       0      30          0        0
         malloc-64     64       0       0           0       0     254          0        0
         malloc-64     64       0       0           0       0     254          0        0
           pcpu-32     32       0       0           0       0     254          0        0
            pcpu-4      4       0       0           0       0     254          0        0
            fakepg    104       0       0           0       0     126          0        0
          UMA Hash    256       0       0           0       0      62          0        0

Crashes (1112):
Time Kernel Commit Syzkaller Config Log Report Syz repro C repro VM info Assets (help?) Manager Title
2023/07/04 19:58 freebsd-src d7614c010c76 ee50e71c console log report syz C ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/09/03 02:56 freebsd-src 32988c1499f8 696ea0d2 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/09/02 17:50 freebsd-src 4a9cd9fc22d7 696ea0d2 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/09/02 13:45 freebsd-src 4a9cd9fc22d7 696ea0d2 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/09/02 10:48 freebsd-src 4a9cd9fc22d7 696ea0d2 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/09/02 09:39 freebsd-src 4a9cd9fc22d7 696ea0d2 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/09/02 06:22 freebsd-src 4a9cd9fc22d7 696ea0d2 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/09/02 02:34 freebsd-src 565c887a7752 696ea0d2 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/09/01 21:06 freebsd-src 565c887a7752 696ea0d2 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/09/01 17:41 freebsd-src 565c887a7752 696ea0d2 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/09/01 16:39 freebsd-src 565c887a7752 696ea0d2 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/09/01 13:56 freebsd-src d88983333447 696ea0d2 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/09/01 10:53 freebsd-src d88983333447 696ea0d2 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/09/01 09:19 freebsd-src d88983333447 696ea0d2 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/09/01 08:08 freebsd-src d88983333447 696ea0d2 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/09/01 05:18 freebsd-src d88983333447 696ea0d2 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/31 23:29 freebsd-src 09ec5e67a7a6 4ad3538b console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/31 21:59 freebsd-src 09ec5e67a7a6 4ad3538b console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/31 21:38 freebsd-src 09ec5e67a7a6 4ad3538b console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/31 20:08 freebsd-src 09ec5e67a7a6 4ad3538b console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/31 17:40 freebsd-src 09ec5e67a7a6 4ad3538b console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/31 15:11 freebsd-src 09ec5e67a7a6 4ad3538b console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/31 11:16 freebsd-src 872e89405684 84803932 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/31 10:16 freebsd-src 872e89405684 84803932 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/31 08:49 freebsd-src 872e89405684 84803932 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/31 06:59 freebsd-src 872e89405684 84803932 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/31 02:54 freebsd-src b22aae410bc7 84803932 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/31 01:17 freebsd-src b22aae410bc7 84803932 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/30 21:54 freebsd-src b22aae410bc7 84803932 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/30 20:35 freebsd-src b22aae410bc7 84803932 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/30 19:45 freebsd-src b22aae410bc7 84803932 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/30 18:36 freebsd-src b22aae410bc7 84803932 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/30 17:10 freebsd-src b22aae410bc7 84803932 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/30 14:12 freebsd-src 47d997021fbc b42ca3a7 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/30 11:01 freebsd-src 47d997021fbc b42ca3a7 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/30 08:58 freebsd-src 47d997021fbc b42ca3a7 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/30 07:20 freebsd-src 47d997021fbc b42ca3a7 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/30 05:46 freebsd-src 47d997021fbc b42ca3a7 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/30 04:33 freebsd-src 47d997021fbc b42ca3a7 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/30 00:47 freebsd-src 2e8edbc285cf ce731e62 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/29 23:43 freebsd-src 2e8edbc285cf ce731e62 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/29 22:32 freebsd-src 2e8edbc285cf ce731e62 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/29 17:33 freebsd-src 2e8edbc285cf ce731e62 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/29 13:46 freebsd-src 570688f0a5c2 7ba13a15 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/29 12:09 freebsd-src 570688f0a5c2 7ba13a15 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/29 09:14 freebsd-src 570688f0a5c2 7ba13a15 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/29 07:41 freebsd-src 570688f0a5c2 7ba13a15 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/29 07:15 freebsd-src 570688f0a5c2 7ba13a15 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/08/29 06:02 freebsd-src 570688f0a5c2 7ba13a15 console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
2023/07/04 17:52 freebsd-src d7614c010c76 ee50e71c console log report ci-freebsd-main panic: mutex Giant owned at /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:LINE
* Struck through repros no longer work on HEAD.