panic: lock (sleep mutex) sctp-inp not locked @ /syzkaller/managers/main/kernel/sys/netinet/sctp_usrreq.c:958
cpuid = 1
time = 1705481434
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0xc6/frame 0xfffffe006a6745b0
kdb_backtrace() at kdb_backtrace+0xd0/frame 0xfffffe006a674710
vpanic() at vpanic+0x271/frame 0xfffffe006a6748b0
panic() at panic+0xb5/frame 0xfffffe006a674970
witness_unlock() at witness_unlock+0x4fd/frame 0xfffffe006a6749f0
__mtx_unlock_flags() at __mtx_unlock_flags+0xec/frame 0xfffffe006a674ab0
sctp_shutdown() at sctp_shutdown+0xf35/frame 0xfffffe006a674c10
soshutdown() at soshutdown+0xb3/frame 0xfffffe006a674c50
kern_shutdown() at kern_shutdown+0x154/frame 0xfffffe006a674d10
amd64_syscall() at amd64_syscall+0x473/frame 0xfffffe006a674f30
fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe006a674f30
--- syscall (0, FreeBSD ELF64, syscall), rip = 0x233d9a, rsp = 0x820fb6b18, rbp = 0x820fb6b30 ---
KDB: enter: panic
[ thread pid 773 tid 100110 ]
Stopped at      kdb_enter+0x6e: movq    $0,0x2194267(%rip)
db> 
db> set $lines = 0
db> set $maxwidth = 0
db> show registers
cs                        0x20
ds                        0x3b
es                        0x3b
fs                        0x13
gs                        0x1b
ss                        0x28
rax                       0x12
rcx         0xfffffe00033eee30
rdx         0xdffff7c000000000
rbx         0xffffffff826f86c0  .str.28
rsp         0xfffffe006a6746f0
rbp         0xfffffe006a674710
rsi                        0x1
rdi                          0
r8                           0
r9                  0xffffffff
r10                          0
r11         0xfffffe006cdb4520
r12                          0
r13         0xfffffe006cdb4000
r14         0xffffffff826f86c0  .str.28
r15                          0
rip         0xffffffff815cadae  kdb_enter+0x6e
rflags                    0x46
kdb_enter+0x6e: movq    $0,0x2194267(%rip)
db> show proc
Process 773 (syz-executor2175380) at 0xfffffe006cdadac0:
 state: NORMAL
 uid: 0  gids: 0, 0, 5
 parent: pid 771 at 0xfffffe0058d7c5c0
 ABI: FreeBSD ELF64
 flag: 0x10004000  flag2: 0
 arguments: ./syz-executor2175380996
 reaper: 0xfffffe00541d1040 reapsubtree: 1
 sigparent: 20
 vmspace: 0xfffffe006cb45738
   (map 0xfffffe006cb45738)
   (map.pmap 0xfffffe006cb457f8)
   (pmap 0xfffffe006cb45868)
 threads: 1
100110                   Run     CPU 1                       syz-executor2175380
db> ps
  pid  ppid  pgrp   uid  state   wmesg   wchan               cmd
  773   771   771     0  R       CPU 1                       syz-executor2175380
  771   769   771     0  Ss      pause   0xfffffe0058d7c670  csh
  769   682   769     0  Ss      select  0xfffffe00571b1ec0  sshd
  748     1   748     0  Ss+     ttyin   0xfffffe005766fcb0  getty
  747     1   747     0  Ss+     ttyin   0xfffffe00587674b0  getty
  746     1   746     0  Ss+     ttyin   0xfffffe00587678b0  getty
  745     1   745     0  Ss+     ttyin   0xfffffe0058767cb0  getty
  744     1   744     0  Ss+     ttyin   0xfffffe00587680b0  getty
  743     1   743     0  Ss+     ttyin   0xfffffe005766ecb0  getty
  742     1   742     0  Ss+     ttyin   0xfffffe00587684b0  getty
  741     1   741     0  Ss+     ttyin   0xfffffe00587688b0  getty
  740     1   740     0  Ss+     ttyin   0xfffffe0058768cb0  getty
  738     1    18     0  S+      piperd  0xfffffe0058a44c70  logger
  737   736    18     0  S+      nanslp  0xffffffff83740400  sleep
  736     1    18     0  S+      wait    0xfffffe0057b9aac0  sh
  686     1   686     0  Ss      nanslp  0xffffffff83740400  cron
  682     1   682     0  Ss      select  0xfffffe00571b60c0  sshd
  495     1   495     0  Ss      select  0xfffffe00571b6140  syslogd
  424     1   424     0  Ss      select  0xfffffe00571b61c0  devd
  423     1   423    65  Ss      select  0xfffffe0057a93b40  dhclient
  338     1   338     0  Ss      select  0xfffffe00571b6340  dhclient
  335     1   335     0  Ss      select  0xfffffe00571b63c0  dhclient
   17     0     0     0  DL      syncer  0xffffffff8385db20  [syncer]
   16     0     0     0  DL      vlruwt  0xfffffe0057181040  [vnlru]
   15     0     0     0  DL      (threaded)                  [bufdaemon]
100079                   D       psleep  0xffffffff8385c100  [bufdaemon]
100082                   D       -       0xffffffff82c0a140  [bufspacedaemon-0]
100095                   D       sdflush 0xfffffe00589b78e8  [/ worker]
    9     0     0     0  DL      psleep  0xffffffff838a55c0  [vmdaemon]
    8     0     0     0  DL      (threaded)                  [pagedaemon]
100077                   D       psleep  0xffffffff8388d438  [dom0]
100080                   D       launds  0xffffffff8388d444  [laundry: dom0]
100081                   D       umarcl  0xffffffff81d61000  [uma]
    7     0     0     0  DL      -       0xffffffff834bdc10  [rand_harvestq]
    6     0     0     0  DL      pftm    0xffffffff843f1ea0  [pf purge]
    5     0     0     0  DL      waiting 0xffffffff842335a0  [sctp_iterator]
    4     0     0     0  DL      (threaded)                  [cam]
100044                   D       -       0xffffffff83488340  [doneq0]
100045                   D       -       0xffffffff834882c0  [async]
100076                   D       -       0xffffffff83488140  [scanner]
    3     0     0     0  DL      (threaded)                  [crypto]
100041                   D       crypto_ 0xffffffff83888c60  [crypto]
100042                   D       crypto_ 0xfffffe00542d5730  [crypto returns 0]
100043                   D       crypto_ 0xfffffe00542d5780  [crypto returns 1]
   14     0     0     0  DL      seqstat 0xfffffe000796e888  [sequencer 00]
   13     0     0     0  DL      (threaded)                  [geom]
100035                   D       -       0xffffffff836e8d40  [g_event]
100036                   D       -       0xffffffff836e8d60  [g_up]
100037                   D       -       0xffffffff836e8d80  [g_down]
    2     0     0     0  WL      (threaded)                  [clock]
100029                   I                                   [clock (0)]
100030                   I                                   [clock (1)]
   12     0     0     0  WL      (threaded)                  [intr]
100012                   I                                   [swi6: task queue]
100013                   I                                   [swi6: Giant taskq]
100015                   I                                   [swi5: fast taskq]
100031                   I                                   [swi1: netisr 0]
100032                   I                                   [swi1: hpts]
100033                   I                                   [swi1: hpts]
100046                   I                                   [irq24: virtio_pci0]
100047                   I                                   [irq25: virtio_pci0]
100048                   I                                   [irq26: virtio_pci0]
100049                   I                                   [irq27: virtio_pci0]
100050                   I                                   [irq28: virtio_pci1]
100051                   I                                   [irq29: virtio_pci1]
100052                   I                                   [irq30: virtio_pci1]
100053                   I                                   [irq31: virtio_pci1]
100054                   I                                   [irq32: virtio_pci1]
100059                   I                                   [irq33: virtio_pci2]
100060                   I                                   [irq34: virtio_pci2]
100061                   I                                   [irq35: virtio_pci2]
100063                   I                                   [irq1: atkbd0]
100064                   I                                   [irq12: psm0]
100065                   I                                   [swi0: uart uart++]
100069                   I                                   [swi1: pf send]
   11     0     0     0  RL      (threaded)                  [idle]
100003                   Run     CPU 0                       [idle: cpu0]
100004                   CanRun                              [idle: cpu1]
    1     0     1     0  SLs     wait    0xfffffe00541d1040  [init]
   10     0     0     0  DL      audit_w 0xffffffff838896c0  [audit]
    0     0     0     0  DLs     (threaded)                  [kernel]
100000                   D       swapin  0xffffffff836e9720  [swapper]
100005                   D       -       0xfffffe0054282100  [softirq_0]
100006                   D       -       0xfffffe0054282000  [softirq_1]
100007                   D       -       0xfffffe0054281e00  [if_io_tqg_0]
100008                   D       -       0xfffffe0054281d00  [if_io_tqg_1]
100009                   D       -       0xfffffe0054281c00  [if_config_tqg_0]
100010                   D       -       0xfffffe0007977900  [pci_hp taskq]
100011                   D       -       0xfffffe0007977800  [kqueue_ctx taskq]
100014                   D       -       0xfffffe0007977500  [thread taskq]
100016                   D       -       0xfffffe0007977300  [aiod_kick taskq]
100017                   D       -       0xfffffe0007977200  [deferred_unmount ta]
100018                   D       -       0xfffffe0007977100  [inm_free taskq]
100019                   D       -       0xfffffe0007977000  [in6m_free taskq]
100020                   D       -       0xfffffe0007976e00  [linuxkpi_irq_wq]
100021                   D       -       0xfffffe0007976d00  [linuxkpi_short_wq_0]
100022                   D       -       0xfffffe0007976d00  [linuxkpi_short_wq_1]
100023                   D       -       0xfffffe0007976d00  [linuxkpi_short_wq_2]
100024                   D       -       0xfffffe0007976d00  [linuxkpi_short_wq_3]
100025                   D       -       0xfffffe0007976c00  [linuxkpi_long_wq_0]
100026                   D       -       0xfffffe0007976c00  [linuxkpi_long_wq_1]
100027                   D       -       0xfffffe0007976c00  [linuxkpi_long_wq_2]
100028                   D       -       0xfffffe0007976c00  [linuxkpi_long_wq_3]
100034                   D       -       0xfffffe0007976a00  [firmware taskq]
100039                   D       -       0xfffffe0007976700  [crypto_0]
100040                   D       -       0xfffffe0007976700  [crypto_1]
100055                   D       -       0xfffffe00571c4b00  [vtnet0 rxq 0]
100056                   D       -       0xfffffe00571c4a00  [vtnet0 txq 0]
100057                   D       -       0xfffffe00571c4900  [vtnet0 rxq 1]
100058                   D       -       0xfffffe00571c4800  [vtnet0 txq 1]
100062                   D       vtbslp  0xfffffe00571b6580  [virtio_balloon]
100066                   D       -       0xffffffff826fd920  [deadlkres]
100070                   D       -       0xfffffe0057bbd100  [acpi_task_0]
100071                   D       -       0xfffffe0057bbd100  [acpi_task_1]
100072                   D       -       0xfffffe0057bbd100  [acpi_task_2]
100074                   D       -       0xfffffe0007977c00  [mca taskq]
100075                   D       -       0xfffffe0007976500  [CAM taskq]
db> show all locks
db> show malloc
              Type        InUse        MemUse     Requests
           pf_hash            5        11524K            5
          tcp_hpts            7         4801K            7
            devbuf         4192         4324K         4217
         sysctloid        34817         2052K        34888
             vtbuf           24         1968K           46
              kobj          326         1304K          488
            newblk          829         1231K          841
          vfscache            3         1025K            3
               pcb           25          671K           44
          inodedep           50          531K           71
         ufs_quota            1          512K            1
          vfs_hash            1          512K            1
           callout            2          512K            2
              intr            4          472K            4
         vnet_data            2          224K            2
          acpitask            1          224K            1
            acpica         1674          184K        60830
           subproc           96          181K          830
           tidhash            3          141K            3
              vmem            3          134K            4
           pagedep           14          132K           18
            linker          352          130K          386
        tfo_ccache            1          128K            1
          IP reass            1          128K            1
               sem            4          106K            4
            DEVFS1          105          105K          114
        gtaskqueue           18           98K           18
               bus          985           81K         5155
          mtx_pool            2           72K            2
          syncache            1           68K            1
     NFSD srvcache            3           68K            3
            module          513           65K          513
       ddb_capture            1           64K            1
              temp           18           37K         1564
         hostcache            1           32K            1
               shm            1           32K            1
           kdtrace          154           31K          888
            DEVFS3          124           31K          134
               msg            4           30K            4
              umtx          240           30K          240
            kbdmux            6           28K            6
        DEVFS_RULE           56           20K           56
               BPF           10           18K           10
         ufs_mount            4           17K            5
              proc            3           17K            3
               tty           16           16K           16
           ithread           97           16K           97
            bus-sc           34           15K         1687
      eventhandler          161           14K          161
            KTRACE          100           13K          100
              kenv           95           12K           95
            ifaddr           30           12K           32
              GEOM           61           11K          481
          routetbl           50           11K          176
              rman           86           11K          431
         CAM queue            5           11K         1528
         bmsafemap            4            9K           39
               rpc            4            9K            4
           devstat            4            9K            4
              UART           12            9K           12
              ksem            1            8K            1
             shmfd            1            8K            1
       pfs_vncache            1            8K            1
     audit_evclass          238            8K          300
         taskqueue           63            7K           63
            sglist            6            7K            6
           CAM DEV            3            6K          510
              cred           23            6K          274
         pfs_nodes           20            5K           20
             hhook           15            5K           17
       ufs_dirhash           24            5K           24
               UMA          267            5K          267
            dirrem           17            5K           28
            plimit           17            5K          322
             ifnet            3            5K            3
                vt           11            5K           11
           memdesc            1            4K            1
               MCA           32            4K           32
          filedesc            1            4K            1
             evdev            4            4K            4
           acpisem           28            4K           28
       ether_multi           40            4K           50
            diradd           25            4K           36
           lltable           11            4K           11
          pf_ifnet            5            3K            6
         in6_multi           25            3K           25
          terminal           11            3K           11
           session           20            3K           31
            kqueue           40            3K          776
           pwddesc           40            3K          774
             clone            9            3K            9
           uidinfo            3            3K            8
         proc-args           62            3K         1696
        local_apic            1            2K            1
           io_apic            1            2K            1
         ipsec-saq            2            2K            2
             selfd           27            2K        12854
            Unitno           27            2K           41
           CAM XPT           22            2K          543
             lockf           15            2K           22
               msi           12            2K           12
         toponodes            6            2K            6
       ipsecpolicy            2            2K            2
           acpidev           20            2K           20
         sctp_stro            1            1K            1
           softdep            1            1K            1
            sahead            1            1K            1
          secasvar            1            1K            1
       vnodemarker            2            1K            8
      NFSD session            1            1K            1
            select            7            1K           29
        CAM periph            4            1K          271
             ipsec            3            1K            3
         sctp_atcl            2            1K            2
          indirdep            3            1K            3
            CC Mem            3            1K            7
             nhops            6            1K            6
              pfil            6            1K            6
            isadev            6            1K            6
             mount           16            1K           89
          pci_link           10            1K           10
          sctp_ifa            5            1K            6
            crypto            4            1K            4
            ip6ndp            4            1K            5
 encap_export_host           12            1K           12
         newdirblk            4            1K            8
             mkdir            4            1K           16
          in_multi            2            1K            4
              cdev            2            1K            2
       lkpikmalloc            8            1K            9
               osd            8            1K           20
           netlink            2            1K           12
    chacha20random            1            1K            1
            biobuf            1            1K            1
       inpcbpolicy           10            1K          137
          sctp_ifn            2            1K            6
               mld            2            1K            2
              igmp            2            1K            2
            vnodes            1            1K            1
      NFSD lckfile            1            1K            1
     NFSD V4client            1            1K            1
            DEVFSP            4            1K            9
             DEVFS            9            1K           10
           CAM SIM            2            1K            2
            feeder            7            1K            7
           tcpfunc            3            1K            3
        loginclass            3            1K            7
            prison            6            1K            6
         sctp_atky            3            1K            3
         cryptodev            2            1K           49
          nexusdev            8            1K            8
            apmdev            1            1K            1
          atkbddev            2            1K            2
          procdesc            1            1K            6
          pmchooks            1            1K            1
          CAM path            4            1K         1034
     CAM dev queue            2            1K            2
 CAM I/O Scheduler            1            1K            1
            soname            4            1K         3329
          filecaps            4            1K           66
               tun            3            1K            3
          sctp_vrf            1            1K            1
          freework            1            1K           26
              vnet            1            1K            1
               pmc            1            1K            1
           entropy            2            1K           35
          acpiintr            1            1K            1
         sctp_athm            2            1K            2
          sctp_map            2            1K            2
              cpus            2            1K            2
    vnet_data_free            1            1K            1
           Per-cpu            1            1K            1
          p1003.1b            1            1K            1
            ipcomp            0            0K            0
               esp            0            0K            0
                ah            0            0K            0
          pf_table            0            0K            0
           pf_rule            0            0K            0
           pf_altq            0            0K            0
           pf_osfp            0            0K            0
     pf_krule_item            0            0K            0
           pf_temp            0            0K            0
        sctp_mcore            0            0K            0
        sctp_socko            0            0K            0
         sctp_iter            0            0K            4
         sctp_mvrf            0            0K            0
         sctp_timw            0            0K            0
         sctp_cpal            0            0K            0
         sctp_cmsg            0            0K            0
         sctp_stre            0            0K            0
         sctp_athi            0            0K            0
         sctp_a_it            0            0K            4
         sctp_aadr            0            0K            0
         sctp_stri            0            0K            0
            tcp_do            0            0K            0
           tcp_fsb            0            0K            0
            mqdata            0            0K            0
           filemon            0            0K            0
        madt_table            0            0K            2
          smartpqi            0            0K            0
               ixl            0            0K            0
        ice-resmgr            0            0K            0
         ice-osdep            0            0K            0
               ice            0            0K            0
              iavf            0            0K            0
             axgbe            0            0K            0
       fpukern_ctx            0            0K            0
          xen_intr            0            0K            0
           xen_hvm            0            0K            0
         legacydrv            0            0K            0
            bounce            0            0K            0
            busdma            0            0K            0
            qpidrv            0            0K            0
      dmar_idpgtbl            0            0K            0
          dmar_dom            0            0K            0
          dmar_ctx            0            0K            0
              isci            0            0K            0
      iommu_dmamap            0            0K            0
     hyperv_socket            0            0K            0
           bxe_ilt            0            0K            0
        aesni_data            0            0K            0
            xenbus            0            0K            0
     vm_fictitious            0            0K            0
           UMAHash            0            0K            0
         vm_pgdata            0            0K            0
           jblocks            0            0K            0
          savedino            0            0K           15
          sentinel            0            0K            0
            jfsync            0            0K            0
            jtrunc            0            0K            0
             sbdep            0            0K            2
           jsegdep            0            0K            0
              jseg            0            0K            0
         jfreefrag            0            0K            0
          jfreeblk            0            0K            0
           jnewblk            0            0K            0
            jmvref            0            0K            0
           jremref            0            0K            0
           jaddref            0            0K            0
           freedep            0            0K            0
          freefile            0            0K            9
          freeblks            0            0K           25
          freefrag            0            0K            1
        allocindir            0            0K            0
       allocdirect            0            0K            0
          ufs_trim            0            0K            0
           mactemp            0            0K            0
     audit_trigger            0            0K            0
 audit_pipe_presel            0            0K            0
     audit_pipeent            0            0K            0
        audit_pipe            0            0K            0
      audit_evname            0            0K            0
         audit_bsm            0            0K            0
      audit_gidset            0            0K            0
        audit_text            0            0K            0
        audit_path            0            0K            0
        audit_data            0            0K            0
        audit_cred            0            0K            0
          ktls_ocf            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
      MLX5E_TLS_RX            0            0K            0
        MLX5EEPROM            0            0K            0
         MLX5E_TLS            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
            MLX5EN            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
          MLX5DUMP            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
       simple_attr            0            0K            0
          seq_file            0            0K            0
           lkpiskb            0            0K            0
             radix            0            0K            0
               idr            0            0K            0
          lkpindev            0            0K            0
           lkpimhi            0            0K            0
            lkpifw            0            0K            0
         lkpi80211            0            0K            0
               NLM            0            0K            0
    ipsec-spdcache            0            0K            0
         ipsec-reg            0            0K            0
        ipsec-misc            0            0K            0
      ipsecrequest            0            0K            0
            ip6opt            0            0K            3
       ip6_msource            0            0K            0
      ip6_moptions            0            0K            0
       in6_mfilter            0            0K            0
             frag6            0            0K            0
            tcplog            0            0K            0
        tcp_hwpace            0            0K            0
               LRO            0            0K            0
        ip_msource            0            0K            0
       ip_moptions            0            0K            0
        in_mfilter            0            0K            0
              ipid            0            0K            0
         80211scan            0            0K            0
      80211ratectl            0            0K            0
        80211power            0            0K            0
       80211nodeie            0            0K            0
         80211node            0            0K            0
      80211mesh_gt            0            0K            0
      80211mesh_rt            0            0K            0
         80211perr            0            0K            0
         80211prep            0            0K            0
         80211preq            0            0K            0
          80211dfs            0            0K            0
       80211crypto            0            0K            0
          80211vap            0            0K            0
             iflib            0            0K            0
              vlan            0            0K            0
               gif            0            0K            0
           ifdescr            0            0K            0
              zlib            0            0K            0
           fadvise            0            0K            0
           VN POLL            0            0K            0
            statfs            0            0K          195
     namei_tracker            0            0K            0
       export_host            0            0K            0
        cl_savebuf            0            0K            3
               aio            0            0K            0
               lio            0            0K            0
               acl            0            0K            0
          mbuf_tag            0            0K            0
              ktls            0            0K            0
              accf            0            0K            0
               pts            0            0K            0
           timerfd            0            0K            0
               iov            0            0K        13552
          ioctlops            0            0K           85
           eventfd            0            0K            0
           Witness            0            0K            0
             stack            0            0K            0
              sbuf            0            0K          288
          firmware            0            0K            0
        compressor            0            0K            0
              SWAP            0            0K            0
         sysctltmp            0            0K          649
            sysctl            0            0K            3
              ekcd            0            0K            0
            dumper            0            0K            0
          sendfile            0            0K            0
              rctl            0            0K            0
             cache            0            0K            0
          kcovinfo            0            0K            0
      prison_racct            0            0K            0
       Fail Points            0            0K            0
             sigio            0            0K            1
filedesc_to_leader            0            0K            0
               pwd            0            0K            0
       tty console            0            0K            0
         boottrace            0            0K            0
        isofs_node            0            0K            0
       isofs_mount            0            0K            0
     tr_raid5_data            0            0K            0
    tr_raid1e_data            0            0K            0
     tr_raid1_data            0            0K            0
     tr_raid0_data            0            0K            0
    tr_concat_data            0            0K            0
       md_sii_data            0            0K            0
   md_promise_data            0            0K            0
    md_nvidia_data            0            0K            0
   md_jmicron_data            0            0K            0
     md_intel_data            0            0K            0
       md_ddf_data            0            0K            0
         raid_data            0            0K           72
     geom_flashmap            0            0K            0
         tmpfs dir            0            0K            0
        tmpfs name            0            0K            0
       tmpfs mount            0            0K            0
     tmpfs extattr            0            0K            0
           NFS FHA            0            0K            0
         newnfsmnt            0            0K            0
  newnfsclient_req            0            0K            0
   NFSCL layrecall            0            0K            0
     NFSCL session            0            0K            0
     NFSCL sockreq            0            0K            0
     NFSCL devinfo            0            0K            0
     NFSCL flayout            0            0K            0
      NFSCL layout            0            0K            0
     NFSD rollback            0            0K            0
      NFSCL diroff            0            0K            0
       NEWdirectio            0            0K            0
        NEWNFSnode            0            0K            0
         NFSCL lck            0            0K            0
      NFSCL lckown            0            0K            0
      NFSCL client            0            0K            0
       NFSCL deleg            0            0K            0
        NFSCL open            0            0K            0
       NFSCL owner            0            0K            0
            NFS fh            0            0K            0
           NFS req            0            0K            0
     NFSD usrgroup            0            0K            0
       NFSD string            0            0K            0
       NFSD V4lock            0            0K            0
      NFSD V4state            0            0K            0
       msdosfs_fat            0            0K            0
     msdosfs_mount            0            0K            0
      msdosfs_node            0            0K            0
            DEVFS4            0            0K            0
            DEVFS2            0            0K            0
            gntdev            0            0K            0
       privcmd_dev            0            0K            0
        evtchn_dev            0            0K            0
          xenstore            0            0K            0
               xnb            0            0K            0
          xen_acpi            0            0K            0
              xbbd            0            0K            0
               xbd            0            0K            0
           Balloon            0            0K            0
          sysmouse            0            0K            0
            vtfont            0            0K            0
            pvscsi            0            0K            0
            USBdev            0            0K            0
               USB            0            0K            0
            twsbuf            0            0K            0
       tcp_log_dev            0            0K            0
      midi buffers            0            0K            0
             mixer            0            0K            0
              ac97            0            0K            0
             hdacc            0            0K            0
              hdac            0            0K            0
              hdaa            0            0K            0
       SIIS driver            0            0K            0
               PUC            0            0K            0
          ppbusdev            0            0K            0
            sr_iov            0            0K            0
               OCS            0            0K            0
               OCS            0            0K            0
              nvme            0            0K            0
               nvd            0            0K            0
            netmap            0            0K            0
            mwldev            0            0K            0
        MVS driver            0            0K            0
          mrsasbuf            0            0K            0
          mpt_user            0            0K            0
          mps_user            0            0K            0
            MPSSAS            0            0K            0
               mps            0            0K            0
          mpr_user            0            0K            0
            MPRSAS            0            0K            0
               mpr            0            0K            0
            mfibuf            0            0K            0
        md_sectors            0            0K            0
           md_disk            0            0K            0
           malodev            0            0K            0
               LED            0            0K            0
          ix_sriov            0            0K            0
                ix            0            0K            0
            ipsbuf            0            0K            0
         ciss_data            0            0K            0
         BACKLIGHT            0            0K            0
           ath_hal            0            0K            0
            athdev            0            0K            0
           ata_pci            0            0K            0
           ata_dma            0            0K            0
       ata_generic            0            0K            0
       AHCI driver            0            0K            0
               agp            0            0K            0
           acpipwr            0            0K            0
         acpi_perf            0            0K            0
         acpicmbat            0            0K            0
        aacraidcam            0            0K            0
       aacraid_buf            0            0K            0
            aaccam            0            0K            0
            aacbuf            0            0K            0
              zstd            0            0K            0
            XZ_DEC            0            0K            0
            nvlist            0            0K            0
          SCSI ENC            0            0K            0
           SCSI sa            0            0K            0
         scsi_pass            0            0K            0
           scsi_da            0            0K           69
            ata_da            0            0K            0
           scsi_ch            0            0K            0
           scsi_cd            0            0K            0
           nvme_da            0            0K            0
           CAM CCB            0            0K          523
     CAM ccb queue            0            0K            0
db> show uma
              Zone   Size    Used    Free    Requests  Sleeps  Bucket  Total Mem    XFree
   mbuf_jumbo_page   4096    8320    1078       15616       0     254   38494208        0
              mbuf    256    8578    1084       18946       0     254    2473472        0
          BUF TRIE    144     192   11624         538       0      62    1701504        0
        malloc-384    384    4203      27        4206       0      30    1624320        0
        malloc-128    128   11504      90       12113       0     126    1484032        0
       malloc-4096   4096     326       2         488       0       2    1343488        0
       UMA Slabs 0    112   10660      26       10660       0     126    1196832        0
      mbuf_cluster   2048     508       0         508       0     254    1040384        0
         vmem btag     56   15458     133       15458       0     254     873096        0
         FFS inode   1168     499      26         508       0       8     613200        0
         sctp_asoc   2256       1     254           1       0     254     575280        0
          lkpicurr    176       2    3080           2       0      62     542432        0
              pbuf   2624       0     198           0       0       2     519552        0
        RADIX NODE    144    3278     217       23511       0      62     503280        0
            socket    960      19     489        1313       0     254     487680        0
      malloc-65536  65536       4       2          48       0       1     393216        0
       malloc-2048   2048       6     162        1038       0       8     344064        0
           sctp_ep   1176       1     259           1       0     254     305760        0
        malloc-256    256    1016     109        1077       0      62     288000        0
        256 Bucket   2048     115      13         974       0       8     262144        0
         malloc-64     64    3792     303        3793       0     254     262080        0
         VM OBJECT    264     924      66       14124       0      30     261360        0
             VNODE    448     529      47         540       0      30     258048        0
         malloc-16     16   14379     371       14440       0     254     236000        0
       malloc-4096   4096      54       2        1268       0       2     229376        0
            DEVCTL   1024       0     220         123       0       0     225280        0
            THREAD   1824     113       7         113       0       8     218880        0
        malloc-128    128    1310     209       28599       0     126     194432        0
        sctp_raddr    736       1     263           1       0     254     194304        0
         UMA Zones    768     239       0         239       0      16     183552        0
         malloc-32     32    5277     393        5287       0     254     181440        0
            lkpimm     56       1    3095           1       0     254     173376        0
       FFS2 dinode    256     499      71         508       0      62     145920        0
      malloc-65536  65536       2       0           2       0       1     131072        0
      malloc-65536  65536       0       2         110       0       1     131072        0
      malloc-65536  65536       2       0           2       0       1     131072        0
       malloc-1024   1024     116      12         284       0      16     131072        0
        malloc-128    128     842     181         879       0     126     130944        0
             unpcb    256       7     503        1158       0     254     130560        0
       mbuf_packet    256       0     508         117       0     254     130048        0
       S VFS Cache    104     966     204        1004       0     126     121680        0
         MAP ENTRY     96     893     367       40277       0     126     120960        0
          ksiginfo    112      36    1008          51       0     126     116928        0
     FPU_save_area    832     115      20         127       0      16     112320        0
      malloc-32768  32768       3       0           3       0       1      98304        0
      malloc-32768  32768       2       1         122       0       1      98304        0
          UMA Kegs    384     226       7         226       0      30      89472        0
        128 Bucket   1024      41      42         250       0      16      84992        0
        malloc-128    128     421     230        4374       0     126      83328        0
      malloc-16384  16384       5       0           5       0       1      81920        0
       malloc-8192   8192       9       1          11       0       1      81920        0
              PROC   1376      39      16         773       0       8      75680        0
       malloc-8192   8192       7       2         108       0       1      73728        0
             g_bio    408       0     180        4481       0      30      73440        0
         malloc-64     64     849     222        2446       0     254      68544        0
         malloc-64     64     579     492       14541       0     254      68544        0
      malloc-65536  65536       0       1           8       0       1      65536        0
      malloc-32768  32768       2       0           2       0       1      65536        0
       malloc-1024   1024      20      44        1404       0      16      65536        0
        malloc-256    256     173      82        1209       0      62      65280        0
        malloc-256    256     149     106         620       0      62      65280        0
       malloc-4096   4096      12       2         222       0       2      57344        0
         udp_inpcb    424       6     120         126       0      30      53424        0
         filedesc0   1072      40       9         774       0       8      52528        0
         32 Bucket    256      54     141        1770       0      62      49920        0
           DIRHASH   1024      35      13          35       0      16      49152        0
             NAMEI   1024       0      48       12073       0      16      49152        0
      malloc-16384  16384       1       2         161       0       1      49152        0
       malloc-2048   2048      11      13          45       0       8      49152        0
          syncache    168       0     264           5       0     254      44352        0
              pipe    728       7      48         282       0      16      40040        0
            da_ccb    544       0      70        1253       0      16      38080        0
            pcpu-8      8    4267     341        4295       0     254      36864        0
         64 Bucket    512      67       5         977       0      30      36864        0
         malloc-64     64      60     507       14261       0     254      36288        0
         malloc-64     64      32     535          50       0     254      36288        0
         malloc-64     64      61     506         820       0     254      36288        0
        malloc-128    128      19     260         143       0     126      35712        0
        malloc-128    128      10     269          10       0     126      35712        0
        malloc-128    128       4     275           4       0     126      35712        0
     routing nhops    256      10     125          17       0      62      34560        0
           ttyoutq    256      72      63         160       0      62      34560        0
        malloc-384    384      49      41          71       0      30      34560        0
        malloc-384    384      62      28          64       0      30      34560        0
        malloc-256    256      34     101         452       0      62      34560        0
        malloc-256    256       1     134          16       0      62      34560        0
        malloc-256    256      11     124          11       0      62      34560        0
        malloc-256    256      39      96         638       0      62      34560        0
        malloc-256    256       3     132           3       0      62      34560        0
      malloc-32768  32768       1       0           1       0       1      32768        0
       malloc-4096   4096       6       2           6       0       2      32768        0
       malloc-2048   2048       2      14          14       0       8      32768        0
       malloc-2048   2048       2      14           2       0       8      32768        0
       malloc-2048   2048       8       8         249       0       8      32768        0
       malloc-1024   1024       5      27          45       0      16      32768        0
       malloc-1024   1024      16      16          16       0      16      32768        0
       malloc-1024   1024       9      23           9       0      16      32768        0
       malloc-1024   1024       4      28           4       0      16      32768        0
        malloc-512    512       1      63         119       0      30      32768        0
        malloc-512    512       3      61          22       0      30      32768        0
        malloc-512    512       3      61          54       0      30      32768        0
        malloc-512    512       8      56           8       0      30      32768        0
           pcpu-64     64     487      25         487       0     254      32768        0
            ttyinq    160     135      65         300       0      62      32000        0
              PGRP    120      20     244          31       0     126      31680        0
            clpbuf   2624       0      12          26       0       4      31488        0
           VMSPACE    616      23      25         758       0      16      29568        0
        sctp_laddr     48       0     588           4       0     254      28224        0
         malloc-32     32     287     595         630       0     254      28224        0
         16 Bucket    144      45     151         250       0      62      28224        0
          4 Bucket     48       6     582           9       0     254      28224        0
         TURNSTILE    136     121      68         121       0      62      25704        0
            cpuset    200       7     121           7       0      62      25600        0
             ripcb    392       1      62           4       0      30      24696        0
       malloc-8192   8192       2       1          28       0       1      24576        0
       malloc-8192   8192       3       0           3       0       1      24576        0
    ertt_txseginfo     40       1     605         292       0     254      24240        0
          rl_entry     40      29     577          29       0     254      24240        0
               PWD     40      10     596          99       0     254      24240        0
           rtentry    168      13     131          17       0      62      24192        0
             Files     80      72     228        6512       0     126      24000        0
          8 Bucket     80      53     247         283       0     126      24000        0
         tcp_inpcb   1312       3      15           7       0       8      23616        0
        malloc-384    384      11      49          11       0      30      23040        0
        malloc-384    384       3      57         364       0      30      23040        0
       Mountpoints   2816       2       6           2       0       4      22528        0
        SLEEPQUEUE     88     121     135         121       0     126      22528        0
         hostcache     64       1     314           1       0     254      20160        0
   udp_inpcb ports     32       3     627          40       0     254      20160        0
              ertt     72       3     277           7       0     126      20160        0
         malloc-64     64       2     313           2       0     254      20160        0
         malloc-64     64       1     314           1       0     254      20160        0
         malloc-32     32      78     552         104       0     254      20160        0
         malloc-32     32     166     464        2428       0     254      20160        0
         malloc-32     32      43     587          96       0     254      20160        0
         malloc-32     32      39     591        3104       0     254      20160        0
         malloc-32     32       5     625           5       0     254      20160        0
         malloc-32     32       2     628           5       0     254      20160        0
          2 Bucket     32      39     591         283       0     254      20160        0
        malloc-128    128       2     153           2       0     126      19840        0
 epoch_record pcpu    256       4      60           4       0      62      16384        0
      malloc-16384  16384       1       0           1       0       1      16384        0
       malloc-4096   4096       2       2          79       0       2      16384        0
       malloc-2048   2048       2       6           2       0       8      16384        0
        malloc-512    512       1      31           1       0      30      16384        0
        malloc-512    512       2      30           9       0      30      16384        0
        malloc-512    512       2      30           2       0      30      16384        0
           SMR CPU     32       7     504           7       0     254      16352        0
         malloc-16     16     504     496        3637       0     254      16000        0
              kenv    258      17      43        1069       0      30      15480        0
            mqnode    416       3      33           3       0      30      14976        0
              vmem   1856       1       7           1       0       8      14848        0
        SMR SHARED     24       7     504           7       0     254      12264        0
   tcp_inpcb ports     32       1     377           1       0     254      12096        0
             KNOTE    160       0      75           7       0      62      12000        0
         malloc-16     16      38     712        1292       0     254      12000        0
         malloc-16     16      12     738          18       0     254      12000        0
         malloc-16     16      30     720       27888       0     254      12000        0
         malloc-16     16      15     735          17       0     254      12000        0
        malloc-384    384       1      29           1       0      30      11520        0
        malloc-384    384      13      17          13       0      30      11520        0
       malloc-8192   8192       1       0           1       0       1       8192        0
       malloc-8192   8192       1       0           1       0       1       8192        0
       malloc-4096   4096       2       0           2       0       2       8192        0
           pcpu-16     16       8     504           8       0     254       8192        0
      vtnet_tx_hdr     24       0     334        2378       0     254       8016        0
         malloc-16     16      10     490          10       0     254       8000        0
         malloc-16     16      11     489          11       0     254       8000        0
       UMA Slabs 1    176       8      14           8       0      62       3872        0
        KMAP ENTRY     96      12      27          14       0       0       3744        0
       FFS1 dinode    128       0       0           0       0     126          0        0
           ada_ccb    272       0       0           0       0      30          0        0
             swblk    136       0       0           0       0      62          0        0
          swpctrie    144       0       0           0       0      62          0        0
   cdg_qdiffsample     16       0       0           0       0     254          0        0
   pf state scrubs     40       0       0           0       0     254          0        0
   pf frag entries     40       0       0           0       0     254          0        0
          pf frags    248       0       0           0       0      62          0        0
  pf table entries    160       0       0           0       0     254          0        0
pf table entry counters     64       0       0           0       0     254          0        0
   pf source nodes    152       0       0           0       0     254          0        0
     pf state keys     88       0       0           0       0     126          0        0
         pf states    360       0       0           0       0     254          0        0
           pf tags    104       0       0           0       0     126          0        0
          pf mtags    184       0       0           0       0      62          0        0
       tcp_bbr_pcb    832       0       0           0       0      16          0        0
       tcp_bbr_map    128       0       0           0       0     126          0        0
      tcp_rack_pcb   1024       0       0           0       0      16          0        0
      tcp_rack_map    128       0       0           0       0     126          0        0
tfo_ccache_entries     80       0       0           0       0     126          0        0
               tfo      4       0       0           0       0     254          0        0
          sackhole     32       0       0           0       0     254          0        0
               ipq     56       0       0           0       0     254          0        0
   sctp_asconf_ack     48       0       0           0       0     254          0        0
       sctp_asconf     40       0       0           0       0     254          0        0
sctp_stream_msg_out    112       0       0           0       0     254          0        0
        sctp_readq    152       0       0           0       0     254          0        0
        sctp_chunk    152       0       0           0       0     254          0        0
   tcp_log_id_node    120       0       0           0       0     126          0        0
 tcp_log_id_bucket    176       0       0           0       0      62          0        0
           tcp_log    416       0       0           0       0     254          0        0
          tcpreass     48       0       0           0       0     254          0        0
udplite_inpcb ports     32       0       0           0       0     254          0        0
     udplite_inpcb    424       0       0           0       0      30          0        0
       ripcb ports     32       0       0           0       0     254          0        0
    IPsec SA lft_c     16       0       0           0       0     254          0        0
            itimer    352       0       0           0       0      30          0        0
            AIOLIO    272       0       0           0       0      30          0        0
             AIOCB    552       0       0           0       0      16          0        0
               AIO    208       0       0           0       0      62          0        0
        mqnotifier    216       0       0           0       0      62          0        0
            mvdata     64       0       0           0       0     254          0        0
            mqueue    248       0       0           0       0      62          0        0
        TMPFS node    232       0       0           0       0      62          0        0
           NCLNODE    608       0       0           0       0      16          0        0
     LTS VFS Cache    360       0       0           0       0      30          0        0
       L VFS Cache    320       0       0           0       0      30          0        0
     STS VFS Cache    144       0       0           0       0      62          0        0
           cryptop    280       0       0           0       0      30          0        0
  linux_dma_object     32       0       0           0       0     254          0        0
  linux_dma_pctrie    144       0       0           0       0      62          0        0
   IOMMU_MAP_ENTRY    104       0       0           0       0     126          0        0
    mbuf_jumbo_16k  16384       0       0           0       0     254          0        0
     mbuf_jumbo_9k   9216       0       0           0       0     254          0        0
      audit_record   1280       0       0           0       0       8          0        0
         domainset     40       0       0           0       0     254          0        0
        MAC labels     40       0       0           0       0     254          0        0
            vnpbuf   2624       0       0           0       0      16          0        0
           nfspbuf   2624       0       0           0       0       4          0        0
            swwbuf   2624       0       0           0       0       2          0        0
            swrbuf   2624       0       0           0       0       4          0        0
          umtx_shm     88       0       0           0       0     126          0        0
           umtx pi     96       0       0           0       0     126          0        0
rangeset pctrie nodes    144       0       0           0       0      62          0        0
      malloc-65536  65536       0       0           0       0       1          0        0
      malloc-65536  65536       0       0           0       0       1          0        0
      malloc-65536  65536       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-16384  16384       0       0           0       0       1          0        0
      malloc-16384  16384       0       0           0       0       1          0        0
      malloc-16384  16384       0       0           0       0       1          0        0
      malloc-16384  16384       0       0           0       0       1          0        0
      malloc-16384  16384       0       0           0       0       1          0        0
       malloc-8192   8192       0       0           0       0       1          0        0
       malloc-8192   8192       0       0           0       0       1          0        0
       malloc-4096   4096       0       0           0       0       2          0        0
       malloc-4096   4096       0       0           0       0       2          0        0
       malloc-2048   2048       0       0           0       0       8          0        0
       malloc-2048   2048       0       0           0       0       8          0        0
       malloc-1024   1024       0       0           0       0      16          0        0
       malloc-1024   1024       0       0           0       0      16          0        0
        malloc-512    512       0       0           0       0      30          0        0
        malloc-384    384       0       0           0       0      30          0        0
           pcpu-32     32       0       0           0       0     254          0        0
            pcpu-4      4       0       0           0       0     254          0        0
            fakepg    104       0       0           0       0     126          0        0
          UMA Hash    256       0       0           0       0      62          0        0