panic: Assertion !tcp_in_hpts(tp) failed at /syzkaller/managers/main/kernel/sys/netinet/tcp_subr.c:2426
cpuid = 1
time = 1701769550
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0xc6/frame 0xfffffe0053fb3030
kdb_backtrace() at kdb_backtrace+0xd0/frame 0xfffffe0053fb3190
vpanic() at vpanic+0x271/frame 0xfffffe0053fb3330
panic() at panic+0xb5/frame 0xfffffe0053fb33f0
tcp_discardcb() at tcp_discardcb+0x5c4/frame 0xfffffe0053fb34e0
tcp_usr_detach() at tcp_usr_detach+0xd4/frame 0xfffffe0053fb3510
sorele_locked() at sorele_locked+0x24a/frame 0xfffffe0053fb3550
tcp_close() at tcp_close+0x380/frame 0xfffffe0053fb3610
tcp_twstart() at tcp_twstart+0x269/frame 0xfffffe0053fb3650
tcp_do_segment() at tcp_do_segment+0x4cdb/frame 0xfffffe0053fb38f0
tcp_input_with_port() at tcp_input_with_port+0x20b7/frame 0xfffffe0053fb3bb0
tcp_input() at tcp_input+0x1f/frame 0xfffffe0053fb3bd0
ip_input() at ip_input+0xac9/frame 0xfffffe0053fb3cf0
swi_net() at swi_net+0x2f3/frame 0xfffffe0053fb3d90
ithread_loop() at ithread_loop+0x4eb/frame 0xfffffe0053fb3ef0
fork_exit() at fork_exit+0xcc/frame 0xfffffe0053fb3f30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0053fb3f30
--- trap 0, rip = 0, rsp = 0, rbp = 0 ---
KDB: enter: panic
[ thread pid 12 tid 100031 ]
Stopped at      kdb_enter+0x6e: movq    $0,0x21936a7(%rip)
db> 
db> set $lines = 0
db> set $maxwidth = 0
db> show registers
cs                        0x20
ds                        0x3b
es                        0x3b
fs                        0x13
gs                        0x1b
ss                           0
rax                       0x12
rcx         0xfffffe00033eee30
rdx         0xdffff7c000000000
rbx         0xffffffff826edae0  .str.28
rsp         0xfffffe0053fb3170
rbp         0xfffffe0053fb3190
rsi                        0x1
rdi                          0
r8                           0
r9                  0xffffffff
r10                          0
r11                        0x2
r12                          0
r13         0xfffffe0054316740
r14         0xffffffff826edae0  .str.28
r15                          0
rip         0xffffffff815c796e  kdb_enter+0x6e
rflags                    0x46
kdb_enter+0x6e: movq    $0,0x21936a7(%rip)
db> show proc
Process 12 (intr) at 0xfffffe005424f580:
 state: NORMAL
 uid: 0  gids: 0
 parent: pid 0 at 0xffffffff836e5700
 ABI: null
 flag: 0x10000284  flag2: 0
 reaper: 0xffffffff836e5700 reapsubtree: 12
 sigparent: 20
 vmspace: 0xffffffff836e66a0
   (map 0xffffffff836e66a0)
   (map.pmap 0xffffffff836e6760)
   (pmap 0xffffffff836e67d0)
 threads: 22
100012                   I                                   [swi6: task queue]
100013                   I                                   [swi6: Giant taskq]
100015                   I                                   [swi5: fast taskq]
100031                   Run     CPU 1                       [swi1: netisr 0]
100032                   I                                   [swi1: hpts]
100033                   I                                   [swi1: hpts]
100046                   I                                   [irq24: virtio_pci0]
100047                   I                                   [irq25: virtio_pci0]
100048                   I                                   [irq26: virtio_pci0]
100049                   I                                   [irq27: virtio_pci0]
100050                   I                                   [irq28: virtio_pci1]
100051                   I                                   [irq29: virtio_pci1]
100052                   I                                   [irq30: virtio_pci1]
100053                   I                                   [irq31: virtio_pci1]
100054                   I                                   [irq32: virtio_pci1]
100059                   I                                   [irq33: virtio_pci2]
100060                   I                                   [irq34: virtio_pci2]
100061                   I                                   [irq35: virtio_pci2]
100063                   I                                   [irq1: atkbd0]
100064                   I                                   [irq12: psm0]
100065                   I                                   [swi0: uart uart++]
100069                   I                                   [swi1: pf send]
db> ps
  pid  ppid  pgrp   uid  state   wmesg   wchan               cmd
  771   769   771     0  Ss      pause   0xfffffe005720c610  csh
  769   682   769     0  Ss      select  0xfffffe0007876a40  sshd
  748     1   748     0  Ss+     ttyin   0xfffffe00574f48b0  getty
  747     1   747     0  Ss+     ttyin   0xfffffe00587a98b0  getty
  746     1   746     0  Ss+     ttyin   0xfffffe00587aa0b0  getty
  745     1   745     0  Ss+     ttyin   0xfffffe00587aa8b0  getty
  744     1   744     0  Ss+     ttyin   0xfffffe00543970b0  getty
  743     1   743     0  Ss+     ttyin   0xfffffe00543978b0  getty
  742     1   742     0  Ss+     ttyin   0xfffffe00543980b0  getty
  741     1   741     0  Ss+     ttyin   0xfffffe00543988b0  getty
  740     1   740     0  Ss+     ttyin   0xfffffe00543990b0  getty
  738     1    18     0  S+      piperd  0xfffffe0058ad2998  logger
  737   736    18     0  S+      nanslp  0xffffffff8373c3c0  sleep
  736     1    18     0  S+      wait    0xfffffe0057a51040  sh
  686     1   686     0  Ss      nanslp  0xffffffff8373c3c0  cron
  682     1   682     0  Ss      select  0xfffffe0057920dc0  sshd
  495     1   495     0  Ss      select  0xfffffe0057921f40  syslogd
  424     1   424     0  Ss      select  0xfffffe0057922140  devd
  423     1   423    65  Ss      select  0xfffffe0007877dc0  dhclient
  338     1   338     0  Ss      select  0xfffffe00078775c0  dhclient
  335     1   335     0  Ss      select  0xfffffe0057921ec0  dhclient
   17     0     0     0  DL      syncer  0xffffffff83859b20  [syncer]
   16     0     0     0  DL      vlruwt  0xfffffe005720e040  [vnlru]
   15     0     0     0  DL      (threaded)                  [bufdaemon]
100079                   D       psleep  0xffffffff83858100  [bufdaemon]
100082                   D       -       0xffffffff82c0a140  [bufspacedaemon-0]
100095                   D       sdflush 0xfffffe0058b9cce8  [/ worker]
    9     0     0     0  DL      psleep  0xffffffff838cad40  [vmdaemon]
    8     0     0     0  DL      (threaded)                  [pagedaemon]
100077                   D       psleep  0xffffffff838b2bb8  [dom0]
100080                   D       launds  0xffffffff838b2bc4  [laundry: dom0]
100081                   D       umarcl  0xffffffff81d5a950  [uma]
    7     0     0     0  DL      -       0xffffffff834b9c10  [rand_harvestq]
    6     0     0     0  DL      pftm    0xffffffff84417a30  [pf purge]
    5     0     0     0  DL      waiting 0xffffffff842154e0  [sctp_iterator]
    4     0     0     0  DL      (threaded)                  [cam]
100044                   D       -       0xffffffff83484340  [doneq0]
100045                   D       -       0xffffffff834842c0  [async]
100076                   D       -       0xffffffff83484140  [scanner]
    3     0     0     0  DL      (threaded)                  [crypto]
100041                   D       crypto_ 0xffffffff838ae3e0  [crypto]
100042                   D       crypto_ 0xfffffe00571b2030  [crypto returns 0]
100043                   D       crypto_ 0xfffffe00571b2080  [crypto returns 1]
   14     0     0     0  DL      seqstat 0xfffffe00085f4488  [sequencer 00]
   13     0     0     0  DL      (threaded)                  [geom]
100035                   D       -       0xffffffff836e4d20  [g_event]
100036                   D       -       0xffffffff836e4d40  [g_up]
100037                   D       -       0xffffffff836e4d60  [g_down]
    2     0     0     0  WL      (threaded)                  [clock]
100029                   I                                   [clock (0)]
100030                   I                                   [clock (1)]
   12     0     0     0  RL      (threaded)                  [intr]
100012                   I                                   [swi6: task queue]
100013                   I                                   [swi6: Giant taskq]
100015                   I                                   [swi5: fast taskq]
100031                   Run     CPU 1                       [swi1: netisr 0]
100032                   I                                   [swi1: hpts]
100033                   I                                   [swi1: hpts]
100046                   I                                   [irq24: virtio_pci0]
100047                   I                                   [irq25: virtio_pci0]
100048                   I                                   [irq26: virtio_pci0]
100049                   I                                   [irq27: virtio_pci0]
100050                   I                                   [irq28: virtio_pci1]
100051                   I                                   [irq29: virtio_pci1]
100052                   I                                   [irq30: virtio_pci1]
100053                   I                                   [irq31: virtio_pci1]
100054                   I                                   [irq32: virtio_pci1]
100059                   I                                   [irq33: virtio_pci2]
100060                   I                                   [irq34: virtio_pci2]
100061                   I                                   [irq35: virtio_pci2]
100063                   I                                   [irq1: atkbd0]
100064                   I                                   [irq12: psm0]
100065                   I                                   [swi0: uart uart++]
100069                   I                                   [swi1: pf send]
   11     0     0     0  RL      (threaded)                  [idle]
100003                   CanRun                              [idle: cpu0]
100004                   CanRun                              [idle: cpu1]
    1     0     1     0  RLs                                 [init]
   10     0     0     0  DL      audit_w 0xffffffff838aee40  [audit]
    0     0     0     0  DLs     (threaded)                  [kernel]
100000                   D       swapin  0xffffffff836e5700  [swapper]
100005                   D       -       0xfffffe0054270000  [softirq_0]
100006                   D       -       0xfffffe005426fe00  [softirq_1]
100007                   D       -       0xfffffe005426fd00  [if_io_tqg_0]
100008                   D       -       0xfffffe005426fc00  [if_io_tqg_1]
100009                   D       -       0xfffffe005426fb00  [if_config_tqg_0]
100010                   D       -       0xfffffe00085fa000  [pci_hp taskq]
100011                   D       -       0xfffffe00085f9e00  [kqueue_ctx taskq]
100014                   D       -       0xfffffe00085f9b00  [thread taskq]
100016                   D       -       0xfffffe00085f9900  [aiod_kick taskq]
100017                   D       -       0xfffffe00085f9800  [deferred_unmount ta]
100018                   D       -       0xfffffe00085f9700  [inm_free taskq]
100019                   D       -       0xfffffe00085f9600  [in6m_free taskq]
100020                   D       -       0xfffffe00085f9500  [linuxkpi_irq_wq]
100021                   D       -       0xfffffe00085f9400  [linuxkpi_short_wq_0]
100022                   D       -       0xfffffe00085f9400  [linuxkpi_short_wq_1]
100023                   D       -       0xfffffe00085f9400  [linuxkpi_short_wq_2]
100024                   D       -       0xfffffe00085f9400  [linuxkpi_short_wq_3]
100025                   D       -       0xfffffe00085f9300  [linuxkpi_long_wq_0]
100026                   D       -       0xfffffe00085f9300  [linuxkpi_long_wq_1]
100027                   D       -       0xfffffe00085f9300  [linuxkpi_long_wq_2]
100028                   D       -       0xfffffe00085f9300  [linuxkpi_long_wq_3]
100034                   D       -       0xfffffe00085f9100  [firmware taskq]
100039                   D       -       0xfffffe00085f8e00  [crypto_0]
100040                   D       -       0xfffffe00085f8e00  [crypto_1]
100055                   D       -       0xfffffe0057293e00  [vtnet0 rxq 0]
100056                   D       -       0xfffffe0057293d00  [vtnet0 txq 0]
100057                   D       -       0xfffffe0057293c00  [vtnet0 rxq 1]
100058                   D       -       0xfffffe0057293b00  [vtnet0 txq 1]
100062                   D       vtbslp  0xfffffe0007dd7e00  [virtio_balloon]
100066                   D       -       0xffffffff826f2d41  [deadlkres]
100070                   D       -       0xfffffe005798ec00  [acpi_task_0]
100071                   D       -       0xfffffe005798ec00  [acpi_task_1]
100072                   D       -       0xfffffe005798ec00  [acpi_task_2]
100074                   D       -       0xfffffe00085fa100  [mca taskq]
100075                   D       -       0xfffffe00085f8c00  [CAM taskq]
  773   771   771     0  RE      CPU 0                       syz-executor2448255
  774     1   771     0  Z                                   syz-executor2448255
db> show all locks
Process 12 (intr) thread 0xfffffe0054316740 (100031)
exclusive rw tcpinp (tcpinp) r = 0 (0xfffffe006ce51560) locked @ /syzkaller/managers/main/kernel/sys/netinet/tcp_usrreq.c:205
db> show malloc
              Type        InUse        MemUse     Requests
           pf_hash            5        11524K            5
          tcp_hpts            7         4801K            7
            devbuf         4192         4324K         4217
         sysctloid        34934         2059K        35005
             vtbuf           24         1968K           46
              kobj          326         1304K          488
            newblk          849         1236K          874
          vfscache            3         1025K            3
               pcb           23          669K           42
          inodedep           51          531K           73
         ufs_quota            1          512K            1
          vfs_hash            1          512K            1
           callout            2          512K            2
              intr            4          472K            4
         vnet_data            2          224K            2
          acpitask            1          224K            1
           subproc           97          185K          831
            acpica         1674          184K        60830
           tidhash            3          141K            3
              vmem            3          134K            4
           pagedep           14          132K           18
            linker          352          130K          386
        tfo_ccache            1          128K            1
          IP reass            1          128K            1
               sem            4          106K            4
            DEVFS1          105          105K          114
        gtaskqueue           18           98K           18
               bus          985           81K         5155
          mtx_pool            2           72K            2
          syncache            1           68K            1
     NFSD srvcache            3           68K            3
            module          512           64K          512
       ddb_capture            1           64K            1
              temp           18           37K         1613
         hostcache            1           32K            1
               shm            1           32K            1
           kdtrace          157           32K          891
            DEVFS3          124           31K          134
               msg            4           30K            4
              umtx          240           30K          240
            kbdmux            6           28K            6
        DEVFS_RULE           56           20K           56
               BPF           10           18K           10
         ufs_mount            4           17K            5
              proc            3           17K            3
               tty           16           16K           16
           ithread           97           16K           97
            bus-sc           34           15K         1687
      eventhandler          161           14K          161
            KTRACE          100           13K          100
              kenv           95           12K           95
            ifaddr           30           12K           32
              GEOM           61           11K          481
          routetbl           50           11K          176
              rman           86           11K          429
         CAM queue            5           11K         1528
         bmsafemap            3            9K           42
               rpc            4            9K            4
           devstat            4            9K            4
              UART           12            9K           12
              ksem            1            8K            1
             shmfd            1            8K            1
       pfs_vncache            1            8K            1
     audit_evclass          238            8K          300
         taskqueue           63            7K           63
            sglist            6            7K            6
           CAM DEV            3            6K          510
              cred           23            6K          274
         pfs_nodes           20            5K           20
             hhook           15            5K           17
       ufs_dirhash           24            5K           24
               UMA          268            5K          268
            dirrem           17            5K           28
            plimit           17            5K          322
             ifnet            3            5K            3
                vt           11            5K           11
           memdesc            1            4K            1
               MCA           32            4K           32
          filedesc            1            4K            1
             evdev            4            4K            4
           acpisem           28            4K           28
            diradd           27            4K           38
       ether_multi           40            4K           50
           lltable           11            4K           11
          pf_ifnet            5            3K            6
         in6_multi           25            3K           25
          terminal           11            3K           11
            kqueue           41            3K          777
           session           20            3K           31
           pwddesc           39            3K          775
             clone            9            3K            9
           uidinfo            3            3K            8
         proc-args           61            3K         1697
        local_apic            1            2K            1
           io_apic            1            2K            1
         ipsec-saq            2            2K            2
             selfd           27            2K        12800
             lockf           16            2K           26
            Unitno           27            2K           41
           CAM XPT           22            2K          543
               msi           12            2K           12
         toponodes            6            2K            6
       ipsecpolicy            2            2K            2
           acpidev           20            2K           20
           softdep            1            1K            1
          indirdep            4            1K            4
            sahead            1            1K            1
          secasvar            1            1K            1
       vnodemarker            2            1K            8
      NFSD session            1            1K            1
            select            7            1K           29
        CAM periph            4            1K          271
             ipsec            3            1K            3
            CC Mem            3            1K            8
             nhops            6            1K            6
              pfil            6            1K            6
            isadev            6            1K            6
             mount           16            1K           89
          pci_link           10            1K           10
          sctp_ifa            5            1K            6
            crypto            4            1K            4
            ip6ndp            4            1K            5
 encap_export_host           12            1K           12
         newdirblk            4            1K            8
             mkdir            4            1K           16
          freefrag            4            1K            5
          in_multi            2            1K            4
              cdev            2            1K            2
               osd            8            1K           21
           netlink            2            1K            2
    chacha20random            1            1K            1
            biobuf            1            1K            1
       inpcbpolicy           11            1K          140
          sctp_ifn            2            1K            6
               mld            2            1K            2
              igmp            2            1K            2
            vnodes            1            1K            1
      NFSD lckfile            1            1K            1
     NFSD V4client            1            1K            1
            DEVFSP            4            1K            9
             DEVFS            9            1K           10
           CAM SIM            2            1K            2
            feeder            7            1K            7
           tcpfunc            3            1K            3
        loginclass            3            1K            7
            prison            6            1K            6
       lkpikmalloc            5            1K            6
         cryptodev            2            1K           49
          nexusdev            8            1K            8
            apmdev            1            1K            1
          atkbddev            2            1K            2
          procdesc            1            1K            6
          pmchooks            1            1K            1
          CAM path            4            1K         1034
     CAM dev queue            2            1K            2
 CAM I/O Scheduler            1            1K            1
            soname            4            1K         3330
          filecaps            4            1K           66
               tun            3            1K            3
          sctp_vrf            1            1K            1
          freework            1            1K           26
              vnet            1            1K            1
               pmc            1            1K            1
           entropy            2            1K           35
          acpiintr            1            1K            1
              cpus            2            1K            2
    vnet_data_free            1            1K            1
           Per-cpu            1            1K            1
          p1003.1b            1            1K            1
            tcp_do            0            0K            0
           tcp_fsb            0            0K            0
          pf_table            0            0K            0
           pf_rule            0            0K            0
           pf_altq            0            0K            0
           pf_osfp            0            0K            0
     pf_krule_item            0            0K            0
           pf_temp            0            0K            0
            mqdata            0            0K            0
        sctp_mcore            0            0K            0
        sctp_socko            0            0K            0
         sctp_iter            0            0K            4
         sctp_mvrf            0            0K            0
         sctp_timw            0            0K            0
         sctp_cpal            0            0K            0
         sctp_cmsg            0            0K            0
         sctp_stre            0            0K            0
         sctp_athi            0            0K            0
         sctp_athm            0            0K            0
         sctp_atky            0            0K            0
         sctp_atcl            0            0K            0
         sctp_a_it            0            0K            4
         sctp_aadr            0            0K            0
         sctp_stro            0            0K            0
         sctp_stri            0            0K            0
          sctp_map            0            0K            0
           filemon            0            0K            0
            ipcomp            0            0K            0
               esp            0            0K            0
                ah            0            0K            0
        madt_table            0            0K            2
          smartpqi            0            0K            0
               ixl            0            0K            0
        ice-resmgr            0            0K            0
         ice-osdep            0            0K            0
               ice            0            0K            0
              iavf            0            0K            0
             axgbe            0            0K            0
       fpukern_ctx            0            0K            0
          xen_intr            0            0K            0
           xen_hvm            0            0K            0
         legacydrv            0            0K            0
            bounce            0            0K            0
            busdma            0            0K            0
            qpidrv            0            0K            0
      dmar_idpgtbl            0            0K            0
          dmar_dom            0            0K            0
          dmar_ctx            0            0K            0
              isci            0            0K            0
      iommu_dmamap            0            0K            0
     hyperv_socket            0            0K            0
           bxe_ilt            0            0K            0
        aesni_data            0            0K            0
            xenbus            0            0K            0
     vm_fictitious            0            0K            0
           UMAHash            0            0K            0
         vm_pgdata            0            0K            0
           jblocks            0            0K            0
          savedino            0            0K           15
          sentinel            0            0K            0
            jfsync            0            0K            0
            jtrunc            0            0K            0
             sbdep            0            0K            2
           jsegdep            0            0K            0
              jseg            0            0K            0
         jfreefrag            0            0K            0
          jfreeblk            0            0K            0
           jnewblk            0            0K            0
            jmvref            0            0K            0
           jremref            0            0K            0
           jaddref            0            0K            0
           freedep            0            0K            0
          freefile            0            0K            9
          freeblks            0            0K           25
        allocindir            0            0K            0
       allocdirect            0            0K            0
          ufs_trim            0            0K            0
           mactemp            0            0K            0
     audit_trigger            0            0K            0
 audit_pipe_presel            0            0K            0
     audit_pipeent            0            0K            0
        audit_pipe            0            0K            0
      audit_evname            0            0K            0
         audit_bsm            0            0K            0
      audit_gidset            0            0K            0
        audit_text            0            0K            0
        audit_path            0            0K            0
        audit_data            0            0K            0
        audit_cred            0            0K            0
          ktls_ocf            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
      MLX5E_TLS_RX            0            0K            0
        MLX5EEPROM            0            0K            0
         MLX5E_TLS            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
            MLX5EN            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
          MLX5DUMP            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
       simple_attr            0            0K            0
          seq_file            0            0K            0
           lkpiskb            0            0K            0
             radix            0            0K            0
               idr            0            0K            0
          lkpindev            0            0K            0
           lkpimhi            0            0K            0
            lkpifw            0            0K            0
         lkpi80211            0            0K            0
               NLM            0            0K            0
    ipsec-spdcache            0            0K            0
         ipsec-reg            0            0K            0
        ipsec-misc            0            0K            0
      ipsecrequest            0            0K            0
            ip6opt            0            0K            3
       ip6_msource            0            0K            0
      ip6_moptions            0            0K            0
       in6_mfilter            0            0K            0
             frag6            0            0K            0
            tcplog            0            0K            0
        tcp_hwpace            0            0K            0
               LRO            0            0K            0
        ip_msource            0            0K            0
       ip_moptions            0            0K            0
        in_mfilter            0            0K            0
              ipid            0            0K            0
         80211scan            0            0K            0
      80211ratectl            0            0K            0
        80211power            0            0K            0
       80211nodeie            0            0K            0
         80211node            0            0K            0
      80211mesh_gt            0            0K            0
      80211mesh_rt            0            0K            0
         80211perr            0            0K            0
         80211prep            0            0K            0
         80211preq            0            0K            0
          80211dfs            0            0K            0
       80211crypto            0            0K            0
          80211vap            0            0K            0
             iflib            0            0K            0
              vlan            0            0K            0
               gif            0            0K            0
           ifdescr            0            0K            0
              zlib            0            0K            0
           fadvise            0            0K            0
           VN POLL            0            0K            0
            statfs            0            0K          195
     namei_tracker            0            0K            0
       export_host            0            0K            0
        cl_savebuf            0            0K            3
               aio            0            0K            0
               lio            0            0K            0
               acl            0            0K            0
          mbuf_tag            0            0K            0
              ktls            0            0K            0
              accf            0            0K            0
               pts            0            0K            0
           timerfd            0            0K            0
               iov            0            0K        13582
          ioctlops            0            0K           86
           eventfd            0            0K            0
           Witness            0            0K            0
             stack            0            0K            0
              sbuf            0            0K          296
          firmware            0            0K            0
        compressor            0            0K            0
              SWAP            0            0K            0
         sysctltmp            0            0K          649
            sysctl            0            0K            3
              ekcd            0            0K            0
            dumper            0            0K            0
          sendfile            0            0K            0
              rctl            0            0K            0
             cache            0            0K            0
          kcovinfo            0            0K            0
      prison_racct            0            0K            0
       Fail Points            0            0K            0
             sigio            0            0K            1
filedesc_to_leader            0            0K            0
               pwd            0            0K            0
       tty console            0            0K            0
         boottrace            0            0K            0
        isofs_node            0            0K            0
       isofs_mount            0            0K            0
     tr_raid5_data            0            0K            0
    tr_raid1e_data            0            0K            0
     tr_raid1_data            0            0K            0
     tr_raid0_data            0            0K            0
    tr_concat_data            0            0K            0
       md_sii_data            0            0K            0
   md_promise_data            0            0K            0
    md_nvidia_data            0            0K            0
   md_jmicron_data            0            0K            0
     md_intel_data            0            0K            0
       md_ddf_data            0            0K            0
         raid_data            0            0K           72
     geom_flashmap            0            0K            0
         tmpfs dir            0            0K            0
        tmpfs name            0            0K            0
       tmpfs mount            0            0K            0
     tmpfs extattr            0            0K            0
           NFS FHA            0            0K            0
         newnfsmnt            0            0K            0
  newnfsclient_req            0            0K            0
   NFSCL layrecall            0            0K            0
     NFSCL session            0            0K            0
     NFSCL sockreq            0            0K            0
     NFSCL devinfo            0            0K            0
     NFSCL flayout            0            0K            0
      NFSCL layout            0            0K            0
     NFSD rollback            0            0K            0
      NFSCL diroff            0            0K            0
       NEWdirectio            0            0K            0
        NEWNFSnode            0            0K            0
         NFSCL lck            0            0K            0
      NFSCL lckown            0            0K            0
      NFSCL client            0            0K            0
       NFSCL deleg            0            0K            0
        NFSCL open            0            0K            0
       NFSCL owner            0            0K            0
            NFS fh            0            0K            0
           NFS req            0            0K            0
     NFSD usrgroup            0            0K            0
       NFSD string            0            0K            0
       NFSD V4lock            0            0K            0
      NFSD V4state            0            0K            0
       msdosfs_fat            0            0K            0
     msdosfs_mount            0            0K            0
      msdosfs_node            0            0K            0
            DEVFS4            0            0K            0
            DEVFS2            0            0K            0
            gntdev            0            0K            0
       privcmd_dev            0            0K            0
        evtchn_dev            0            0K            0
          xenstore            0            0K            0
               xnb            0            0K            0
          xen_acpi            0            0K            0
              xbbd            0            0K            0
               xbd            0            0K            0
           Balloon            0            0K            0
          sysmouse            0            0K            0
            vtfont            0            0K            0
            pvscsi            0            0K            0
            USBdev            0            0K            0
               USB            0            0K            0
            twsbuf            0            0K            0
       tcp_log_dev            0            0K            0
      midi buffers            0            0K            0
             mixer            0            0K            0
              ac97            0            0K            0
             hdacc            0            0K            0
              hdac            0            0K            0
              hdaa            0            0K            0
       SIIS driver            0            0K            0
               PUC            0            0K            0
          ppbusdev            0            0K            0
            sr_iov            0            0K            0
               OCS            0            0K            0
               OCS            0            0K            0
              nvme            0            0K            0
               nvd            0            0K            0
            netmap            0            0K            0
            mwldev            0            0K            0
        MVS driver            0            0K            0
          mrsasbuf            0            0K            0
          mpt_user            0            0K            0
          mps_user            0            0K            0
            MPSSAS            0            0K            0
               mps            0            0K            0
          mpr_user            0            0K            0
            MPRSAS            0            0K            0
               mpr            0            0K            0
            mfibuf            0            0K            0
        md_sectors            0            0K            0
           md_disk            0            0K            0
           malodev            0            0K            0
               LED            0            0K            0
          ix_sriov            0            0K            0
                ix            0            0K            0
            ipsbuf            0            0K            0
         ciss_data            0            0K            0
         BACKLIGHT            0            0K            0
           ath_hal            0            0K            0
            athdev            0            0K            0
           ata_pci            0            0K            0
           ata_dma            0            0K            0
       ata_generic            0            0K            0
       AHCI driver            0            0K            0
               agp            0            0K            0
           acpipwr            0            0K            0
         acpi_perf            0            0K            0
         acpicmbat            0            0K            0
        aacraidcam            0            0K            0
       aacraid_buf            0            0K            0
            aaccam            0            0K            0
            aacbuf            0            0K            0
              zstd            0            0K            0
            XZ_DEC            0            0K            0
            nvlist            0            0K            0
          SCSI ENC            0            0K            0
           SCSI sa            0            0K            0
         scsi_pass            0            0K            0
           scsi_da            0            0K           69
            ata_da            0            0K            0
           scsi_ch            0            0K            0
           scsi_cd            0            0K            0
           nvme_da            0            0K            0
           CAM CCB            0            0K          523
     CAM ccb queue            0            0K            0
db> show uma
              Zone   Size    Used    Free    Requests  Sleeps  Bucket  Total Mem    XFree
   mbuf_jumbo_page   4096    8320    1078       15703       0     254   38494208        0
              mbuf    256    8579    1083       19264       0     254    2473472        0
          BUF TRIE    144     205   11611         562       0      62    1701504        0
        malloc-384    384    4118      52        4119       0      30    1601280        0
        malloc-128    128   11749     124       11841       0     126    1519744        0
       malloc-4096   4096     327       3         489       0       2    1351680        0
       UMA Slabs 0    112   10662      24       10662       0     126    1196832        0
      mbuf_cluster   2048     508       0         508       0     254    1040384        0
         vmem btag     56   15471      48       15471       0     254     869064        0
         FFS inode   1168     501      17         510       0       8     605024        0
            lkpimm    168       1    3095           1       0      62     520128        0
          lkpicurr    168       2    3094           2       0      62     520128        0
              pbuf   2624       0     198           0       0       2     519552        0
        RADIX NODE    144    3273     222       28012       0      62     503280        0
            socket    960      19     489        1315       0     254     487680        0
      malloc-65536  65536       6       0           6       0       1     393216        0
         malloc-64     64    4290     309       17274       0     254     294336        0
        malloc-256    256    1089      36        1896       0      62     288000        0
        256 Bucket   2048     116      20         841       0       8     278528        0
         VM OBJECT    264     921      69       14120       0      30     261360        0
             VNODE    448     531      45         542       0      30     258048        0
         malloc-16     16   14620     380       17611       0     254     240000        0
            DEVCTL   1024       0     220         123       0       0     225280        0
            THREAD   1824     115       5         115       0       8     218880        0
         UMA Zones    768     240       4         240       0      16     187392        0
         malloc-32     32    5331     339        6101       0     254     181440        0
       malloc-4096   4096      41       3         775       0       2     180224        0
        malloc-128    128    1211     184       28428       0     126     178560        0
       malloc-2048   2048       6      74        1038       0       8     163840        0
       malloc-1024   1024     129      31         138       0      16     163840        0
        malloc-128    128     960     187        5102       0     126     146816        0
       FFS2 dinode    256     501      69         510       0      62     145920        0
      malloc-65536  65536       0       2          46       0       1     131072        0
      malloc-65536  65536       0       2         110       0       1     131072        0
             unpcb    256       7     503        1158       0     254     130560        0
       mbuf_packet    256       0     508         117       0     254     130048        0
       S VFS Cache    104     967     203        1009       0     126     121680        0
         MAP ENTRY     96     881     379       40288       0     126     120960        0
     FPU_save_area    832     117      27         129       0      16     119808        0
          ksiginfo    112      39    1005          55       0     126     116928        0
      malloc-32768  32768       1       2         121       0       1      98304        0
      malloc-32768  32768       3       0           3       0       1      98304        0
          UMA Kegs    384     227       6         227       0      30      89472        0
        128 Bucket   1024      41      42         238       0      16      84992        0
       malloc-8192   8192       8       2          10       0       1      81920        0
       malloc-4096   4096      18       2          29       0       2      81920        0
        malloc-256    256     205     110         459       0      62      80640        0
              PROC   1376      40      15         774       0       8      75680        0
         malloc-64     64     534     537        1679       0     254      68544        0
      malloc-65536  65536       1       0           1       0       1      65536        0
      malloc-65536  65536       1       0           1       0       1      65536        0
      malloc-65536  65536       0       1           8       0       1      65536        0
      malloc-32768  32768       2       0           2       0       1      65536        0
             g_bio    408       0     150        4749       0      30      61200        0
       malloc-8192   8192       6       1         107       0       1      57344        0
         64 Bucket    512      62      42        1030       0      30      53248        0
         filedesc0   1072      39      10         775       0       8      52528        0
         32 Bucket    256      58     137        1490       0      62      49920        0
           DIRHASH   1024      35      13          35       0      16      49152        0
             NAMEI   1024       0      48       12075       0      16      49152        0
      malloc-16384  16384       3       0           3       0       1      49152        0
       malloc-1024   1024       8      40        1430       0      16      49152        0
          syncache    168       0     264           5       0     254      44352        0
       malloc-8192   8192       4       1          30       0       1      40960        0
       malloc-4096   4096       7       3          12       0       2      40960        0
         udp_inpcb    424       6      84         128       0      30      38160        0
            pcpu-8      8    4271     337        4299       0     254      36864        0
         malloc-64     64      47     520         275       0     254      36288        0
         malloc-64     64      14     553       13276       0     254      36288        0
         malloc-64     64      60     507        1365       0     254      36288        0
         malloc-64     64      64     503         839       0     254      36288        0
         malloc-64     64     264     303        1038       0     254      36288        0
         malloc-64     64     116     451         121       0     254      36288        0
        malloc-128    128       7     272          71       0     126      35712        0
        malloc-128    128      29     250          39       0     126      35712        0
        malloc-128    128      79     200         171       0     126      35712        0
        malloc-128    128      62     217         210       0     126      35712        0
        malloc-128    128      59     220         313       0     126      35712        0
     routing nhops    256      10     125          17       0      62      34560        0
           ttyoutq    256      72      63         160       0      62      34560        0
        malloc-384    384      52      38          71       0      30      34560        0
        malloc-384    384      62      28          85       0      30      34560        0
        malloc-384    384      70      20          70       0      30      34560        0
        malloc-256    256       2     133         103       0      62      34560        0
        malloc-256    256      30     105         321       0      62      34560        0
        malloc-256    256      60      75         502       0      62      34560        0
        malloc-256    256      31     104         394       0      62      34560        0
        malloc-256    256      17     118         115       0      62      34560        0
        malloc-256    256      13     122         269       0      62      34560        0
      malloc-32768  32768       1       0           1       0       1      32768        0
      malloc-32768  32768       1       0           1       0       1      32768        0
      malloc-16384  16384       0       2         160       0       1      32768        0
      malloc-16384  16384       2       0           2       0       1      32768        0
       malloc-4096   4096       4       4         754       0       2      32768        0
       malloc-2048   2048       2      14           2       0       8      32768        0
       malloc-2048   2048       3      13           3       0       8      32768        0
       malloc-2048   2048       2      14          43       0       8      32768        0
       malloc-2048   2048       5      11          14       0       8      32768        0
       malloc-2048   2048       0      16          50       0       8      32768        0
       malloc-2048   2048       7       9           7       0       8      32768        0
       malloc-2048   2048       5      11         196       0       8      32768        0
       malloc-1024   1024       0      32           4       0      16      32768        0
       malloc-1024   1024      10      22          11       0      16      32768        0
       malloc-1024   1024       4      28           8       0      16      32768        0
       malloc-1024   1024       5      27         175       0      16      32768        0
       malloc-1024   1024      10      22          11       0      16      32768        0
       malloc-1024   1024       3      29           3       0      16      32768        0
        malloc-512    512      13      51          19       0      30      32768        0
        malloc-512    512       0      64         131       0      30      32768        0
        malloc-512    512       1      63          11       0      30      32768        0
        malloc-512    512       4      60          53       0      30      32768        0
           pcpu-64     64     487      25         487       0     254      32768        0
            ttyinq    160     135      65         300       0      62      32000        0
              PGRP    120      20     244          31       0     126      31680        0
            clpbuf   2624       0      12          26       0       4      31488        0
           VMSPACE    616      22      26         758       0      16      29568        0
        sctp_laddr     48       0     588           4       0     254      28224        0
         malloc-32     32     283     599        4423       0     254      28224        0
         16 Bucket    144      54     142         257       0      62      28224        0
          4 Bucket     48       6     582           8       0     254      28224        0
            da_ccb    544       0      49        1320       0      16      26656        0
         TURNSTILE    136     121      68         121       0      62      25704        0
            cpuset    200       7     121           7       0      62      25600        0
             ripcb    392       1      62           4       0      30      24696        0
       malloc-4096   4096       5       1           5       0       2      24576        0
    ertt_txseginfo     40       0     606         293       0     254      24240        0
          rl_entry     40      31     575          31       0     254      24240        0
               PWD     40      10     596          99       0     254      24240        0
           rtentry    168      13     131          17       0      62      24192        0
              pipe    728       7      26         282       0      16      24024        0
             Files     80      71     229        6515       0     126      24000        0
          8 Bucket     80      45     255         482       0     126      24000        0
         tcp_inpcb   1312       4      14           8       0       8      23616        0
        malloc-384    384       1      59         346       0      30      23040        0
        malloc-384    384      34      26          37       0      30      23040        0
        malloc-384    384       4      56           4       0      30      23040        0
       Mountpoints   2816       2       6           2       0       4      22528        0
        SLEEPQUEUE     88     121     135         121       0     126      22528        0
         hostcache     64       1     314           1       0     254      20160        0
   udp_inpcb ports     32       3     627          40       0     254      20160        0
              ertt     72       3     277           8       0     126      20160        0
         malloc-32     32      26     604         177       0     254      20160        0
         malloc-32     32      54     576         197       0     254      20160        0
         malloc-32     32      37     593          43       0     254      20160        0
         malloc-32     32      74     556         129       0     254      20160        0
         malloc-32     32      16     614          17       0     254      20160        0
          2 Bucket     32      48     582         291       0     254      20160        0
       tcp_bbr_map    128       0     155           3       0     126      19840        0
 epoch_record pcpu    256       4      60           4       0      62      16384        0
      malloc-16384  16384       1       0           1       0       1      16384        0
      malloc-16384  16384       1       0           1       0       1      16384        0
      malloc-16384  16384       0       1           1       0       1      16384        0
       malloc-8192   8192       2       0           2       0       1      16384        0
        malloc-512    512       2      30           2       0      30      16384        0
        malloc-512    512       0      32           1       0      30      16384        0
           SMR CPU     32       7     504           7       0     254      16352        0
              kenv    258      17      43        1069       0      30      15480        0
       tcp_bbr_pcb    832       0      18           1       0      16      14976        0
            mqnode    416       3      33           3       0      30      14976        0
              vmem   1856       1       7           1       0       8      14848        0
        SMR SHARED     24       7     504           7       0     254      12264        0
   tcp_inpcb ports     32       1     377           2       0     254      12096        0
         malloc-32     32      89     289         643       0     254      12096        0
             KNOTE    160       0      75           8       0      62      12000        0
         malloc-16     16      22     728        1220       0     254      12000        0
         malloc-16     16      15     735          47       0     254      12000        0
         malloc-16     16      42     708         252       0     254      12000        0
         malloc-16     16      23     727          39       0     254      12000        0
         malloc-16     16      31     719         158       0     254      12000        0
         malloc-16     16     284     466       28025       0     254      12000        0
         malloc-16     16       7     743           9       0     254      12000        0
        malloc-384    384       0      30           1       0      30      11520        0
       malloc-8192   8192       1       0           1       0       1       8192        0
       malloc-8192   8192       1       0           1       0       1       8192        0
       malloc-8192   8192       1       0           1       0       1       8192        0
       malloc-4096   4096       1       1           2       0       2       8192        0
           pcpu-16     16       8     504           8       0     254       8192        0
      vtnet_tx_hdr     24       0     334        2588       0     254       8016        0
       UMA Slabs 1    176       8      14           8       0      62       3872        0
        KMAP ENTRY     96      12      27          14       0       0       3744        0
       FFS1 dinode    128       0       0           0       0     126          0        0
           ada_ccb    272       0       0           0       0      30          0        0
             swblk    136       0       0           0       0      62          0        0
          swpctrie    144       0       0           0       0      62          0        0
   cdg_qdiffsample     16       0       0           0       0     254          0        0
   pf state scrubs     40       0       0           0       0     254          0        0
   pf frag entries     40       0       0           0       0     254          0        0
          pf frags    248       0       0           0       0      62          0        0
  pf table entries    160       0       0           0       0     254          0        0
pf table entry counters     64       0       0           0       0     254          0        0
   pf source nodes    152       0       0           0       0     254          0        0
     pf state keys     88       0       0           0       0     126          0        0
         pf states    352       0       0           0       0     254          0        0
           pf tags    104       0       0           0       0     126          0        0
          pf mtags    184       0       0           0       0      62          0        0
      tcp_rack_pcb   1024       0       0           0       0      16          0        0
      tcp_rack_map    128       0       0           0       0     126          0        0
tfo_ccache_entries     80       0       0           0       0     126          0        0
               tfo      4       0       0           0       0     254          0        0
          sackhole     32       0       0           0       0     254          0        0
               ipq     56       0       0           0       0     254          0        0
   sctp_asconf_ack     48       0       0           0       0     254          0        0
       sctp_asconf     40       0       0           0       0     254          0        0
sctp_stream_msg_out    112       0       0           0       0     254          0        0
        sctp_readq    152       0       0           0       0     254          0        0
        sctp_chunk    152       0       0           0       0     254          0        0
        sctp_raddr    736       0       0           0       0     254          0        0
         sctp_asoc   2256       0       0           0       0     254          0        0
           sctp_ep   1176       0       0           0       0     254          0        0
   tcp_log_id_node    120       0       0           0       0     126          0        0
 tcp_log_id_bucket    176       0       0           0       0      62          0        0
           tcp_log    416       0       0           0       0     254          0        0
          tcpreass     48       0       0           0       0     254          0        0
udplite_inpcb ports     32       0       0           0       0     254          0        0
     udplite_inpcb    424       0       0           0       0      30          0        0
       ripcb ports     32       0       0           0       0     254          0        0
    IPsec SA lft_c     16       0       0           0       0     254          0        0
           netlink   2048       0       0           0       0       8          0        0
            itimer    352       0       0           0       0      30          0        0
            AIOLIO    272       0       0           0       0      30          0        0
             AIOCB    552       0       0           0       0      16          0        0
               AIO    208       0       0           0       0      62          0        0
        mqnotifier    216       0       0           0       0      62          0        0
            mvdata     64       0       0           0       0     254          0        0
            mqueue    248       0       0           0       0      62          0        0
        TMPFS node    232       0       0           0       0      62          0        0
           NCLNODE    608       0       0           0       0      16          0        0
     LTS VFS Cache    360       0       0           0       0      30          0        0
       L VFS Cache    320       0       0           0       0      30          0        0
     STS VFS Cache    144       0       0           0       0      62          0        0
           cryptop    280       0       0           0       0      30          0        0
  linux_dma_object     32       0       0           0       0     254          0        0
  linux_dma_pctrie    144       0       0           0       0      62          0        0
   IOMMU_MAP_ENTRY    104       0       0           0       0     126          0        0
    mbuf_jumbo_16k  16384       0       0           0       0     254          0        0
     mbuf_jumbo_9k   9216       0       0           0       0     254          0        0
      audit_record   1280       0       0           0       0       8          0        0
         domainset     40       0       0           0       0     254          0        0
        MAC labels     40       0       0           0       0     254          0        0
            vnpbuf   2624       0       0           0       0      16          0        0
           nfspbuf   2624       0       0           0       0       4          0        0
            swwbuf   2624       0       0           0       0       2          0        0
            swrbuf   2624       0       0           0       0       4          0        0
          umtx_shm     88       0       0           0       0     126          0        0
           umtx pi     96       0       0           0       0     126          0        0
rangeset pctrie nodes    144       0       0           0       0      62          0        0
      malloc-65536  65536       0       0           0       0       1          0        0
      malloc-65536  65536       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-16384  16384       0       0           0       0       1          0        0
      malloc-16384  16384       0       0           0       0       1          0        0
       malloc-8192   8192       0       0           0       0       1          0        0
       malloc-4096   4096       0       0           0       0       2          0        0
        malloc-512    512       0       0           0       0      30          0        0
        malloc-512    512       0       0           0       0      30          0        0
           pcpu-32     32       0       0           0       0     254          0        0
            pcpu-4      4       0       0           0       0     254          0        0
            fakepg    104       0       0           0       0     126          0        0
          UMA Hash    256       0       0           0       0      62          0        0