login: panic: Lock pf config not exclusively locked @ /syzkaller/managers/main/kernel/sys/netpfil/pf/pf_ioctl.c:538

cpuid = 1
time = 1649862029
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0xc7/frame 0xfffffe0054170ef0
kdb_backtrace() at kdb_backtrace+0xd3/frame 0xfffffe0054171050
vpanic() at vpanic+0x2b8/frame 0xfffffe0054171130
panic() at panic+0xb5/frame 0xfffffe00541711f0
_sx_xunlock_hard() at _sx_xunlock_hard/frame 0xfffffe0054171240
pf_free_rule() at pf_free_rule+0x4c/frame 0xfffffe0054171270
pfioctl() at pfioctl+0xc4d8/frame 0xfffffe0054172610
devfs_ioctl() at devfs_ioctl+0x22a/frame 0xfffffe00541726f0
VOP_IOCTL_APV() at VOP_IOCTL_APV+0xb0/frame 0xfffffe0054172730
vn_ioctl() at vn_ioctl+0x215/frame 0xfffffe0054172a80
devfs_ioctl_f() at devfs_ioctl_f+0x71/frame 0xfffffe0054172ad0
kern_ioctl() at kern_ioctl+0x631/frame 0xfffffe0054172bd0
sys_ioctl() at sys_ioctl+0x31f/frame 0xfffffe0054172d30
amd64_syscall() at amd64_syscall+0x40c/frame 0xfffffe0054172f30
fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0054172f30
--- syscall (0, FreeBSD ELF64, nosys), rip = 0x2b3bfa, rsp = 0x82041e4c8, rbp = 0x82041e4e0 ---
KDB: enter: panic
[ thread pid 779 tid 100093 ]
Stopped at      kdb_enter+0x6b: movq    $0,0x2705a6a(%rip)
db> 
db> set $lines = 0
db> set $maxwidth = 0
db> show registers
cs                        0x20
ds                        0x3b
es                        0x3b
fs                        0x13
gs                        0x1b
ss                        0x28
rax                       0x12
rcx         0xd76bc781b8ab64e5
rdx         0xdffff7c000000000
rbx                          0
rsp         0xfffffe0054171030
rbp         0xfffffe0054171050
rsi                        0x1
rdi                          0
r8                         0x3
r9                  0xffffffff
r10                          0
r11         0xfffffe0058c1f190
r12         0xfffffe0058c1ec80
r13         0xfffffe0054171001
r14         0xffffffff82bc5540  .str.26
r15         0xffffffff82bc5540  .str.26
rip         0xffffffff8177492b  kdb_enter+0x6b
rflags                    0x46
kdb_enter+0x6b: movq    $0,0x2705a6a(%rip)
db> show proc
Process 779 (syz-executor0166091) at 0xfffffe0058c1c548:
 state: NORMAL
 uid: 0  gids: 0, 0, 5
 parent: pid 777 at 0xfffffe008fe9c000
 ABI: FreeBSD ELF64
 flag: 0x10004000  flag2: 0
 arguments: ./syz-executor016609179
 reaper: 0xfffffe0053ddc000 reapsubtree: 1
 sigparent: 20
 vmspace: 0xfffffe00929ee000
   (map 0xfffffe00929ee000)
   (map.pmap 0xfffffe00929ee0c0)
   (pmap 0xfffffe00929ee128)
 threads: 1
100093                   Run     CPU 1                       syz-executor0166091
db> ps
  pid  ppid  pgrp   uid  state   wmesg   wchan               cmd
  779   777   777     0  R       CPU 1                       syz-executor0166091
  777   775   777     0  Ss      pause   0xfffffe008fe9c0b0  csh
  775   688   775     0  Rs      CPU 0                       sshd
  754     1   754     0  Ss+     ttyin   0xfffffe0056feb4b0  getty
  753     1   753     0  Ss+     ttyin   0xfffffe0056fe6cb0  getty
  752     1   752     0  Ss+     ttyin   0xfffffe0056fe70b0  getty
  751     1   751     0  Ss+     ttyin   0xfffffe0056fe74b0  getty
  750     1   750     0  Ss+     ttyin   0xfffffe00081f18b0  getty
  749     1   749     0  Ss+     ttyin   0xfffffe0056fe78b0  getty
  748     1   748     0  Ss+     ttyin   0xfffffe0056fe7cb0  getty
  747     1   747     0  Ss+     ttyin   0xfffffe0056fe90b0  getty
  746     1   746     0  Ss+     ttyin   0xfffffe0056fe94b0  getty
  744     1    18     0  S+      piperd  0xfffffe0058b59ba0  logger
  743   742    18     0  S+      nanslp  0xffffffff83e47a00  sleep
  742     1    18     0  S+      wait    0xfffffe0092976548  sh
  692     1   692     0  Ss      nanslp  0xffffffff83e47a01  cron
  688     1   688     0  Ss      select  0xfffffe0053dd1c40  sshd
  501     1   501     0  Ss      select  0xfffffe0053dd18c0  syslogd
  430     1   430     0  Ss      select  0xfffffe0053dd1040  devd
  429     1   429    65  Ss      select  0xfffffe0053dd17c0  dhclient
  344     1   344     0  Ss      select  0xfffffe0053dd0ec0  dhclient
  341     1   341     0  Ss      select  0xfffffe0053dd15c0  dhclient
   17     0     0     0  DL      syncer  0xffffffff83f6d260  [syncer]
   16     0     0     0  DL      vlruwt  0xfffffe0053dda000  [vnlru]
   15     0     0     0  DL      (threaded)                  [bufdaemon]
100080                   D       psleep  0xffffffff83f6b860  [bufdaemon]
100083                   D       -       0xffffffff83211f80  [bufspacedaemon-0]
100094                   D       sdflush 0xfffffe0053f690e8  [/ worker]
    9     0     0     0  DL      psleep  0xffffffff83f9f340  [vmdaemon]
    8     0     0     0  DL      (threaded)                  [pagedaemon]
100078                   D       psleep  0xffffffff83f931f8  [dom0]
100081                   D       launds  0xffffffff83f93204  [laundry: dom0]
100082                   D       umarcl  0xffffffff81eae6b0  [uma]
    7     0     0     0  DL      -       0xffffffff83c03788  [rand_harvestq]
    6     0     0     0  DL      pftm    0xffffffff84a5a530  [pf purge]
    5     0     0     0  DL      waiting 0xffffffff848954a0  [sctp_iterator]
    4     0     0     0  DL      (threaded)                  [cam]
100045                   D       -       0xffffffff83aa56c0  [doneq0]
100046                   D       -       0xffffffff83aa5640  [async]
100077                   D       -       0xffffffff83aa54c0  [scanner]
   14     0     0     0  DL      seqstat 0xfffffe0056f1ac88  [sequencer 00]
    3     0     0     0  DL      (threaded)                  [crypto]
100041                   D       crypto_ 0xffffffff83f8ea00  [crypto]
100042                   D       crypto_ 0xfffffe0053effd30  [crypto returns 0]
100043                   D       crypto_ 0xfffffe0053effd80  [crypto returns 1]
   13     0     0     0  DL      (threaded)                  [geom]
100036                   D       -       0xffffffff83e1cfc0  [g_event]
100037                   D       -       0xffffffff83e1cfe0  [g_up]
100038                   D       -       0xffffffff83e1d000  [g_down]
    2     0     0     0  WL      (threaded)                  [clock]
100030                   I                                   [clock (0)]
100031                   I                                   [clock (1)]
   12     0     0     0  WL      (threaded)                  [intr]
100010                   I                                   [swi6: Giant taskq]
100017                   I                                   [swi5: fast taskq]
100020                   I                                   [swi6: task queue]
100029                   I                                   [swi1: netisr 0]
100032                   I                                   [swi3: busdma]
100033                   I                                   [swi1: hpts]
100034                   I                                   [swi1: hpts]
100047                   I                                   [irq24: virtio_pci0]
100048                   I                                   [irq25: virtio_pci0]
100049                   I                                   [irq26: virtio_pci0]
100050                   I                                   [irq27: virtio_pci0]
100051                   I                                   [irq28: virtio_pci1]
100052                   I                                   [irq29: virtio_pci1]
100053                   I                                   [irq30: virtio_pci1]
100054                   I                                   [irq31: virtio_pci1]
100055                   I                                   [irq32: virtio_pci1]
100060                   I                                   [irq33: virtio_pci2]
100061                   I                                   [irq34: virtio_pci2]
100062                   I                                   [irq35: virtio_pci2]
100064                   I                                   [irq1: atkbd0]
100065                   I                                   [irq12: psm0]
100066                   I                                   [swi0: uart uart++]
100070                   I                                   [swi1: pf send]
   11     0     0     0  RL      (threaded)                  [idle]
100003                   CanRun                              [idle: cpu0]
100004                   CanRun                              [idle: cpu1]
    1     0     1     0  SLs     wait    0xfffffe0053ddc000  [init]
   10     0     0     0  DL      audit_w 0xffffffff83f8f500  [audit]
    0     0     0     0  DLs     (threaded)                  [kernel]
100000                   D       swapin  0xffffffff83e1da00  [swapper]
100005                   D       -       0xfffffe0007978100  [softirq_0]
100006                   D       -       0xfffffe0007978000  [softirq_1]
100007                   D       -       0xfffffe0007977e00  [if_io_tqg_0]
100008                   D       -       0xfffffe0007977d00  [if_io_tqg_1]
100009                   D       -       0xfffffe0007977c00  [if_config_tqg_0]
100011                   D       -       0xfffffe0007977a00  [aiod_kick taskq]
100012                   D       -       0xfffffe0007977900  [inm_free taskq]
100013                   D       -       0xfffffe0007977800  [linuxkpi_irq_wq]
100014                   D       -       0xfffffe0007977700  [in6m_free taskq]
100015                   D       -       0xfffffe0007977600  [deferred_unmount ta]
100016                   D       -       0xfffffe0007977500  [thread taskq]
100018                   D       -       0xfffffe0007977300  [pci_hp taskq]
100019                   D       -       0xfffffe0007977200  [kqueue_ctx taskq]
100021                   D       -       0xfffffe0007977000  [linuxkpi_short_wq_0]
100022                   D       -       0xfffffe0007977000  [linuxkpi_short_wq_1]
100023                   D       -       0xfffffe0007977000  [linuxkpi_short_wq_2]
100024                   D       -       0xfffffe0007977000  [linuxkpi_short_wq_3]
100025                   D       -       0xfffffe0007976e00  [linuxkpi_long_wq_0]
100026                   D       -       0xfffffe0007976e00  [linuxkpi_long_wq_1]
100027                   D       -       0xfffffe0007976e00  [linuxkpi_long_wq_2]
100028                   D       -       0xfffffe0007976e00  [linuxkpi_long_wq_3]
100035                   D       -       0xfffffe0053f31300  [firmware taskq]
100039                   D       -       0xfffffe0053f31100  [crypto_0]
100040                   D       -       0xfffffe0053f31100  [crypto_1]
100056                   D       -       0xfffffe0053f2e000  [vtnet0 rxq 0]
100057                   D       -       0xfffffe0007979e00  [vtnet0 txq 0]
100058                   D       -       0xfffffe0007979d00  [vtnet0 rxq 1]
100059                   D       -       0xfffffe0007979c00  [vtnet0 txq 1]
100063                   D       vtbslp  0xfffffe0057011800  [virtio_balloon]
100067                   D       -       0xffffffff82bcb3c1  [deadlkres]
100071                   D       -       0xfffffe0007978200  [mca taskq]
100072                   D       -       0xfffffe00574c3200  [acpi_task_0]
100073                   D       -       0xfffffe00574c3200  [acpi_task_1]
100074                   D       -       0xfffffe00574c3200  [acpi_task_2]
100076                   D       -       0xfffffe0053f30e00  [CAM taskq]
db> show all locks
Process 779 (syz-executor0166091) thread 0xfffffe0058c1ec80 (100093)
exclusive rm pf rulesets (pf rulesets) r = 0 (0xffffffff84b155a0) locked @ /syzkaller/managers/main/kernel/sys/netpfil/pf/pf_ioctl.c:3399
Process 775 (sshd) thread 0xfffffe0058c1d720 (100096)
exclusive rw tcpinp (tcpinp) r = 0 (0xfffffe00928d0a00) locked @ /syzkaller/managers/main/kernel/sys/netinet/tcp_usrreq.c:1008
exclusive sx so_snd_sx (so_snd_sx) r = 0 (0xfffffe0058bfd500) locked @ /syzkaller/managers/main/kernel/sys/kern/uipc_socket.c:4193
db> show malloc
              Type        InUse        MemUse     Requests
           pf_hash            5        11524K            5
          tcp_hpts            6         4801K            6
            devbuf         4217         4323K         4242
         sysctloid        35306         2080K        35377
             vtbuf           24         1968K           46
              kobj          328         1312K          489
            newblk          706         1201K          718
          vfscache            3         1025K            3
               pcb           19          537K           38
          inodedep           49          530K           71
         ufs_quota            1          512K            1
          vfs_hash            1          512K            1
           callout            2          512K            2
              intr            4          472K            4
            acpica         1674          184K        57552
           subproc           89          179K          829
         vnet_data            1          168K            1
           tidhash            3          141K            3
              vmem            3          138K            4
            linker          358          134K          386
           pagedep           14          132K           18
        tfo_ccache            1          128K            1
               sem            4          106K            4
            DEVFS1          105          105K          114
               bus          994           81K         5207
          mtx_pool            2           72K            2
          syncache            1           68K            1
            module          513           65K          513
          acpitask            1           64K            1
       ddb_capture            1           64K            1
              temp           17           33K         1606
         hostcache            1           32K            1
               shm            1           32K            1
           kdtrace          157           32K          897
            DEVFS3          124           31K          134
              umtx          242           31K          242
               msg            4           30K            4
        gtaskqueue           18           26K           18
            kbdmux            6           22K            6
        DEVFS_RULE           56           20K           56
               BPF           10           18K           10
         ufs_mount            4           17K            5
              proc            3           17K            3
               tty           16           16K           16
           ithread          100           16K          100
            bus-sc           34           15K         1681
            KTRACE          100           13K          100
              kenv           95           12K           95
      eventhandler          134           12K          134
            ifaddr           30           12K           32
              rman           88           11K          431
              GEOM           61           11K          490
          routetbl           50           11K          176
         CAM queue            5           11K         1528
              cred           36            9K          234
         bmsafemap            4            9K           39
              ksem            2            9K            2
              UART           12            9K           12
           devstat            4            9K            4
               rpc            2            8K            2
             shmfd            1            8K            1
       pfs_vncache            1            8K            1
         pfs_nodes           20            8K           20
     audit_evclass          237            8K          296
         taskqueue           63            7K           63
            sglist            5            7K            5
           CAM DEV            3            6K          510
       ufs_dirhash           24            5K           24
               UMA          272            5K          272
            dirrem           17            5K           28
            plimit           17            5K          322
                vt           11            5K           11
             ifnet            3            5K            3
           memdesc            1            4K            1
               MCA           32            4K           32
          ioctlops            1            4K           87
          filedesc            1            4K            1
             evdev            4            4K            4
           acpisem           28            4K           28
             hhook           15            4K           17
       ether_multi           40            4K           50
            diradd           25            4K           36
           lltable           11            4K           11
          pf_ifnet            5            3K            6
         in6_multi           25            3K           25
          terminal           11            3K           11
           session           20            3K           31
            kqueue           40            3K          782
           pwddesc           40            3K          780
           uidinfo            3            3K            8
         proc-args           62            3K         1721
           pf_rule            1            2K            1
        local_apic            1            2K            1
           io_apic            1            2K            1
       fpukern_ctx            2            2K            2
         ipsec-saq            2            2K            2
            Unitno           27            2K           39
           CAM XPT           22            2K          543
             lockf           15            2K           22
               msi           12            2K           12
       ipsecpolicy            2            2K            2
           acpidev           20            2K           20
             selfd           20            2K        11222
             clone            9            2K            9
           softdep            1            1K            1
            sahead            1            1K            1
          secasvar            1            1K            1
       vnodemarker            2            1K            8
      NFSD session            1            1K            1
        CAM periph            4            1K          271
            select            7            1K           29
             ipsec            3            1K            3
          indirdep            3            1K            3
             nhops            6            1K            6
         toponodes            6            1K            6
            isadev            6            1K            6
             mount           16            1K           89
          pci_link           10            1K           10
          sctp_ifa            5            1K            6
            crypto            4            1K            4
            ip6ndp            4            1K            5
 encap_export_host           12            1K           12
         newdirblk            4            1K            8
             mkdir            4            1K           16
          in_multi            2            1K            4
              pfil            4            1K            4
              cdev            2            1K            2
    chacha20random            1            1K            1
               osd            7            1K           18
       inpcbpolicy           10            1K          139
          sctp_ifn            2            1K            6
      NFSD lckfile            1            1K            1
     NFSD V4client            1            1K            1
            DEVFSP            4            1K            9
             DEVFS            9            1K           10
          freework            1            1K           26
               mld            2            1K            2
              igmp            2            1K            2
            vnodes            1            1K            1
           CAM SIM            2            1K            2
            feeder            7            1K            7
           tcpfunc            3            1K            3
            CC Mem            3            1K            7
        loginclass            3            1K            7
            prison            6            1K            6
       lkpikmalloc            5            1K            6
        aesni_data            2            1K            2
         cryptodev            2            1K           49
          nexusdev            8            1K            8
            apmdev            1            1K            1
          atkbddev            2            1K            2
     CAM dev queue            2            1K            2
 CAM I/O Scheduler            1            1K            1
          CAM path            4            1K         1034
          procdesc            1            1K            6
          pmchooks            1            1K            1
            soname            4            1K         3471
          filecaps            4            1K           66
               tun            3            1K            3
          sctp_vrf            1            1K            1
              vnet            1            1K            1
           entropy            2            1K           35
               pmc            1            1K            1
          acpiintr            1            1K            1
              cpus            2            1K            2
    vnet_data_free            1            1K            1
           Per-cpu            1            1K            1
          p1003.1b            1            1K            1
            tcp_do            0            0K            0
           tcp_fsb            0            0K            0
          pf_table            0            0K            0
           pf_altq            0            0K            0
           pf_osfp            0            0K            0
           pf_temp            0            0K            0
        sctp_mcore            0            0K            0
        sctp_socko            0            0K            0
         sctp_iter            0            0K            4
         sctp_mvrf            0            0K            0
         sctp_timw            0            0K            0
         sctp_cpal            0            0K            0
         sctp_cmsg            0            0K            0
         sctp_stre            0            0K            0
         sctp_athi            0            0K            0
         sctp_athm            0            0K            0
         sctp_atky            0            0K            0
         sctp_atcl            0            0K            0
         sctp_a_it            0            0K            4
         sctp_aadr            0            0K            0
         sctp_stro            0            0K            0
         sctp_stri            0            0K            0
          sctp_map            0            0K            0
            mqdata            0            0K            0
            ipcomp            0            0K            0
               esp            0            0K            0
                ah            0            0K            0
           filemon            0            0K            0
     NFSCL sockreq            0            0K            0
     NFSCL devinfo            0            0K            0
        madt_table            0            0K            2
          smartpqi            0            0K            0
     NFSCL flayout            0            0K            0
      NFSCL layout            0            0K            0
     NFSD rollback            0            0K            0
               ixl            0            0K            0
      NFSCL diroff            0            0K            0
       NEWdirectio            0            0K            0
        NEWNFSnode            0            0K            0
         NFSCL lck            0            0K            0
      NFSCL lckown            0            0K            0
      NFSCL client            0            0K            0
       NFSCL deleg            0            0K            0
        ice-resmgr            0            0K            0
         ice-osdep            0            0K            0
               ice            0            0K            0
              iavf            0            0K            0
             axgbe            0            0K            0
        NFSCL open            0            0K            0
       NFSCL owner            0            0K            0
            NFS fh            0            0K            0
           NFS req            0            0K            0
     NFSD usrgroup            0            0K            0
       NFSD string            0            0K            0
       NFSD V4lock            0            0K            0
      NFSD V4state            0            0K            0
          xen_intr            0            0K            0
     NFSD srvcache            0            0K            0
       msdosfs_fat            0            0K            0
           xen_hvm            0            0K            0
         legacydrv            0            0K            0
            bounce            0            0K            0
            busdma            0            0K            0
            qpidrv            0            0K            0
     msdosfs_mount            0            0K            0
      msdosfs_node            0            0K            0
      dmar_idpgtbl            0            0K            0
          dmar_dom            0            0K            0
          dmar_ctx            0            0K            0
              isci            0            0K            0
      iommu_dmamap            0            0K            0
            DEVFS4            0            0K            0
     hyperv_socket            0            0K            0
           bxe_ilt            0            0K            0
            xenbus            0            0K            0
            DEVFS2            0            0K            0
            gntdev            0            0K            0
     vm_fictitious            0            0K            0
       privcmd_dev            0            0K            0
        evtchn_dev            0            0K            0
          xenstore            0            0K            0
         scsi_pass            0            0K            0
         ciss_data            0            0K            0
               xnb            0            0K            0
          xen_acpi            0            0K            0
              xbbd            0            0K            0
               xbd            0            0K            0
           Balloon            0            0K            0
          sysmouse            0            0K            0
           UMAHash            0            0K            0
            vtfont            0            0K            0
         vm_pgdata            0            0K            0
           jblocks            0            0K            0
          savedino            0            0K           15
          sentinel            0            0K            0
            jfsync            0            0K            0
            jtrunc            0            0K            0
             sbdep            0            0K            2
           jsegdep            0            0K            0
              jseg            0            0K            0
         jfreefrag            0            0K            0
          jfreeblk            0            0K            0
           jnewblk            0            0K            0
            jmvref            0            0K            0
           jremref            0            0K            0
           jaddref            0            0K            0
           freedep            0            0K            0
          freefile            0            0K            9
          freeblks            0            0K           25
          freefrag            0            0K            1
        allocindir            0            0K            0
       allocdirect            0            0K            0
          ufs_trim            0            0K            0
           mactemp            0            0K            0
     audit_trigger            0            0K            0
 audit_pipe_presel            0            0K            0
     audit_pipeent            0            0K            0
        audit_pipe            0            0K            0
      audit_evname            0            0K            0
         audit_bsm            0            0K            0
      audit_gidset            0            0K            0
        audit_text            0            0K            0
        audit_path            0            0K            0
        audit_data            0            0K            0
        audit_cred            0            0K            0
         BACKLIGHT            0            0K            0
           ath_hal            0            0K            0
            athdev            0            0K            0
           ata_pci            0            0K            0
           ata_dma            0            0K            0
       ata_generic            0            0K            0
            pvscsi            0            0K            0
           scsi_da            0            0K           69
            ata_da            0            0K            0
           scsi_ch            0            0K            0
           scsi_cd            0            0K            0
          ktls_ocf            0            0K            0
       AHCI driver            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
      MLX5E_TLS_RX            0            0K            0
        MLX5EEPROM            0            0K            0
         MLX5E_TLS            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
            MLX5EN            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
          MLX5DUMP            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
          seq_file            0            0K            0
           lkpiskb            0            0K            0
             radix            0            0K            0
               idr            0            0K            0
          lkpindev            0            0K            0
            lkpifw            0            0K            0
         lkpi80211            0            0K            0
               NLM            0            0K            0
    ipsec-spdcache            0            0K            0
         ipsec-reg            0            0K            0
        ipsec-misc            0            0K            0
      ipsecrequest            0            0K            0
            ip6opt            0            0K            3
       ip6_msource            0            0K            0
      ip6_moptions            0            0K            0
       in6_mfilter            0            0K            0
             frag6            0            0K            0
            tcplog            0            0K            0
        tcp_hwpace            0            0K            0
            USBdev            0            0K            0
               USB            0            0K            0
               LRO            0            0K            0
        ip_msource            0            0K            0
       ip_moptions            0            0K            0
        in_mfilter            0            0K            0
              ipid            0            0K            0
         80211scan            0            0K            0
      80211ratectl            0            0K            0
        80211power            0            0K            0
       80211nodeie            0            0K            0
         80211node            0            0K            0
      80211mesh_gt            0            0K            0
      80211mesh_rt            0            0K            0
         80211perr            0            0K            0
         80211prep            0            0K            0
         80211preq            0            0K            0
          80211dfs            0            0K            0
       80211crypto            0            0K            0
          80211vap            0            0K            0
             iflib            0            0K            0
              vlan            0            0K            0
               gif            0            0K            0
           ifdescr            0            0K            0
              zlib            0            0K            0
           fadvise            0            0K            0
           VN POLL            0            0K            0
               agp            0            0K            0
            statfs            0            0K          195
     namei_tracker            0            0K            0
       export_host            0            0K            0
        cl_savebuf            0            0K            3
           nvme_da            0            0K            0
           acpipwr            0            0K            0
         acpi_perf            0            0K            0
            twsbuf            0            0K            0
      twe_commands            0            0K            0
       tcp_log_dev            0            0K            0
      midi buffers            0            0K            0
             mixer            0            0K            0
              ac97            0            0K            0
             hdacc            0            0K            0
              hdac            0            0K            0
              hdaa            0            0K            0
         acpicmbat            0            0K            0
       SIIS driver            0            0K            0
           CAM CCB            0            0K          523
            biobuf            0            0K            0
              aios            0            0K            0
               lio            0            0K            0
               acl            0            0K            0
          mbuf_tag            0            0K           27
              ktls            0            0K            0
               PUC            0            0K            0
          ppbusdev            0            0K            0
agtiapi_MemAlloc malloc            0            0K            0
    osti_cacheable            0            0K            0
          tempbuff            0            0K            0
          tempbuff            0            0K            0
ag_tgt_map_t malloc            0            0K            0
ag_slr_map_t malloc            0            0K            0
lDevFlags * malloc            0            0K            0
tiDeviceHandle_t * malloc            0            0K            0
ag_portal_data_t malloc            0            0K            0
ag_device_t malloc            0            0K            0
     STLock malloc            0            0K            0
          CCB List            0            0K            0
            sr_iov            0            0K            0
               OCS            0            0K            0
               OCS            0            0K            0
              nvme            0            0K            0
               nvd            0            0K            0
            netmap            0            0K            0
            mwldev            0            0K            0
        MVS driver            0            0K            0
     CAM ccb queue            0            0K            0
              accf            0            0K            0
               pts            0            0K            0
               iov            0            0K        13623
           eventfd            0            0K            0
           Witness            0            0K            0
             stack            0            0K            0
          mrsasbuf            0            0K            0
          mpt_user            0            0K            0
          mps_user            0            0K            0
            MPSSAS            0            0K            0
               mps            0            0K            0
              sbuf            0            0K          288
          mpr_user            0            0K            0
          firmware            0            0K            0
        compressor            0            0K            0
            MPRSAS            0            0K            0
              SWAP            0            0K            0
               mpr            0            0K            0
            mfibuf            0            0K            0
         sysctltmp            0            0K          658
            sysctl            0            0K            3
        md_sectors            0            0K            0
              ekcd            0            0K            0
            dumper            0            0K            0
          sendfile            0            0K            0
              rctl            0            0K            0
           md_disk            0            0K            0
           malodev            0            0K            0
               LED            0            0K            0
          ix_sriov            0            0K            0
             cache            0            0K            0
        aacraidcam            0            0K            0
          kcovinfo            0            0K            0
      prison_racct            0            0K            0
       Fail Points            0            0K            0
             sigio            0            0K            1
filedesc_to_leader            0            0K            0
               pwd            0            0K            0
       tty console            0            0K            0
                ix            0            0K            0
            ipsbuf            0            0K            0
       aacraid_buf            0            0K            0
            aaccam            0            0K            0
         boottrace            0            0K            0
            aacbuf            0            0K            0
              zstd            0            0K            0
            XZ_DEC            0            0K            0
            nvlist            0            0K            0
          SCSI ENC            0            0K            0
           SCSI sa            0            0K            0
        isofs_node            0            0K            0
       isofs_mount            0            0K            0
     tr_raid5_data            0            0K            0
    tr_raid1e_data            0            0K            0
     tr_raid1_data            0            0K            0
     tr_raid0_data            0            0K            0
    tr_concat_data            0            0K            0
       md_sii_data            0            0K            0
   md_promise_data            0            0K            0
    md_nvidia_data            0            0K            0
   md_jmicron_data            0            0K            0
     md_intel_data            0            0K            0
       md_ddf_data            0            0K            0
         raid_data            0            0K           72
     geom_flashmap            0            0K            0
         tmpfs dir            0            0K            0
        tmpfs name            0            0K            0
       tmpfs mount            0            0K            0
           NFS FHA            0            0K            0
         newnfsmnt            0            0K            0
  newnfsclient_req            0            0K            0
   NFSCL layrecall            0            0K            0
     NFSCL session            0            0K            0
db> show uma
              Zone   Size    Used    Free    Requests  Sleeps  Bucket  Total Mem    XFree
   mbuf_jumbo_page   4096    8320    1078       14737       0     254   38494208        0
              mbuf    256    8579    1083       17463       0     254    2473472        0
              pbuf   2624       0     794           0       0       2    2083456        0
          BUF TRIE    144     184   11604         502       0      62    1697472        0
        malloc-384    384    4165       5        4165       0      30    1601280        0
       malloc-4096   4096     369       3        1355       0       2    1523712        0
        malloc-128    128   11630     243       11636       0     126    1519744        0
       UMA Slabs 0    112   10559      19       10559       0     126    1184736        0
      mbuf_cluster   2048     508       0         508       0     254    1040384        0
         vmem btag     56   16251      60       16251       0     254     913416        0
         FFS inode   1160     499      26         508       0       8     609000        0
             tcpcb   1104       3     508           7       0     254     564144        0
        RADIX NODE    144    3298     197       19962       0      62     503280        0
            socket    960      18     490        1335       0     254     487680        0
         VM OBJECT    264    1410     120       24436       0      30     403920        0
          lkpicurr    168       2    2350           2       0      62     395136        0
            lkpimm    168       1    2327           1       0      62     391104        0
      malloc-65536  65536       5       0           5       0       1     327680        0
        256 Bucket   2048     122      22         891       0       8     294912        0
         malloc-64     64    3865     482        3867       0     254     278208        0
             VNODE    448     530      46         541       0      30     258048        0
         malloc-16     16   14605     145       14665       0     254     236000        0
            DEVCTL   1024       0     220         126       0       0     225280        0
        malloc-256    256     746     124         851       0      62     222720        0
            THREAD   1808     116       5         116       0       8     218768        0
      malloc-65536  65536       3       0           3       0       1     196608        0
        malloc-128    128    1325     194       27485       0     126     194432        0
         MAP ENTRY     96    1559     457       84320       0     126     193536        0
         UMA Zones    768     244       0         244       0      16     187392        0
         malloc-32     32    5317     353        5326       0     254     181440        0
       FFS2 dinode    256     499      71         508       0      62     145920        0
      malloc-65536  65536       0       2          46       0       1     131072        0
      malloc-65536  65536       0       2         144       0       1     131072        0
       malloc-1024   1024     116      12         282       0      16     131072        0
             unpcb    256       7     503        1179       0     254     130560        0
       mbuf_packet    256       0     508          77       0     254     130048        0
       S VFS Cache    104     966     204        1004       0     126     121680        0
     FPU_save_area    832     118      26         130       0      16     119808        0
          ksiginfo    112      38    1006          53       0     126     116928        0
        malloc-128    128     564     211        3870       0     126      99200        0
        malloc-128    128     626     149        1164       0     126      99200        0
        malloc-256    256     293      82         918       0      62      96000        0
       malloc-8192   8192       7       4         110       0       1      90112        0
          UMA Kegs    384     230       3         230       0      30      89472        0
        128 Bucket   1024      46      37         484       0      16      84992        0
           VMSPACE   2552      23      10         764       0       4      84216        0
      malloc-16384  16384       1       4         161       0       1      81920        0
       malloc-2048   2048       9      31        1282       0       9      81920        0
       malloc-8192   8192       9       0           9       0       1      73728        0
             g_bio    408       0     180        4463       0      30      73440        0
         malloc-64     64     763     308       13477       0     254      68544        0
         malloc-64     64     567     504        2417       0     254      68544        0
      malloc-65536  65536       1       0           1       0       1      65536        0
      malloc-32768  32768       0       2         120       0       1      65536        0
      malloc-32768  32768       2       0           2       0       1      65536        0
      malloc-16384  16384       4       0           4       0       1      65536        0
       malloc-4096   4096      15       1          26       0       2      65536        0
       malloc-1024   1024      21      43         546       0      16      65536        0
              PROC   1352      39       9         779       0       8      64896        0
         filedesc0   1072      40      16         780       0       8      60032        0
        malloc-256    256     121      74         776       0      62      49920        0
         32 Bucket    256      65     130       13970       0      62      49920        0
           DIRHASH   1024      34      14          34       0      16      49152        0
             NAMEI   1024       0      48       12228       0      16      49152        0
       malloc-4096   4096      10       2         553       0       2      49152        0
       malloc-1024   1024      19      29         887       0      16      49152        0
          syncache    168       0     264           5       0     254      44352        0
            clpbuf   2624       0      16          22       0      16      41984        0
         udp_inpcb    424       6      84         128       0      30      38160        0
            pcpu-8      8    4229     379        4257       0     254      36864        0
         64 Bucket    512      65       7        1729       0      30      36864        0
         malloc-64     64      32     535       13601       0     254      36288        0
         malloc-64     64      26     541          42       0     254      36288        0
         malloc-64     64     116     451         141       0     254      36288        0
         malloc-64     64      10     557          42       0     254      36288        0
         malloc-64     64      48     519         788       0     254      36288        0
        malloc-128    128      28     251         157       0     126      35712        0
        malloc-128    128       1     278          11       0     126      35712        0
        malloc-128    128      95     184         491       0     126      35712        0
        malloc-128    128      20     259          25       0     126      35712        0
     routing nhops    256      10     125          17       0      62      34560        0
           ttyoutq    256      72      63         160       0      62      34560        0
        malloc-384    384      51      39         391       0      30      34560        0
        malloc-384    384      48      42          89       0      30      34560        0
        malloc-256    256      23     112         441       0      62      34560        0
        malloc-256    256       2     133          52       0      62      34560        0
        malloc-256    256       6     129         288       0      62      34560        0
        malloc-256    256      70      65         475       0      62      34560        0
        malloc-256    256      35     100          40       0      62      34560        0
      malloc-32768  32768       1       0           1       0       1      32768        0
      malloc-16384  16384       2       0          17       0       1      32768        0
       malloc-8192   8192       4       0           4       0       1      32768        0
       malloc-2048   2048       0      16          12       0       8      32768        0
       malloc-2048   2048      11       5          12       0       8      32768        0
       malloc-2048   2048       1      15          25       0       8      32768        0
       malloc-2048   2048       2      14          11       0       8      32768        0
       malloc-1024   1024       2      30          45       0      16      32768        0
       malloc-1024   1024       2      30           6       0      16      32768        0
       malloc-1024   1024       3      29           7       0      16      32768        0
        malloc-512    512       0      64         118       0      30      32768        0
        malloc-512    512       2      62           2       0      30      32768        0
        malloc-512    512       3      61           9       0      30      32768        0
        malloc-512    512       0      64           9       0      30      32768        0
        malloc-512    512      10      54          60       0      30      32768        0
        malloc-512    512       3      61           3       0      30      32768        0
           pcpu-64     64     493      19         493       0     254      32768        0
            ttyinq    160     135      65         300       0      62      32000        0
            cpuset    104       7     272           7       0     126      29016        0
        sctp_laddr     48       0     588           4       0     254      28224        0
         malloc-32     32     376     506        4475       0     254      28224        0
         16 Bucket    144      46     150         237       0      62      28224        0
          4 Bucket     48       6     582          53       0     254      28224        0
         tcp_inpcb    424       3      60           7       0      30      26712        0
             ripcb    424       1      62           4       0      30      26712        0
            da_ccb    544       0      49        1237       0      16      26656        0
              pipe    744       7      28         284       0      16      26040        0
         TURNSTILE    136     122      67         122       0      62      25704        0
       malloc-8192   8192       3       0           3       0       1      24576        0
       malloc-4096   4096       6       0           6       0       2      24576        0
           rtentry    176      13     125          17       0      62      24288        0
              PGRP     88      20     256          31       0     126      24288        0
    ertt_txseginfo     40       1     605         248       0     254      24240        0
          rl_entry     40      31     575          31       0     254      24240        0
             Files     80      73     227        6582       0     126      24000        0
          8 Bucket     80      35     265         448       0     126      24000        0
        malloc-384    384      11      49          11       0      30      23040        0
        malloc-384    384      10      50          13       0      30      23040        0
        malloc-384    384      54       6          64       0      30      23040        0
        SLEEPQUEUE     88     122     134         122       0     126      22528        0
         hostcache     64       1     314           1       0     254      20160        0
             udpcb     32       6     624         128       0     254      20160        0
   udp_inpcb ports     32       3     627          40       0     254      20160        0
              ertt     72       3     277           7       0     126      20160        0
               PWD     32      10     620         100       0     254      20160        0
         malloc-32     32      34     596         196       0     254      20160        0
         malloc-32     32       7     623          60       0     254      20160        0
         malloc-32     32     132     498        1386       0     254      20160        0
         malloc-32     32      31     599          56       0     254      20160        0
         malloc-32     32      29     601          31       0     254      20160        0
         malloc-32     32       7     623          10       0     254      20160        0
          2 Bucket     32      44     586         295       0     254      20160        0
             KNOTE    160       0     125           8       0      62      20000        0
       Mountpoints   2752       2       5           2       0       4      19264        0
 epoch_record pcpu    256       4      60           4       0      62      16384        0
       malloc-8192   8192       0       2          28       0       1      16384        0
       malloc-8192   8192       2       0           2       0       1      16384        0
       malloc-4096   4096       1       3         197       0       2      16384        0
       malloc-4096   4096       1       3           2       0       2      16384        0
       malloc-2048   2048       3       5           3       0       8      16384        0
       malloc-2048   2048       2       6           2       0       8      16384        0
       malloc-2048   2048       4       4           4       0       8      16384        0
       malloc-1024   1024       4      12           5       0      16      16384        0
       malloc-1024   1024       1      15           1       0      16      16384        0
        malloc-512    512       0      32           1       0      30      16384        0
           SMR CPU     32       7     504           7       0     254      16352        0
         malloc-16     16     511     489        3506       0     254      16000        0
              kenv    258      15      45        1037       0      30      15480        0
            mqnode    416       3      33           3       0      30      14976        0
              vmem   1856       1       7           1       0       8      14848        0
        SMR SHARED     24       7     504           7       0     254      12264        0
   tcp_inpcb ports     32       1     377           1       0     254      12096        0
         malloc-16     16      46     704        1356       0     254      12000        0
         malloc-16     16       9     741          14       0     254      12000        0
         malloc-16     16      15     735          64       0     254      12000        0
         malloc-16     16      49     701       26492       0     254      12000        0
         malloc-16     16       6     744           8       0     254      12000        0
        malloc-384    384       0      30           1       0      30      11520        0
       malloc-8192   8192       1       0           1       0       1       8192        0
       malloc-4096   4096       0       2           2       0       2       8192        0
      vtnet_tx_hdr     24       1     333        1811       0     254       8016        0
         malloc-16     16       0     500           2       0     254       8000        0
           pcpu-16     16       7     249           7       0     254       4096        0
       UMA Slabs 1    176       8      14           8       0      62       3872        0
        KMAP ENTRY     96      12      27          14       0       0       3744        0
       FFS1 dinode    128       0       0           0       0     126          0        0
           ada_ccb    272       0       0           0       0      30          0        0
             swblk    136       0       0           0       0      62          0        0
          swpctrie    144       0       0           0       0      62          0        0
   cdg_qdiffsample     16       0       0           0       0     254          0        0
   pf state scrubs     40       0       0           0       0     254          0        0
   pf frag entries     40       0       0           0       0     254          0        0
          pf frags    248       0       0           0       0      62          0        0
  pf table entries    160       0       0           0       0     254          0        0
pf table entry counters     64       0       0           0       0     254          0        0
   pf source nodes    136       0       0           0       0     254          0        0
     pf state keys     88       0       0           0       0     126          0        0
         pf states    312       0       0           0       0     254          0        0
           pf tags    104       0       0           0       0     126          0        0
          pf mtags     56       0       0           0       0     254          0        0
      tcp_rack_pcb    896       0       0           0       0      16          0        0
      tcp_rack_map    120       0       0           0       0     126          0        0
       tcp_bbr_pcb    832       0       0           0       0      16          0        0
       tcp_bbr_map    128       0       0           0       0     126          0        0
tfo_ccache_entries     80       0       0           0       0     126          0        0
               tfo      4       0       0           0       0     254          0        0
          sackhole     32       0       0           0       0     254          0        0
             tcptw     72       0       0           0       0     254          0        0
               ipq     56       0       0           0       0     254          0        0
      tcp_log_node    120       0       0           0       0     126          0        0
    tcp_log_bucket    176       0       0           0       0      62          0        0
           tcp_log    416       0       0           0       0     254          0        0
          tcpreass     48       0       0           0       0     254          0        0
   sctp_asconf_ack     48       0       0           0       0     254          0        0
       sctp_asconf     40       0       0           0       0     254          0        0
sctp_stream_msg_out    112       0       0           0       0     254          0        0
        sctp_readq    152       0       0           0       0     254          0        0
        sctp_chunk    152       0       0           0       0     254          0        0
        sctp_raddr    736       0       0           0       0     254          0        0
         sctp_asoc   2256       0       0           0       0     254          0        0
           sctp_ep   1208       0       0           0       0     254          0        0
       ripcb ports     32       0       0           0       0     254          0        0
udplite_inpcb ports     32       0       0           0       0     254          0        0
     udplite_inpcb    424       0       0           0       0      30          0        0
    IPsec SA lft_c     16       0       0           0       0     254          0        0
            itimer    352       0       0           0       0      30          0        0
            AIOLIO    272       0       0           0       0      30          0        0
             AIOCB    552       0       0           0       0      16          0        0
              AIOP     32       0       0           0       0     254          0        0
               AIO    208       0       0           0       0      62          0        0
           NCLNODE    608       0       0           0       0      16          0        0
        mqnotifier    216       0       0           0       0      62          0        0
            mvdata     64       0       0           0       0     254          0        0
            mqueue    248       0       0           0       0      62          0        0
        TMPFS node    224       0       0           0       0      62          0        0
     LTS VFS Cache    360       0       0           0       0      30          0        0
       L VFS Cache    320       0       0           0       0      30          0        0
     STS VFS Cache    144       0       0           0       0      62          0        0
           cryptop    280       0       0           0       0      30          0        0
  linux_dma_object     32       0       0           0       0     254          0        0
  linux_dma_pctrie    144       0       0           0       0      62          0        0
   IOMMU_MAP_ENTRY    120       0       0           0       0     126          0        0
    mbuf_jumbo_16k  16384       0       0           0       0     254          0        0
     mbuf_jumbo_9k   9216       0       0           0       0     254          0        0
      audit_record   1280       0       0           0       0       8          0        0
         domainset     40       0       0           0       0     254          0        0
        MAC labels     40       0       0           0       0     254          0        0
            vnpbuf   2624       0       0           0       0      64          0        0
            mdpbuf   2624       0       0           0       0       3          0        0
           nfspbuf   2624       0       0           0       0      16          0        0
            swwbuf   2624       0       0           0       0       8          0        0
            swrbuf   2624       0       0           0       0      16          0        0
          umtx_shm     88       0       0           0       0     126          0        0
           umtx pi     96       0       0           0       0     126          0        0
rangeset pctrie nodes    144       0       0           0       0      62          0        0
      malloc-65536  65536       0       0           0       0       1          0        0
      malloc-65536  65536       0       0           0       0       1          0        0
      malloc-65536  65536       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-16384  16384       0       0           0       0       1          0        0
      malloc-16384  16384       0       0           0       0       1          0        0
      malloc-16384  16384       0       0           0       0       1          0        0
      malloc-16384  16384       0       0           0       0       1          0        0
      malloc-16384  16384       0       0           0       0       1          0        0
       malloc-8192   8192       0       0           0       0       1          0        0
       malloc-4096   4096       0       0           0       0       2          0        0
        malloc-512    512       0       0           0       0      30          0        0
        malloc-384    384       0       0           0       0      30          0        0
           pcpu-32     32       0       0           0       0     254          0        0
            pcpu-4      4       0       0           0       0     254          0        0
            fakepg    104       0       0           0       0     126          0        0
          UMA Hash    256       0       0           0       0      62          0        0