panic: mutex so_rcv not owned at /syzkaller/managers/main/kernel/sys/kern/uipc_usrreq.c:1750 cpuid = 1 time = 1746928007 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0xc6/frame 0xfffffe005773b6d0 kdb_backtrace() at kdb_backtrace+0xd0/frame 0xfffffe005773b830 vpanic() at vpanic+0x257/frame 0xfffffe005773b9f0 panic() at panic+0xb5/frame 0xfffffe005773bab0 __mtx_assert() at __mtx_assert+0x224/frame 0xfffffe005773baf0 knote() at knote+0xaa/frame 0xfffffe005773bb60 sowwakeup_locked() at sowwakeup_locked+0x213/frame 0xfffffe005773bba0 socantsendmore() at socantsendmore+0x61/frame 0xfffffe005773bbc0 uipc_shutdown() at uipc_shutdown+0x1b5/frame 0xfffffe005773bc10 soshutdown() at soshutdown+0xab/frame 0xfffffe005773bc50 kern_shutdown() at kern_shutdown+0x15b/frame 0xfffffe005773bd10 amd64_syscall() at amd64_syscall+0x4af/frame 0xfffffe005773bf30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe005773bf30 --- syscall (198, FreeBSD ELF64, __syscall), rip = 0x3a197a, rsp = 0x822cf2f08, rbp = 0x822cf2f80 --- KDB: enter: panic [ thread pid 960 tid 100308 ] Stopped at kdb_enter+0x6e: movq $0,0x25bda97(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0x28 rax 0x12 rcx 0xfffffe006ec00000 rdx 0x7ffff rbx 0xffffffff827aff60 .str.27 rsp 0xfffffe005773b810 rbp 0xfffffe005773b830 rsi 0x80001 rdi 0xffffffff81614589 printf+0x149 r8 0 r9 0xffffffff r10 0xc6 r11 0xfffffe0054947c60 r12 0xfffffe0054947740 r13 0xfffffffffffffffd r14 0xffffffff827aff60 .str.27 r15 0 rip 0xffffffff815fe6fe kdb_enter+0x6e rflags 0x46 kdb_enter+0x6e: movq $0,0x25bda97(%rip) db> show proc Process 960 (syz-executor) at 0xfffffe00549385a0: state: NORMAL uid: 0 gids: 0, 0, 5 parent: pid 764 at 0xfffffe00548ee020 ABI: FreeBSD ELF64 flag: 0x10100080 flag2: 0x1 arguments: ./syz-executor exec reaper: 0xfffffe0008009040 reapsubtree: 1 sigparent: 20 vmspace: 0xfffffe0054943490 (map 0xfffffe0054943490) (map.pmap 0xfffffe0054943530) (pmap 0xfffffe00549435a0) threads: 2 100291 Run CPU 0 syz-executor 100308 Run CPU 1 syz-executor db> ps pid ppid pgrp uid state wmesg wchan cmd 960 764 764 0 R (threaded) syz-executor 100291 Run CPU 0 syz-executor 100308 Run CPU 1 syz-executor 959 766 766 0 S (threaded) syz-executor 100172 S nanslp 0xffffffff83b9d501 syz-executor 100305 S msgrcv 0xfffffe0058d582d0 syz-executor 100307 S uwait 0xfffffe0058957c00 syz-executor 958 767 767 0 S (threaded) syz-executor 100216 S nanslp 0xffffffff83b9d501 syz-executor 100304 S kqread 0xfffffe006e732100 syz-executor 100306 S uwait 0xfffffe006eb66b80 syz-executor 957 1 766 0 S uwait 0xfffffe0058d07880 syz-executor 954 765 765 0 S (threaded) syz-executor 100221 S nanslp 0xffffffff83b9d501 syz-executor 100300 S sbwait 0xfffffe005a0d920c syz-executor 950 1 766 60929 S uwait 0xfffffe0007f7eb00 syz-executor 945 1 937 0 S uwait 0xfffffe0058d07b80 syz-executor 941 1 764 0 S uwait 0xfffffe006eb68c80 syz-executor 934 1 934 0 S uwait 0xfffffe0058d08000 syz-executor 927 1 766 0 S uwait 0xfffffe0058d07a80 syz-executor 925 1 766 0 S uwait 0xfffffe006eb67780 syz-executor 924 1 766 0 S uwait 0xfffffe006eb66c00 syz-executor 915 914 767 0 SV uwait 0xfffffe006eb66200 syz-executor 914 1 767 0 DV ppwait 0xfffffe0054925540 syz-executor 911 1 766 0 S uwait 0xfffffe006eb67280 syz-executor 910 1 765 0 S uwait 0xfffffe0058d07c80 syz-executor 899 1 764 0 SV uwait 0xfffffe006eb67380 syz-executor 895 1 895 0 Ss select 0xfffffe0077f0eec0 rtsol 894 1 894 0 Ss select 0xfffffe006e3dec40 rtsol 893 1 893 0 Ss select 0xfffffe0077f0ef40 rtsol 890 777 424 0 S kqread 0xfffffe006e733800 rtsol 887 1 767 0 S uwait 0xfffffe0058958980 syz-executor 873 1 767 0 S uwait 0xfffffe0058958380 syz-executor 861 1 766 0 S uwait 0xfffffe006eb68380 syz-executor 860 1 765 0 S uwait 0xfffffe0058d07980 syz-executor 847 1 764 0 S uwait 0xfffffe0007f7ea00 syz-executor 831 0 0 0 DL (threaded) [KTLS] 100145 D - 0xfffffe0059e82700 [thr_0] 100146 D - 0xfffffe0059e82780 [thr_1] 100147 D - 0xffffffff83caec28 [reclaim_0] 825 1 766 0 S uwait 0xfffffe0007f7ec00 syz-executor 821 0 0 0 DL aiordy 0xfffffe005490fac0 [aiod4] 820 0 0 0 DL aiordy 0xfffffe0054910020 [aiod3] 819 0 0 0 DL aiordy 0xfffffe0054910ae0 [aiod2] 817 0 0 0 DL aiordy 0xfffffe0054910580 [aiod1] 811 1 767 0 S uwait 0xfffffe006eb68580 syz-executor 777 771 424 0 S wait 0xfffffe0054807ae0 sh 771 424 424 0 S wait 0xfffffe0008026560 sh 767 763 767 0 S nanslp 0xffffffff83b9d500 syz-executor 766 763 766 0 S nanslp 0xffffffff83b9d501 syz-executor 765 763 765 0 S nanslp 0xffffffff83b9d501 syz-executor 764 763 764 0 S nanslp 0xffffffff83b9d500 syz-executor 763 761 761 0 S select 0xfffffe006e3deec0 syz-executor 761 759 761 0 Ss pause 0xfffffe0054809670 csh 759 682 759 0 Ss select 0xfffffe006e3df040 sshd 748 1 748 0 Ss+ ttyin 0xfffffe0058a9c8b0 getty 747 1 747 0 Ss+ ttyin 0xfffffe0059cbf8b0 getty 746 1 746 0 Ss+ ttyin 0xfffffe0059cc00b0 getty 745 1 745 0 Ss+ ttyin 0xfffffe0059cc08b0 getty 744 1 744 0 Ss+ ttyin 0xfffffe005476e0b0 getty 743 1 743 0 Ss+ ttyin 0xfffffe005476e8b0 getty 742 1 742 0 Ss+ ttyin 0xfffffe005476f0b0 getty 741 1 741 0 Ss+ ttyin 0xfffffe005476f8b0 getty 740 1 740 0 Ss+ ttyin 0xfffffe00547700b0 getty 738 1 18 0 S+ piperd 0xfffffe005a03d8a0 logger 737 736 18 0 S+ nanslp 0xffffffff83b9d501 sleep 736 1 18 0 S+ wait 0xfffffe00548d3b00 sh 686 1 686 0 Ss nanslp 0xffffffff83b9d501 cron 682 1 682 0 Ss select 0xfffffe0059f0a540 sshd 495 1 495 0 Ss select 0xfffffe0059f0a2c0 syslogd 424 1 424 0 Ss wait 0xfffffe0054807580 devd 423 1 423 65 Ss select 0xfffffe0059f0a440 dhclient 338 1 338 0 Ss select 0xfffffe0059f0a340 dhclient 335 1 335 0 Ss select 0xfffffe0059f0a4c0 dhclient 17 0 0 0 DL syncer 0xffffffff83cbada0 [syncer] 16 0 0 0 DL vlruwt 0xfffffe0008028040 [vnlru] 15 0 0 0 DL (threaded) [bufdaemon] 100080 D psleep 0xffffffff83cb9360 [bufdaemon] 100083 D - 0xffffffff83002140 [bufspacedaemon-0] 100094 D sdflush 0xfffffe0058ddfce8 [/ worker] 9 0 0 0 DL psleep 0xffffffff83d04380 [vmdaemon] 8 0 0 0 DL (threaded) [pagedaemon] 100078 D psleep 0xffffffff83cea2f8 [dom0] 100081 D launds 0xffffffff83cea304 [laundry: dom0] 100082 D umarcl 0xffffffff81dceb60 [uma] 7 0 0 0 DL - 0xffffffff8391acd0 [rand_harvestq] 6 0 0 0 DL pftm 0xffffffff84798850 [pf purge] 5 0 0 0 DL waiting 0xffffffff844aa700 [sctp_iterator] 4 0 0 0 DL (threaded) [cam] 100046 D - 0xffffffff838e5340 [doneq0] 100047 D - 0xffffffff838e52c0 [async] 100076 D - 0xffffffff838e5140 [scanner] 3 0 0 0 DL (threaded) [crypto] 100043 D crypto_ 0xffffffff83ce5b40 [crypto] 100044 D crypto_ 0xfffffe005856a330 [crypto returns 0] 100045 D crypto_ 0xfffffe005856a380 [crypto returns 1] 14 0 0 0 DL seqstat 0xfffffe00547edc88 [sequencer 00] 13 0 0 0 DL (threaded) [geom] 100037 D - 0xffffffff83b45f00 [g_event] 100038 D - 0xffffffff83b45f20 [g_up] 100039 D - 0xffffffff83b45f40 [g_down] 2 0 0 0 WL (threaded) [clock] 100031 I [clock (0)] 100032 I [clock (1)] 12 0 0 0 WL (threaded) [intr] 100013 I [swi6: task queue] 100014 I [swi6: Giant taskq] 100016 I [swi5: fast taskq] 100033 I [swi1: netisr 0] 100034 I [swi1: hpts] 100035 I [swi1: hpts] 100048 I [irq24: virtio_pci0] 100049 I [irq25: virtio_pci0] 100050 I [irq26: virtio_pci0] 100051 I [irq27: virtio_pci0] 100052 I [irq28: virtio_pci1] 100053 I [irq29: virtio_pci1] 100054 I [irq30: virtio_pci1] 100055 I [irq31: virtio_pci1] 100056 I [irq32: virtio_pci1] 100061 I [irq10: virtio_pci2] 100063 I [irq1: atkbd0] 100064 I [irq12: psm0] 100065 I [swi0: uart uart++] 100069 I [swi1: pf send] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffffe0008009040 [init] 10 0 0 0 DL audit_w 0xffffffff83ce65e0 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D parked 0xffffffff84c2cff0 [swapper] 100005 D - 0xfffffe005462d700 [softirq_0] 100006 D - 0xfffffe005462d600 [softirq_1] 100007 D - 0xfffffe005462d500 [if_io_tqg_0] 100008 D - 0xfffffe005462d400 [if_io_tqg_1] 100009 D - 0xfffffe005462d300 [if_config_tqg_0] 100010 D - 0xfffffe0008bf9500 [kqueue_ctx taskq] 100011 D - 0xfffffe0008bf9300 [jail_remove taskq] 100012 D - 0xfffffe0008bf9100 [bus taskq] 100015 D - 0xfffffe0008bf8a00 [thread taskq] 100017 D - 0xfffffe0008bf8600 [aiod_kick taskq] 100018 D - 0xfffffe0008bf8400 [deferred_unmount ta] 100019 D - 0xfffffe0008bf8200 [inm_free taskq] 100020 D - 0xfffffe0008bf8000 [in6m_free taskq] 100021 D - 0xfffffe0008bf7d00 [linuxkpi_irq_wq] 100022 D - 0xfffffe0008bf7b00 [linuxkpi_short_wq_0] 100023 D - 0xfffffe0008bf7b00 [linuxkpi_short_wq_1] 100024 D - 0xfffffe0008bf7b00 [linuxkpi_short_wq_2] 100025 D - 0xfffffe0008bf7b00 [linuxkpi_short_wq_3] 100026 D - 0xfffffe0008bf7600 [linuxkpi_long_wq_0] 100027 D - 0xfffffe0008bf7600 [linuxkpi_long_wq_1] 100028 D - 0xfffffe0008bf7600 [linuxkpi_long_wq_2] 100029 D - 0xfffffe0008bf7600 [linuxkpi_long_wq_3] 100036 D - 0xfffffe0008bf6a00 [firmware taskq] 100041 D - 0xfffffe0008bf6300 [crypto_0] 100042 D - 0xfffffe0008bf6300 [crypto_1] 100057 D - 0xfffffe0058569900 [vtnet0 rxq 0] 100058 D - 0xfffffe0058569800 [vtnet0 txq 0] 100059 D - 0xfffffe0058569700 [vtnet0 rxq 1] 100060 D - 0xfffffe0058569600 [vtnet0 txq 1] 100062 D vtbslp 0xfffffe0058591000 [virtio_balloon] 100066 D - 0xffffffff827b52a0 [deadlkres] 100070 D - 0xfffffe0058d0dd00 [acpi_task_0] 100071 D - 0xfffffe0058d0dd00 [acpi_task_1] 100072 D - 0xfffffe0058d0dd00 [acpi_task_2] 100074 D - 0xfffffe0008bfa100 [mca taskq] 100075 D - 0xfffffe005856a000 [CAM taskq] 100077 D - 0xfffffe0058568400 [ipsec_offload] 100272 D - 0xfffffe0078026b00 [netlink_socket (PID] db> show all locks Process 960 (syz-executor) thread 0xfffffe0054947740 (100308) exclusive sleep mutex so_snd (so_snd) r = 0 (0xfffffe005a0c79a0) locked @ /syzkaller/managers/main/kernel/sys/kern/uipc_sockbuf.c:393 exclusive sleep mutex unp (unp) r = 0 (0xfffffe005a375140) locked @ /syzkaller/managers/main/kernel/sys/kern/uipc_usrreq.c:2628 Process 954 (syz-executor) thread 0xfffffe005491f000 (100300) exclusive sx so_snd_sx (so_snd_sx) r = 0 (0xfffffe005a30e980) locked @ /syzkaller/managers/main/kernel/sys/kern/uipc_socket.c:4840 db> show malloc Type InUse MemUse Requests pf_hash 6 12804K 6 linker 394 5088K 626 tcp_hpts 7 4801K 7 devbuf 4188 4324K 4213 sysctloid 35071 2066K 35146 vtbuf 24 1968K 46 kobj 331 1324K 497 newblk 447 1136K 1135 vfscache 3 1025K 3 pcb 39 678K 129 inodedep 34 525K 235 ufs_quota 1 512K 1 vfs_hash 1 512K 1 callout 2 512K 2 intr 4 472K 4 filedesc 54 425K 257 subproc 168 350K 1049 vnet_data 2 224K 2 acpitask 1 224K 1 KTRACE 101 201K 5444 acpica 1674 184K 54432 vmem 5 144K 6 tidhash 3 141K 3 pagedep 22 134K 119 tfo_ccache 1 128K 1 IP reass 1 128K 1 DEVFS1 109 109K 126 sem 4 106K 4 gtaskqueue 18 98K 18 bus 1002 82K 5074 mtx_pool 3 74K 3 syncache 1 68K 1 NFSD srvcache 3 68K 3 module 522 66K 526 ddb_capture 1 64K 1 kdtrace 236 45K 1270 umtx 352 44K 352 temp 36 37K 1952 shm 2 34K 7 hostcache 1 32K 1 DEVFS3 128 32K 138 msg 4 30K 4 kbdmux 6 28K 6 DEVFS_RULE 56 20K 56 ifaddr 66 19K 68 BPF 14 19K 20 ufs_mount 4 17K 5 proc 3 17K 3 LRO 16 17K 16 tty 16 16K 16 routetbl 124 16K 395 ithread 90 15K 90 lltable 46 15K 48 bus-sc 34 15K 1650 eventhandler 163 14K 163 ifnet 7 13K 7 ether_multi 152 13K 167 kenv 95 12K 95 GEOM 61 11K 477 CAM queue 5 11K 1528 rman 82 10K 447 rpc 8 9K 8 in6_multi 65 9K 65 bmsafemap 2 9K 204 plimit 22 9K 319 devstat 4 9K 4 UART 12 9K 12 ksem 1 8K 1 filemon 1 8K 67 shmfd 1 8K 6 pfs_vncache 1 8K 1 audit_evclass 239 8K 301 kqueue 82 8K 1125 taskqueue 72 8K 75 cred 28 7K 200 sglist 6 7K 6 CAM DEV 3 6K 510 pfs_nodes 22 6K 22 pwddesc 79 5K 972 ufs_dirhash 24 5K 24 UMA 268 5K 268 pf_ifnet 10 5K 19 tcp_fsb_rack 2 5K 8 vt 11 5K 11 memdesc 1 4K 1 MCA 32 4K 32 evdev 4 4K 4 acpisem 28 4K 28 DEVFSP 50 4K 140 selfd 49 4K 58884 proc-args 106 4K 1996 session 23 3K 34 terminal 11 3K 11 acpidev 20 3K 20 uidinfo 4 3K 12 hhook 8 3K 10 clone 9 3K 9 kcovinfo 36 3K 36 pf_rule 1 2K 1 local_apic 1 2K 1 io_apic 1 2K 1 ipsec-saq 2 2K 2 ip6ndp 12 2K 14 newdirblk 14 2K 101 dirrem 7 2K 177 Unitno 28 2K 52 sctp_ifa 13 2K 14 CAM XPT 22 2K 543 lockf 15 2K 22 sctp_atcl 4 2K 32 CC Mem 12 2K 126 in_multi 6 2K 9 tun 4 2K 4 select 12 2K 36 toponodes 6 2K 6 diradd 11 2K 191 ipsecpolicy 2 2K 2 msi 9 2K 9 netlink 2 2K 74 softdep 1 1K 1 indirdep 4 1K 145 sahead 1 1K 1 secasvar 1 1K 1 nhops 6 1K 8 vnodemarker 2 1K 12 NFSD session 1 1K 1 inpcbpolicy 28 1K 341 CAM periph 4 1K 271 ipsec 3 1K 3 sctp_timw 3 1K 3 sctp_ifn 6 1K 14 freefile 6 1K 130 mld 6 1K 6 igmp 6 1K 6 pfil 6 1K 6 isadev 6 1K 8 mount 16 1K 303 osd 17 1K 142 pci_link 10 1K 10 mkdir 5 1K 202 crypto 4 1K 20 encap_export_host 12 1K 12 procdesc 4 1K 10 cdev 2 1K 2 lkpikmalloc 8 1K 9 cryptodev 7 1K 85 chacha20random 1 1K 1 biobuf 1 1K 1 freework 2 1K 207 tcp_pcm_rack 1 1K 4 freeblks 1 1K 117 ip6opt 1 1K 10 ip_msource 4 1K 11 vnodes 1 1K 2 ktls 1 1K 5 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 DEVFS 9 1K 10 CAM SIM 2 1K 2 feeder 7 1K 7 tcpfunc 3 1K 3 loginclass 3 1K 5 prison 6 1K 6 sctp_atky 4 1K 38 nexusdev 8 1K 8 apmdev 1 1K 1 atkbddev 2 1K 2 aio 4 1K 5 pmchooks 1 1K 1 CAM path 4 1K 1034 CAM dev queue 2 1K 2 CAM I/O Scheduler 1 1K 1 soname 4 1K 3373 filecaps 4 1K 78 sctp_vrf 1 1K 1 sctp_athm 4 1K 34 ip6_msource 1 1K 2 vnet 1 1K 1 accf 1 1K 1 pmc 1 1K 1 sigio 1 1K 2 filedesc_to_leader 1 1K 2 entropy 2 1K 33 acpiintr 1 1K 1 cpus 2 1K 2 ip6_moptions 1 1K 3 vnet_data_free 1 1K 1 iov 1 1K 14765 Per-cpu 1 1K 1 p1003.1b 1 1K 1 ipcomp 0 0K 0 esp 0 0K 0 ah 0 0K 0 mqdata 0 0K 0 pf_table 0 0K 0 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_krule_item 0 0K 0 pf_temp 0 0K 0 tcp_do_rack 0 0K 0 sctp_mcore 0 0K 0 sctp_socko 0 0K 6 sctp_iter 0 0K 10 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 0 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 10 sctp_aadr 0 0K 1 sctp_stro 0 0K 3 sctp_stri 0 0K 2 sctp_map 0 0K 6 madt_table 0 0K 2 smartpqi 0 0K 0 ixl 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 iavf 0 0K 0 axgbe 0 0K 0 fpukern_ctx 0 0K 0 xen_intr 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 NMI handlers 0 0K 0 bounce 0 0K 0 busdma 0 0K 0 qpidrv 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 amdiommu_dom 0 0K 0 amdiommu_ctx 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 aesni_data 0 0K 3 xenbus 0 0K 0 vm_fictitious 0 0K 0 UMAHash 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 258 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 4 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 58 allocindir 0 0K 0 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 ktls_ocf 0 0K 1 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS_RX 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 simple_attr 0 0K 0 seq_file 0 0K 0 lkpiskb 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpindev 0 0K 0 lkpimhi 0 0K 0 lkpifw 0 0K 0 lkpi80211 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 in6_mfilter 0 0K 4 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 ip_moptions 0 0K 6 in_mfilter 0 0K 17 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 24 fadvise 0 0K 1 VN POLL 0 0K 1 statfs 0 0K 188 namei_tracker 0 0K 1 export_host 0 0K 0 cl_savebuf 0 0K 29 lio 0 0K 7 acl 0 0K 0 mbuf_tag 0 0K 0 pts 0 0K 0 timerfd 0 0K 0 ioctlops 0 0K 144 eventfd 0 0K 2 Witness 0 0K 0 stack 0 0K 0 sbuf 0 0K 310 firmware 0 0K 0 compressor 0 0K 0 SWAP 0 0K 0 sysctltmp 0 0K 635 sysctl 0 0K 3 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 4 rctl 0 0K 0 cache 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 boottrace 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 tmpfs extattr 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xenstore 0 0K 0 xnb 0 0K 0 xen_acpi 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 sysmouse 0 0K 0 vtfont 0 0K 0 pvscsi 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 twsbuf 0 0K 0 tcp_log_dev 0 0K 1 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 SIIS driver 0 0K 0 PUC 0 0K 0 ppbusdev 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 mpi3mrbuf 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 md_sectors 0 0K 0 md_disk 0 0K 0 malodev 0 0K 0 LED 0 0K 0 ix_sriov 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 ciss_data 0 0K 0 BACKLIGHT 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 AHCI driver 0 0K 0 agp 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 acpicmbat 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 scsi_da 0 0K 70 ata_da 0 0K 0 scsi_ch 0 0K 0 scsi_cd 0 0K 0 nvme_da 0 0K 0 CAM CCB 0 0K 523 CAM ccb queue 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8327 1071 13281 0 254 38494208 0 mbuf 256 8669 993 18701 0 254 2473472 0 BUF TRIE 152 239 11565 908 0 62 1794208 0 malloc-384 384 4170 30 4196 0 30 1612800 0 malloc-128 128 11504 214 14983 0 126 1499904 0 malloc-4096 4096 350 6 1085 0 2 1458176 0 UMA Slabs 0 112 11081 1 11081 0 126 1241184 0 RADIX NODE 152 7549 220 31595 0 62 1180888 0 sctp_asoc 2256 0 510 3 0 254 1150560 0 malloc-65536 65536 15 2 21 0 1 1114112 0 malloc-64 64 323 17002 59322 0 254 1108800 0 mbuf_cluster 2048 508 0 508 0 254 1040384 0 vmem btag 56 16693 122 16693 0 254 941640 0 FFS inode 1168 549 32 682 0 8 678608 0 sctp_ep 1152 4 507 28 0 254 588672 0 malloc-16384 16384 31 4 135 0 1 573440 0 malloc-8192 8192 5 64 97 0 1 565248 0 socket 1024 56 452 1601 0 254 520192 0 lkpicurr 168 2 3094 2 0 62 520128 0 pbuf 2624 0 198 0 0 2 519552 0 256 Bucket 2048 200 8 1154 0 8 425984 0 sctp_raddr 736 0 517 3 0 254 380512 0 VM OBJECT 248 1359 81 14876 0 62 357120 0 malloc-65536 65536 0 5 68 0 1 327680 0 malloc-4096 4096 79 1 965 0 2 327680 0 THREAD 1824 156 20 308 0 8 321024 0 VNODE 440 588 105 723 0 30 304920 0 malloc-2048 2048 110 26 654 0 8 278528 0 malloc-64 64 4115 232 8490 0 254 278208 0 malloc-16 16 14525 225 17122 0 254 236000 0 malloc-32768 32768 1 6 132 0 1 229376 0 DEVCTL 1024 22 198 147 0 0 225280 0 tcp_log 416 0 513 4 0 254 213408 0 malloc-32 32 5611 563 7769 0 254 197568 0 malloc-65536 65536 1 2 11 0 1 196608 0 malloc-128 128 1310 209 25821 0 126 194432 0 MAP ENTRY 96 1892 124 47272 0 126 193536 0 UMA Zones 768 240 4 240 0 16 187392 0 lkpimm 56 1 3095 1 0 254 173376 0 unpcb 320 20 496 1198 0 254 165120 0 FPU_save_area 832 158 40 391 0 16 164736 0 malloc-1024 1024 144 16 167 0 16 163840 0 FFS2 dinode 256 549 81 682 0 62 161280 0 malloc-256 256 448 182 1429 0 62 161280 0 S VFS Cache 104 998 289 1183 0 126 133848 0 malloc-65536 65536 0 2 110 0 1 131072 0 mbuf_packet 256 40 468 1227 0 254 130048 0 PROC 1376 78 10 960 0 8 121088 0 tcp_inpcb 1304 12 78 126 0 8 117360 0 ksiginfo 112 70 974 116 0 126 116928 0 malloc-128 128 647 252 1682 0 126 115072 0 malloc-32768 32768 0 3 120 0 1 98304 0 malloc-32768 32768 3 0 3 0 1 98304 0 filedesc0 1072 79 12 972 0 8 97552 0 UMA Kegs 384 225 8 225 0 30 89472 0 128 Bucket 1024 49 34 261 0 16 84992 0 malloc-2048 2048 7 33 599 0 8 81920 0 malloc-384 384 130 80 331 0 30 80640 0 malloc-256 256 185 130 479 0 62 80640 0 malloc-256 256 191 124 1190 0 62 80640 0 sctp_chunk 152 0 520 2 0 254 79040 0 malloc-8192 8192 5 4 12 0 1 73728 0 malloc-4096 4096 11 7 29 0 2 73728 0 g_bio 408 0 180 7300 0 30 73440 0 malloc-64 64 644 427 16037 0 254 68544 0 malloc-128 128 426 101 928 0 126 67456 0 malloc-128 128 302 225 372 0 126 67456 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-32768 32768 1 1 3 0 1 65536 0 malloc-32768 32768 2 0 2 0 1 65536 0 malloc-2048 2048 17 15 44 0 8 65536 0 64 Bucket 512 87 41 1992 0 30 65536 0 malloc-8192 8192 6 1 8 0 1 57344 0 malloc-64 64 460 359 661 0 254 52416 0 Files 80 217 433 7871 0 126 52000 0 malloc-128 128 6 397 270 0 126 51584 0 malloc-256 256 106 89 241 0 62 49920 0 malloc-256 256 64 131 1033 0 62 49920 0 malloc-256 256 77 118 182 0 62 49920 0 malloc-256 256 67 128 584 0 62 49920 0 32 Bucket 256 76 119 746 0 62 49920 0 DIRHASH 1024 34 14 34 0 16 49152 0 NAMEI 1024 0 48 13657 0 16 49152 0 malloc-1024 1024 11 37 522 0 16 49152 0 VMSPACE 584 56 28 935 0 16 49056 0 da_ccb 544 0 84 1933 0 16 45696 0 syncache 168 0 264 3 0 254 44352 0 pcpu-8 8 4735 385 4919 0 254 40960 0 pipe 736 21 34 331 0 16 40480 0 udp_inpcb 408 7 83 170 0 30 36720 0 malloc-64 64 106 461 153 0 254 36288 0 malloc-64 64 14 553 19 0 254 36288 0 malloc-64 64 47 520 224 0 254 36288 0 malloc-64 64 17 550 699 0 254 36288 0 malloc-128 128 110 169 198 0 126 35712 0 malloc-128 128 10 269 20 0 126 35712 0 routing nhops 256 26 109 33 0 62 34560 0 ttyoutq 256 72 63 160 0 62 34560 0 malloc-384 384 63 27 63 0 30 34560 0 malloc-256 256 12 123 302 0 62 34560 0 TURNSTILE 136 177 75 177 0 62 34272 0 SLEEPQUEUE 88 177 207 177 0 126 33792 0 malloc-16384 16384 2 0 2 0 1 32768 0 malloc-16384 16384 2 0 2 0 1 32768 0 malloc-16384 16384 0 2 190 0 1 32768 0 malloc-8192 8192 2 2 26 0 1 32768 0 malloc-8192 8192 4 0 4 0 1 32768 0 malloc-4096 4096 4 4 10 0 2 32768 0 malloc-2048 2048 1 15 13 0 8 32768 0 malloc-2048 2048 7 9 7 0 8 32768 0 malloc-2048 2048 0 16 50 0 8 32768 0 malloc-2048 2048 3 13 194 0 8 32768 0 malloc-2048 2048 7 9 33 0 8 32768 0 malloc-1024 1024 2 30 42 0 16 32768 0 malloc-1024 1024 6 26 1070 0 16 32768 0 malloc-1024 1024 4 28 925 0 16 32768 0 malloc-1024 1024 10 22 167 0 16 32768 0 malloc-512 512 4 60 51 0 30 32768 0 malloc-512 512 1 63 61 0 30 32768 0 malloc-512 512 4 60 56 0 30 32768 0 malloc-512 512 8 56 8 0 30 32768 0 pcpu-64 64 496 16 496 0 254 32768 0 ttyinq 160 135 65 300 0 62 32000 0 PGRP 120 29 235 42 0 126 31680 0 clpbuf 2624 0 12 34 0 4 31488 0 sctp_laddr 48 0 588 13 0 254 28224 0 rl_entry 48 3 585 9 0 254 28224 0 malloc-32 32 69 813 875 0 254 28224 0 16 Bucket 144 64 132 321 0 62 28224 0 4 Bucket 48 5 583 10 0 254 28224 0 AIO 208 1 132 15 0 62 27664 0 udplite_inpcb 408 1 62 8 0 30 25704 0 cpuset 200 9 119 13 0 62 25600 0 malloc-8192 8192 1 2 82 0 1 24576 0 malloc-4096 4096 3 3 206 0 2 24576 0 malloc-4096 4096 3 3 7 0 2 24576 0 ertt_txseginfo 40 1 605 1393 0 254 24240 0 PWD 40 36 570 172 0 254 24240 0 rtentry 168 29 115 33 0 62 24192 0 8 Bucket 80 60 240 386 0 126 24000 0 malloc-384 384 1 59 6 0 30 23040 0 malloc-384 384 1 59 39 0 30 23040 0 malloc-384 384 1 59 344 0 30 23040 0 malloc-384 384 20 40 24 0 30 23040 0 malloc-384 384 27 33 352 0 30 23040 0 ripcb 376 8 52 37 0 30 22560 0 Mountpoints 2816 2 6 4 0 4 22528 0 hostcache 64 2 313 2 0 254 20160 0 ertt 72 12 268 126 0 126 20160 0 malloc-32 32 71 559 105 0 254 20160 0 malloc-32 32 110 520 1222 0 254 20160 0 malloc-32 32 79 551 325 0 254 20160 0 malloc-32 32 33 597 765 0 254 20160 0 malloc-32 32 72 558 2709 0 254 20160 0 2 Bucket 32 99 531 510 0 254 20160 0 KNOTE 160 4 121 48 0 62 20000 0 cryptop 280 0 70 5 0 30 19600 0 AIOCB 552 0 35 74 0 16 19320 0 ktls_session 256 0 75 1 0 62 19200 0 L VFS Cache 320 0 60 3 0 30 19200 0 AIOLIO 272 0 70 7 0 30 19040 0 vmem 1856 2 7 2 0 8 16704 0 epoch_record pcpu 256 4 60 4 0 62 16384 0 malloc-8192 8192 2 0 2 0 1 16384 0 malloc-4096 4096 0 4 40 0 2 16384 0 malloc-1024 1024 1 15 1 0 16 16384 0 malloc-1024 1024 0 16 1 0 16 16384 0 malloc-512 512 2 30 112 0 30 16384 0 malloc-512 512 3 29 8 0 30 16384 0 malloc-512 512 0 32 3 0 30 16384 0 SMR CPU 32 8 503 8 0 254 16352 0 pf tags 104 1 155 1 0 126 16224 0 tcp_bbr_pcb 896 0 18 1 0 16 16128 0 sctp_asconf_ack 48 0 336 1 0 254 16128 0 malloc-16 16 300 700 625 0 254 16000 0 kenv 258 17 43 1065 0 30 15480 0 tcp_rack_pcb 1088 1 13 4 0 8 15232 0 mqnode 416 3 33 3 0 30 14976 0 domainset 40 0 315 3 0 254 12600 0 SMR SHARED 24 8 503 8 0 254 12264 0 malloc-32 32 80 298 81 0 254 12096 0 malloc-16 16 31 719 181 0 254 12000 0 malloc-16 16 44 706 101 0 254 12000 0 malloc-16 16 189 561 3018 0 254 12000 0 malloc-16 16 35 715 24882 0 254 12000 0 malloc-16 16 22 728 80 0 254 12000 0 tfo_ccache_entries 80 1 127 1 0 126 10240 0 malloc-8192 8192 1 0 1 0 1 8192 0 malloc-4096 4096 1 1 4 0 2 8192 0 pcpu-16 16 8 504 8 0 254 8192 0 vtnet_tx_hdr 24 0 334 1438 0 254 8016 0 malloc-16 16 18 482 19 0 254 8000 0 UMA Slabs 1 176 9 13 9 0 62 3872 0 KMAP ENTRY 96 15 24 31 0 0 3744 0 FFS1 dinode 128 0 0 0 0 126 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 152 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 256 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 254 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf UDP mappings 104 0 0 0 0 126 0 0 pf source nodes 152 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 384 0 0 0 0 254 0 0 pf mtags 184 0 0 0 0 62 0 0 tcp_bbr_map 128 0 0 0 0 126 0 0 tcp_rack_map 128 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 sctp_stream_msg_out 112 0 0 0 0 254 0 0 sctp_readq 152 0 0 0 0 254 0 0 tcp_log_id_node 120 0 0 0 0 126 0 0 tcp_log_id_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 itimer 352 0 0 0 0 30 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 TMPFS node 240 0 0 0 0 62 0 0 NCLNODE 608 0 0 0 0 16 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 linux_dma_object 32 0 0 0 0 254 0 0 linux_dma_pctrie 152 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 112 0 0 0 0 126 0 0 skbuff 1808 0 0 0 0 8 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 16 0 0 nfspbuf 2624 0 0 0 0 4 0 0 swwbuf 2624 0 0 0 0 2 0 0 swrbuf 2624 0 0 0 0 4 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 152 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-512 512 0 0 0 0 30 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0