panic: mtx_lock() of spin mutex (null) @ /syzkaller/managers/i386/kernel/sys/netinet/sctputil.c:4802 cpuid = 1 time = 1593977919 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x47/frame 0xfffffe002c090680 vpanic() at vpanic+0x1c7/frame 0xfffffe002c0906e0 panic() at panic+0x43/frame 0xfffffe002c090740 __mtx_lock_flags() at __mtx_lock_flags+0x202/frame 0xfffffe002c0907a0 sctp_add_to_readq() at sctp_add_to_readq+0x4a2/frame 0xfffffe002c090810 sctp_ulp_notify() at sctp_ulp_notify+0xe66/frame 0xfffffe002c090880 sctp_setopt() at sctp_setopt+0x9e99/frame 0xfffffe002c090920 sctp_ctloutput() at sctp_ctloutput+0x22c/frame 0xfffffe002c090980 sosetopt() at sosetopt+0x101/frame 0xfffffe002c090a00 kern_setsockopt() at kern_setsockopt+0x158/frame 0xfffffe002c090a80 sys_setsockopt() at sys_setsockopt+0x33/frame 0xfffffe002c090ab0 ia32_syscall() at ia32_syscall+0x24e/frame 0xfffffe002c090bf0 int0x80_syscall_common() at int0x80_syscall_common+0x9c/frame 0xfbfbaf78 KDB: enter: panic [ thread pid 1345 tid 100287 ] Stopped at kdb_enter+0x67: movq $0,0x14a3436(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b ll+0x1a es 0x3b ll+0x1a fs 0x13 gs 0x1b ss 0 rax 0x12 rcx 0xfffffe0026200000 rdx 0x3ffff rbx 0 rsp 0xfffffe002c090660 rbp 0xfffffe002c090680 rsi 0x40001 rdi 0xffffffff810c14e6 vprintf+0x176 r8 0 r9 0xffffffff r10 0 r11 0xfffffe0025c6ba10 r12 0xffffffff82068f70 ddb_dbbe r13 0 r14 0xffffffff819a9572 r15 0xffffffff819a9572 rip 0xffffffff810b67f7 kdb_enter+0x67 rflags 0x200086 kernphys+0x86 kdb_enter+0x67: movq $0,0x14a3436(%rip) db> show proc Process 1345 (syz-executor.0) at 0xfffff8001091d520: state: NORMAL uid: 0 gids: 0, 0, 5 parent: pid 770 at 0xfffff8001091d000 ABI: FreeBSD ELF32 arguments: /root/syz-executor.0 reaper: 0xfffff80003310000 reapsubtree: 1 sigparent: 20 vmspace: 0xfffffe00258893d0 (map 0xfffffe00258893d0) (map.pmap 0xfffffe0025889490) (pmap 0xfffffe00258894f0) threads: 4 100330 Run CPU 0 syz-executor.0 100272 S connec 0xfffffe00239a2478 syz-executor.0 100273 S connec 0xfffffe002399ef10 syz-executor.0 100287 Run CPU 1 syz-executor.0 db> ps pid ppid pgrp uid state wmesg wchan cmd 1348 771 771 0 S (threaded) syz-executor.1 100103 S nanslp 0xffffffff8252f241 syz-executor.1 100284 S accept 0xfffffe00239e3698 syz-executor.1 100285 S uwait 0xfffff80003317300 syz-executor.1 1347 783 783 0 S (threaded) syz-executor.3 100195 S nanslp 0xffffffff8252f241 syz-executor.3 100276 S connec 0xfffffe00239a0800 syz-executor.3 100278 S uwait 0xfffff80010ae4d00 syz-executor.3 1346 775 775 0 R (threaded) syz-executor.2 100265 RunQ syz-executor.2 100277 S connec 0xfffffe00239a00f0 syz-executor.2 100279 S accept 0xfffffe002399ec00 syz-executor.2 100280 S uwait 0xfffff80003317500 syz-executor.2 1345 770 770 0 R (threaded) syz-executor.0 100330 Run CPU 0 syz-executor.0 100272 S connec 0xfffffe00239a2478 syz-executor.0 100273 S connec 0xfffffe002399ef10 syz-executor.0 100287 Run CPU 1 syz-executor.0 825 812 825 0 Ss select 0xfffff80003df5e40 dhclient 818 1 818 0 Ss select 0xfffff80003df5d40 dhclient 812 799 424 65 S select 0xfffff80003bf4940 dhclient 799 424 424 0 S wait 0xfffff8001091ea40 sh 783 768 783 0 Rs syz-executor.3 775 768 775 0 Ss nanslp 0xffffffff8252f240 syz-executor.2 771 768 771 0 Rs syz-executor.1 770 768 770 0 Ss nanslp 0xffffffff8252f241 syz-executor.0 768 766 766 0 S (threaded) syz-fuzzer 100091 S uwait 0xfffff800037ef780 syz-fuzzer 100104 S uwait 0xfffff800037ec200 syz-fuzzer 100105 S uwait 0xfffff800037ec500 syz-fuzzer 100106 S uwait 0xfffff800037ec600 syz-fuzzer 100107 S uwait 0xfffff80003a43480 syz-fuzzer 100108 S uwait 0xfffff80003a43580 syz-fuzzer 100109 S uwait 0xfffff800037ec700 syz-fuzzer 100110 S uwait 0xfffff800037ec800 syz-fuzzer 100112 S kqread 0xfffff80003dd1a00 syz-fuzzer 100113 S uwait 0xfffff80003a40c80 syz-fuzzer 766 764 766 0 Ss pause 0xfffff80003dc20a8 csh 764 682 764 0 Ss select 0xfffff80003e04a40 sshd 748 1 748 0 Ss+ ttyin 0xfffff80003806cb0 getty 747 1 747 0 Ss+ ttyin 0xfffff80003b06cb0 getty 746 1 746 0 Ss+ ttyin 0xfffff80003b090b0 getty 745 1 745 0 Ss+ ttyin 0xfffff80003b094b0 getty 744 1 744 0 Ss+ ttyin 0xfffff80003b098b0 getty 743 1 743 0 Ss+ ttyin 0xfffff80003b09cb0 getty 742 1 742 0 Ss+ ttyin 0xfffff80003b080b0 getty 741 1 741 0 Ss+ ttyin 0xfffff80003b084b0 getty 740 1 740 0 Ss+ ttyin 0xfffff80003b088b0 getty 686 1 686 0 Ss nanslp 0xffffffff8252f240 cron 682 1 682 0 Ss select 0xfffff80003bf4b40 sshd 495 1 495 0 Ss select 0xfffff80003e041c0 syslogd 424 1 424 0 Ss wait 0xfffff80003dc2520 devd 423 1 423 65 Ss select 0xfffff80003bf4c40 dhclient 338 1 338 0 Ss select 0xfffff80003bf4ec0 dhclient 335 1 335 0 Ss select 0xfffff80003bf5040 dhclient 23 0 0 0 DL syncer 0xffffffff8261a498 [syncer] 22 0 0 0 DL vlruwt 0xfffff800033eca40 [vnlru] 21 0 0 0 DL (threaded) [bufdaemon] 100069 D qsleep 0xffffffff826197e0 [bufdaemon] 100076 D - 0xffffffff8200aa00 [bufspacedaemon-0] 100087 D sdflush 0xfffff8000342c4e8 [/ worker] 20 0 0 0 DL psleep 0xffffffff82640708 [vmdaemon] 19 0 0 0 DL (threaded) [pagedaemon] 100067 D psleep 0xffffffff82634b98 [dom0] 100074 D launds 0xffffffff82634ba4 [laundry: dom0] 100075 D umarcl 0xffffffff8154f650 [uma] 18 0 0 0 DL - 0xffffffff82362e78 [rand_harvestq] 17 0 0 0 DL pftm 0xffffffff82c583a0 [pf purge] 16 0 0 0 DL waiting 0xffffffff8261cc10 [sctp_iterator] 15 0 0 0 DL - 0xffffffff82618dec [soaiod4] 9 0 0 0 DL - 0xffffffff82618dec [soaiod3] 8 0 0 0 DL - 0xffffffff82618dec [soaiod2] 7 0 0 0 DL - 0xffffffff82618dec [soaiod1] 6 0 0 0 DL (threaded) [cam] 100033 D - 0xffffffff8223abc0 [doneq0] 100066 D - 0xffffffff8223aa90 [scanner] 5 0 0 0 DL crypto_ 0xfffff80003212d90 [crypto returns 1] 4 0 0 0 DL crypto_ 0xfffff80003212d30 [crypto returns 0] 3 0 0 0 DL crypto_ 0xffffffff82632440 [crypto] 14 0 0 0 DL seqstat 0xfffff800030d5488 [sequencer 00] 13 0 0 0 DL (threaded) [geom] 100024 D - 0xffffffff8250e1c0 [g_event] 100025 D - 0xffffffff8250e1c8 [g_up] 100026 D - 0xffffffff8250e1d0 [g_down] 2 0 0 0 DL (threaded) [KTLS] 100017 D - 0xfffff80003093700 [thr_0] 100018 D - 0xfffff80003093740 [thr_1] 12 0 0 0 WL (threaded) [intr] 100012 I [swi6: task queue] 100013 I [swi6: Giant taskq] 100016 I [swi5: fast taskq] 100019 I [swi3: vm] 100020 I [swi1: netisr 0] 100021 I [swi4: clock (0)] 100022 I [swi4: clock (1)] 100034 I [irq24: virtio_pci0] 100035 I [irq25: virtio_pci0] 100036 I [irq26: virtio_pci0] 100037 I [irq27: virtio_pci0] 100038 I [irq28: virtio_pci1] 100039 I [irq29: virtio_pci1] 100040 I [irq30: virtio_pci1] 100041 I [irq31: virtio_pci1] 100042 I [irq32: virtio_pci1] 100047 I [irq10: virtio_pci2] 100049 I [irq1: atkbd0] 100050 I [irq12: psm0] 100051 I [swi0: uart uart++] 100060 I [swi1: pf send] 100072 I [swi1: hpts] 100073 I [swi1: hpts] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffff80003310000 [init] 10 0 0 0 DL audit_w 0xffffffff82632858 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff8250e750 [swapper] 100005 D - 0xfffff80003356100 [softirq_0] 100006 D - 0xfffff80003356000 [softirq_1] 100007 D - 0xfffff80003357e00 [if_io_tqg_0] 100008 D - 0xfffff80003357d00 [if_io_tqg_1] 100009 D - 0xfffff80003357c00 [if_config_tqg_0] 100010 D - 0xfffff80003351800 [kqueue_ctx taskq] 100011 D - 0xfffff80003351700 [aiod_kick taskq] 100014 D - 0xfffff80003351200 [in6m_free taskq] 100015 D - 0xfffff80003351100 [thread taskq] 100023 D - 0xfffff8000334e900 [firmware taskq] 100028 D - 0xfffff8000334e800 [crypto_0] 100029 D - 0xfffff8000334e800 [crypto_1] 100043 D - 0xfffff8000334e200 [vtnet0 rxq 0] 100044 D - 0xfffff8000334e100 [vtnet0 txq 0] 100045 D - 0xfffff8000334e000 [vtnet0 rxq 1] 100046 D - 0xfffff8000334fe00 [vtnet0 txq 1] 100048 D vtbslp 0xfffff800034f3580 [virtio_balloon] 100052 D - 0xfffff8000334f700 [mca taskq] 100057 D - 0xffffffff81d4edb0 [deadlkres] 100061 D - 0xfffff80003b66200 [acpi_task_0] 100062 D - 0xfffff80003b66200 [acpi_task_1] 100063 D - 0xfffff80003b66200 [acpi_task_2] 100065 D - 0xfffff8000334e700 [CAM taskq] db> show all locks Process 1345 (syz-executor.0) thread 0xfffffe0025c2b500 (100330) exclusive sleep mutex umtxql (umtxql) r = 0 (0xffffffff825536e0) locked @ /syzkaller/managers/i386/kernel/sys/kern/kern_umtx.c:512 Process 1345 (syz-executor.0) thread 0xfffffe0025c6b500 (100287) exclusive sleep mutex sctp-tcb (tcb) r = 0 (0xfffffe00259f98b0) locked @ /syzkaller/managers/i386/kernel/sys/netinet/sctp_usrreq.c:5188 exclusive sleep mutex sctp-inp (inp) r = 0 (0xfffff80010c29e68) locked @ /syzkaller/managers/i386/kernel/sys/netinet/sctp_usrreq.c:5124 db> show malloc Type InUse MemUse Requests pf_hash 5 11524K 5 devbuf 4212 4850K 4240 tcp_hpts 5 3201K 5 vtbuf 24 1968K 46 sysctloid 27951 1631K 28015 kobj 334 1336K 493 newblk 88 1046K 5745 vfscache 4 1025K 4 pcb 39 551K 1014 inodedep 32 528K 820 ufs_quota 1 512K 1 vfs_hash 1 512K 1 callout 2 512K 2 intr 4 472K 4 subproc 128 257K 1422 acpica 1674 185K 55859 vnet_data 1 168K 1 filedesc 21 149K 1117 pagedep 9 130K 556 tfo_ccache 1 128K 1 sem 4 106K 4 DEVFS1 105 105K 122 linker 239 96K 280 bus 996 80K 3392 mtx_pool 2 72K 2 syncache 1 68K 1 acpitask 1 64K 1 ddb_capture 1 64K 1 module 501 63K 501 umtx 342 43K 342 BPF 22 36K 22 kdtrace 183 36K 3961 temp 35 33K 1849 hostcache 1 32K 1 shm 1 32K 2 DEVFS3 124 31K 134 msg 4 30K 4 DEVFS_RULE 56 27K 56 vmem 3 26K 5 gtaskqueue 18 26K 18 ifaddr 73 25K 73 kbdmux 6 22K 6 lltable 48 18K 70 ufs_mount 5 17K 6 proc 3 17K 3 tty 16 16K 16 tidhash 1 16K 1 ithread 98 16K 98 ether_multi 172 14K 177 bus-sc 30 14K 1439 KTRACE 100 13K 100 ifnet 7 13K 7 kenv 92 12K 92 eventhandler 132 12K 132 in6_multi 89 11K 89 pfs_nodes 20 10K 20 freework 40 10K 2247 GEOM 60 10K 489 rman 82 10K 423 bmsafemap 3 9K 924 UART 12 9K 12 devstat 4 9K 4 rpc 2 8K 2 shmfd 1 8K 2 pfs_vncache 1 8K 1 audit_evclass 233 8K 291 cred 27 7K 465 sctp_timw 26 7K 26 CAM DEV 3 6K 510 sctp_atcl 12 6K 402 sctp_stro 6 6K 110 kqueue 57 6K 1353 vt 11 6K 11 plimit 21 6K 352 sglist 5 6K 5 CAM queue 5 6K 1528 taskqueue 45 5K 45 ufs_dirhash 24 5K 24 DEVFSP 72 5K 76 pf_ifnet 10 5K 19 UMA 249 5K 249 memdesc 1 4K 1 MCA 32 4K 32 dirrem 16 4K 745 evdev 4 4K 4 kcovinfo 64 4K 68 routetbl 23 4K 23 freeblks 15 4K 818 hhook 13 4K 13 session 25 4K 35 pgrp 25 4K 37 acpisem 22 3K 22 terminal 11 3K 11 freefile 20 3K 743 lockf 24 3K 791 proc-args 44 3K 541 uidinfo 3 3K 7 sctp_ifa 17 3K 17 local_apic 1 2K 1 io_apic 1 2K 1 indirdep 8 2K 2566 ipsec-saq 2 2K 2 ip6ndp 12 2K 21 select 15 2K 15 Unitno 28 2K 379 CAM XPT 22 2K 543 in_multi 6 2K 7 acpidev 20 2K 20 msi 9 2K 9 tun 7 2K 7 softdep 1 1K 1 ipsecpolicy 1 1K 1 sahead 1 1K 1 secasvar 1 1K 1 clone 8 1K 8 vnodemarker 2 1K 22 NFSD session 1 1K 1 CAM periph 4 1K 271 mkdir 6 1K 1088 diradd 6 1K 780 mld 6 1K 6 sctp_ifn 6 1K 6 sctp_atky 18 1K 512 igmp 6 1K 6 nhops 6 1K 6 toponodes 6 1K 6 isadev 6 1K 6 mount 16 1K 86 pci_link 10 1K 10 crypto 3 1K 3 newdirblk 4 1K 544 pfil 4 1K 4 chacha20random 1 1K 1 CAM SIM 2 1K 2 epoch 4 1K 4 cdev 2 1K 2 encap_export_host 8 1K 8 inpcbpolicy 13 1K 708 osd 3 1K 9 vnodes 1 1K 1 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 DEVFS 9 1K 10 feeder 7 1K 7 sctp_athm 12 1K 402 sctp_map 12 1K 220 soname 8 1K 6237 loginclass 3 1K 6 CAM dev queue 2 1K 2 CAM I/O Scheduler 1 1K 1 apmdev 1 1K 1 atkbddev 2 1K 2 CAM path 4 1K 1034 tcpfunc 2 1K 2 ktls 1 1K 1 pmchooks 1 1K 1 prison 4 1K 4 nexusdev 5 1K 5 entropy 2 1K 37 sctp_vrf 1 1K 1 vnet 1 1K 1 acpiintr 1 1K 1 pmc 1 1K 1 cpus 2 1K 2 vnet_data_free 1 1K 1 Per-cpu 1 1K 1 filecaps 2 1K 72 sctp_socko 1 1K 59 p1003.1b 1 1K 1 pf_table 0 0K 0 pf_rule 0 0K 0 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_temp 0 0K 0 vtfont 0 0K 0 madt_table 0 0K 2 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 amr 0 0K 0 scsi_da 0 0K 69 pvscsi 0 0K 0 smartpqi 0 0K 0 ata_da 0 0K 0 scsi_ch 0 0K 0 scsi_cd 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 AHCI driver 0 0K 0 agp 0 0K 0 iavf 0 0K 0 ixl 0 0K 0 nvme_da 0 0K 0 acpipwr 0 0K 0 twsbuf 0 0K 0 twe_commands 0 0K 0 twa_commands 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 fpukern_ctx 0 0K 0 tcp_log_dev 0 0K 0 xen_intr 0 0K 0 midi buffers 0 0K 0 mixer 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 qpidrv 0 0K 0 ac97 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 dmar_dmamap 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 isci 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 acpi_perf 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 vm_fictitious 0 0K 0 CAM CCB 0 0K 10648 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 UMAHash 0 0K 0 tempbuff 0 0K 0 ag_tgt_map_t malloc 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 719 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 10 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 8 allocindir 0 0K 0 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 xform 0 0K 0 ag_slr_map_t malloc 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 10 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 lDevFlags * malloc 0 0K 0 LRO 0 0K 0 sctp_mcore 0 0K 0 sctp_iter 0 0K 10 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 0 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 10 sctp_aadr 0 0K 0 sctp_stri 0 0K 0 newreno data 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 1 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 statfs 0 0K 715 export_host 0 0K 0 cl_savebuf 0 0K 3 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 biobuf 0 0K 0 aios 0 0K 0 lio 0 0K 0 acl 0 0K 0 MPSSAS 0 0K 0 mbuf_tag 0 0K 131 accf 0 0K 0 pts 0 0K 0 iov 0 0K 14030 ioctlops 0 0K 105 Witness 0 0K 0 stack 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 md_sectors 0 0K 0 sbuf 0 0K 288 md_disk 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 malodev 0 0K 0 SWAP 0 0K 0 LED 0 0K 0 sysctltmp 0 0K 591 sysctl 0 0K 1 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 1 rctl 0 0K 0 ix_sriov 0 0K 0 aacraidcam 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 iirbuf 0 0K 0 cache 0 0K 0 aacraid_buf 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 1 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroffdiroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xenstore 0 0K 0 scsi_pass 0 0K 0 ciss_data 0 0K 0 xnb 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 sysmouse 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_cluster 2048 9780 380 9782 0 254 20807680 0 mbuf_packet 256 8237 1415 639848 0 254 2470912 0 512 512 4140 36 4142 0 30 2138112 0 mbuf_jumbo_page 4096 0 508 92 0 254 2080768 0 BUF TRIE 144 226 13270 7026 0 62 1943424 0 4096 4096 336 2 946 0 2 1384448 0 RADIX NODE 144 9083 238 55301 0 62 1342224 0 128 128 9723 135 10866 0 126 1261824 0 sctp_asoc 2288 6 504 110 0 254 1166880 0 UMA Slabs 0 112 7117 5 7117 0 126 797664 0 pbuf 832 0 953 0 0 2 792896 0 sctp_ep 1280 6 504 292 0 254 652800 0 tcpcb 1040 9 508 527 0 254 537680 0 socket 904 28 485 2080 0 254 463752 0 sctp_raddr 736 6 511 110 0 254 380512 0 256 Bucket 2048 156 8 1578 0 8 335872 0 mbuf 256 579 704 315570 0 254 328448 0 VM OBJECT 264 1123 107 21628 0 30 324720 0 VNODE 488 544 88 1289 0 30 308416 0 THREAD 1792 141 30 1331 0 8 306432 0 256 256 153 972 8169 0 62 288000 0 65536 65536 4 0 4 0 1 262144 0 4096 4096 55 7 1349 0 2 253952 0 ripcb 488 2 510 11 0 254 249856 0 udplite_inpcb 488 0 512 14 0 254 249856 0 tcp_inpcb 488 9 503 527 0 254 249856 0 udp_inpcb 488 2 510 156 0 254 249856 0 16384 16384 10 4 559 0 1 229376 0 2048 2048 0 112 10680 0 8 229376 0 64 64 2810 277 3041 0 254 197568 0 65536 65536 1 2 62 0 1 196608 0 32768 32768 0 6 2379 0 1 196608 0 16 16 11776 224 11884 0 254 192000 0 g_bio 408 0 450 40246 0 30 183600 0 128 128 1160 235 25370 0 126 178560 0 UMA Zones 768 223 3 223 0 16 173568 0 MAP ENTRY 96 1354 410 70181 0 126 169344 0 32 32 4647 267 5291 0 254 157248 0 vmem btag 56 2163 488 2163 0 254 148456 0 1024 1024 117 27 136 0 16 147456 0 FFS2 dinode 256 507 63 1250 0 62 145920 0 65536 65536 2 0 2 0 1 131072 0 65536 65536 1 1 9 0 1 131072 0 unpcb 256 11 499 1054 0 254 130560 0 ksiginfo 112 53 991 100 0 126 116928 0 4096 4096 27 1 39 0 2 114688 0 VMSPACE 2536 32 13 1332 0 4 114120 0 S VFS Cache 108 613 395 1918 0 126 108864 0 PROC 1312 54 18 1348 0 8 94464 0 FFS inode 160 507 68 1250 0 62 92000 0 128 Bucket 1024 50 33 428 0 16 84992 0 filedesc0 1088 55 22 1349 0 8 83776 0 128 128 392 259 1475 0 126 83328 0 UMA Kegs 384 209 6 209 0 30 82560 0 256 256 179 136 3021 0 62 80640 0 sctp_readq 152 2 518 2 0 254 79040 0 sctp_chunk 152 4 516 4 0 254 79040 0 512 512 99 53 141 0 30 77824 0 1024 1024 10 58 388 0 16 69632 0 64 Bucket 512 87 49 1227 0 30 69632 0 64 64 520 551 1589 0 254 68544 0 128 128 323 204 550 0 126 67456 0 65536 65536 1 0 1 0 1 65536 0 65536 65536 0 1 112 0 1 65536 0 16384 16384 4 0 4 0 1 65536 0 2048 2048 8 24 520 0 8 65536 0 2048 2048 7 25 293 0 8 65536 0 256 256 157 98 530 0 62 65280 0 256 256 103 152 1031 0 62 65280 0 256 256 69 186 2066 0 62 65280 0 32 Bucket 256 53 202 974 0 62 65280 0 512 512 34 86 1330 0 30 61440 0 sctp_stream_msg_out 112 0 540 6 0 254 60480 0 8192 8192 6 1 34 0 1 57344 0 clpbuf 832 0 64 231 0 16 53248 0 64 64 294 525 13592 0 254 52416 0 64 64 465 354 750 0 254 52416 0 128 128 45 358 6763 0 126 51584 0 128 128 186 217 1840 0 126 51584 0 256 256 36 159 2076 0 62 49920 0 256 256 86 109 876 0 62 49920 0 DIRHASH 1024 34 14 34 0 16 49152 0 NAMEI 1024 0 48 15275 0 16 49152 0 512 512 53 43 85 0 30 49152 0 512 512 39 57 429 0 30 49152 0 syncache 168 0 264 4 0 254 44352 0 selfd 64 33 534 8296 0 254 36288 0 64 64 14 553 50 0 254 36288 0 64 64 64 503 1382 0 254 36288 0 64 64 7 560 381 0 254 36288 0 64 64 244 323 1906 0 254 36288 0 8 Bucket 80 58 392 13489 0 126 36000 0 128 128 42 237 89 0 126 35712 0 128 128 157 122 251 0 126 35712 0 routing nhops 256 17 118 39 0 62 34560 0 ttyoutq 256 72 63 160 0 62 34560 0 256 256 18 117 919 0 62 34560 0 TURNSTILE 136 172 80 172 0 62 34272 0 32768 32768 1 0 1 0 1 32768 0 32768 32768 1 0 1 0 1 32768 0 32768 32768 0 1 112 0 1 32768 0 32768 32768 1 0 1 0 1 32768 0 8192 8192 3 1 86 0 1 32768 0 8192 8192 4 0 4 0 1 32768 0 4096 4096 7 1 7 0 2 32768 0 2048 2048 9 7 12 0 8 32768 0 2048 2048 2 14 4 0 8 32768 0 2048 2048 9 7 9 0 8 32768 0 2048 2048 2 14 449 0 8 32768 0 1024 1024 16 16 16 0 16 32768 0 1024 1024 3 29 1032 0 16 32768 0 1024 1024 17 15 17 0 16 32768 0 512 512 21 43 139 0 30 32768 0 512 512 11 53 12 0 30 32768 0 mt_stats_zone 64 443 69 443 0 254 32768 0 64 pcpu 8 3760 336 3762 0 254 32768 0 16 Bucket 144 62 162 3543 0 62 32256 0 KNOTE 160 27 173 24557 0 62 32000 0 ttyinq 160 135 65 300 0 62 32000 0 Files 80 196 204 8971 0 126 32000 0 cpuset 104 7 272 7 0 126 29016 0 sctp_laddr 48 0 588 16 0 254 28224 0 hostcache 96 1 293 1 0 254 28224 0 PWD 32 19 863 631 0 254 28224 0 4 Bucket 48 6 582 6028 0 254 28224 0 KMAP ENTRY 96 12 279 12 0 126 27936 0 rtentry 208 35 98 39 0 62 27664 0 pipe 760 19 16 328 0 16 26600 0 8192 8192 2 1 4 0 1 24576 0 rl_entry 40 74 532 74 0 254 24240 0 2 Bucket 32 62 694 13582 0 254 24192 0 vtnet_tx_hdr 24 0 1002 310463 0 254 24048 0 SLEEPQUEUE 88 172 84 172 0 126 22528 0 udpcb 32 2 628 170 0 254 20160 0 32 32 28 602 230 0 254 20160 0 32 32 115 515 874 0 254 20160 0 32 32 116 514 949 0 254 20160 0 32 32 9 621 38 0 254 20160 0 32 32 106 524 1677 0 254 20160 0 32 32 46 584 3790 0 254 20160 0 procdesc 136 2 143 8 0 62 19720 0 epoch_record pcpu 256 4 60 4 0 62 16384 0 16384 16384 1 0 1 0 1 16384 0 16384 16384 0 1 136 0 1 16384 0 16384 16384 1 0 1 0 1 16384 0 8192 8192 2 0 2 0 1 16384 0 8192 8192 2 0 2 0 1 16384 0 8192 8192 2 0 2 0 1 16384 0 4096 4096 1 3 7 0 2 16384 0 4096 4096 4 0 5 0 2 16384 0 4096 4096 1 3 716 0 2 16384 0 2048 2048 1 7 13 0 8 16384 0 1024 1024 2 14 42 0 16 16384 0 1024 1024 6 10 6 0 16 16384 0 1024 1024 1 15 1 0 16 16384 0 mt_zone 24 443 225 443 0 254 16032 0 kenv 258 3 57 1112 0 30 15480 0 MAP 216 2 69 2 0 62 15336 0 tcp_bbr_pcb 832 0 18 2 0 16 14976 0 FPU_save_area 832 1 17 1 0 16 14976 0 vmem 1856 1 7 1 0 8 14848 0 32 32 12 366 207 0 254 12096 0 16 16 28 722 267 0 254 12000 0 16 16 17 733 524 0 254 12000 0 16 16 28 722 325 0 254 12000 0 16 16 202 548 1417 0 254 12000 0 16 16 4 746 5 0 254 12000 0 16 16 264 486 670 0 254 12000 0 16 16 22 728 25390 0 254 12000 0 Mountpoints 2816 2 2 2 0 4 11264 0 SMR CPU 32 2 253 2 0 254 8160 0 SMR SHARED 24 2 253 2 0 254 6120 0 512 512 2 6 323 0 30 4096 0 UMA Slabs 1 176 8 14 8 0 62 3872 0 int pcpu 4 18 494 18 0 254 2048 0 FFS1 dinode 128 0 0 0 0 126 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 62 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 296 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 48 0 0 0 0 254 0 0 tcp_bbr_map 128 0 0 0 0 126 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 tcp_log_node 120 0 0 0 0 126 0 0 tcp_log_bucket 176 0 0 0 0 62 0 0 tcp_log 416 0 0 0 0 254 0 0 tcpreass 48 0 0 0 0 254 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 88 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 itimer 352 0 0 0 0 30 0 0 AIOLIO 280 0 0 0 0 30 0 0 AIOCB 752 0 0 0 0 16 0 0 AIOP 32 0 0 0 0 254 0 0 AIO 208 0 0 0 0 62 0 0 NCLNODE 592 0 0 0 0 16 0 0 LTS VFS Cache 368 0 0 0 0 30 0 0 L VFS Cache 328 0 0 0 0 30 0 0 STS VFS Cache 148 0 0 0 0 62 0 0 rentr 24 0 0 0 0 254 0 0 VNODEPOLL 120 0 0 0 0 126 0 0 crypto_session 72 0 0 0 0 126 0 0 cryptop 280 0 0 0 0 30 0 0 nvme_request 128 0 0 0 0 126 0 0 DMAR_MAP_ENTRY 120 0 0 0 0 126 0 0 ktls_session 192 0 0 0 0 62 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 domainset 40 0 0 0 0 254 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 832 0 0 0 0 62 0 0 mdpbuf 832 0 0 0 0 4 0 0 nfspbuf 832 0 0 0 0 16 0 0 swwbuf 832 0 0 0 0 8 0 0 swrbuf 832 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 65536 65536 0 0 0 0 1 0 0 65536 65536 0 0 0 0 1 0 0 32768 32768 0 0 0 0 1 0 0 32768 32768 0 0 0 0 1 0 0 32768 32768 0 0 0 0 1 0 0 16384 16384 0 0 0 0 1 0 0 16384 16384 0 0 0 0 1 0 0 16384 16384 0 0 0 0 1 0 0 8192 8192 0 0 0 0 1 0 0 4096 4096 0 0 0 0 2 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0