panic: mtx_lock() of spin mutex (null) @ /syzkaller/managers/i386/kernel/sys/netinet/sctputil.c:3275 cpuid = 0 time = 1602714849 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x47/frame 0xfffffe000365b0a0 vpanic() at vpanic+0x1c7/frame 0xfffffe000365b100 panic() at panic+0x43/frame 0xfffffe000365b160 __mtx_lock_flags() at __mtx_lock_flags+0x202/frame 0xfffffe000365b1c0 sctp_notify_assoc_change() at sctp_notify_assoc_change+0x5dd/frame 0xfffffe000365b230 sctp_ulp_notify() at sctp_ulp_notify+0x288/frame 0xfffffe000365b290 sctp_handle_cookie_ack() at sctp_handle_cookie_ack+0x252/frame 0xfffffe000365b2d0 sctp_process_control() at sctp_process_control+0x19b3/frame 0xfffffe000365b750 sctp_common_input_processing() at sctp_common_input_processing+0x7db/frame 0xfffffe000365b8e0 sctp_input_with_port() at sctp_input_with_port+0x308/frame 0xfffffe000365b9d0 sctp_input() at sctp_input+0x1f/frame 0xfffffe000365b9f0 ip_input() at ip_input+0x388/frame 0xfffffe000365ba90 swi_net() at swi_net+0x20d/frame 0xfffffe000365bb10 ithread_loop() at ithread_loop+0x33f/frame 0xfffffe000365bbb0 fork_exit() at fork_exit+0xb3/frame 0xfffffe000365bbf0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe000365bbf0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- KDB: enter: panic [ thread pid 12 tid 100023 ] Stopped at kdb_enter+0x67: movq $0,0x1481526(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b ll+0x1a es 0x3b ll+0x1a fs 0x13 gs 0x1b ss 0 rax 0x12 rcx 0xffffffff810ddad0 vprintf+0x140 rdx 0x1 rbx 0 rsp 0xfffffe000365b080 rbp 0xfffffe000365b0a0 rsi 0 rdi 0xffffffff810ddb06 vprintf+0x176 r8 0 r9 0xffffffff r10 0x9757 ll+0x9736 r11 0x2b4e ll+0x2b2d r12 0xffffffff82066b10 ddb_dbbe r13 0 r14 0xffffffff819674ac r15 0xffffffff819674ac rip 0xffffffff810d2057 kdb_enter+0x67 rflags 0x82 ll+0x61 kdb_enter+0x67: movq $0,0x1481526(%rip) db> show proc Process 12 (intr) at 0xfffff80004316520: state: NORMAL uid: 0 gids: 0 parent: pid 0 at 0xffffffff82506210 ABI: null reaper: 0xffffffff82506210 reapsubtree: 12 sigparent: 20 vmspace: 0xffffffff82506e60 (map 0xffffffff82506e60) (map.pmap 0xffffffff82506f20) (pmap 0xffffffff82506f80) threads: 23 100010 I [swi5: fast taskq] 100014 I [swi6: task queue] 100015 I [swi6: Giant taskq] 100020 I [swi3: vm] 100021 I [swi4: clock (0)] 100022 I [swi4: clock (1)] 100023 Run CPU 0 [swi1: netisr 0] 100035 I [irq24: virtio_pci0] 100036 I [irq25: virtio_pci0] 100037 I [irq26: virtio_pci0] 100038 I [irq27: virtio_pci0] 100039 I [irq28: virtio_pci1] 100040 I [irq29: virtio_pci1] 100041 I [irq30: virtio_pci1] 100042 I [irq31: virtio_pci1] 100043 I [irq32: virtio_pci1] 100048 I [irq10: virtio_pci2] 100050 I [irq1: atkbd0] 100051 I [irq12: psm0] 100052 I [swi0: uart uart++] 100060 I [swi1: pf send] 100073 I [swi1: hpts] 100074 I [swi1: hpts] db> ps pid ppid pgrp uid state wmesg wchan cmd 7652 777 777 0 R (threaded) syz-executor.2 100120 Run CPU 1 syz-executor.2 100967 S accept 0xfffff80004e12ca8 syz-executor.2 100973 RunQ syz-executor.2 100978 S uwait 0xfffff8001fd57800 syz-executor.2 100979 RunQ syz-executor.2 6791 6785 6791 0 Ss select 0xfffff800045904c0 dhclient 6788 1 6788 0 Ss select 0xfffff80004a60640 dhclient 6785 6778 424 65 S select 0xfffff80004580cc0 dhclient 6778 424 424 0 S wait 0xfffff8001ffbaa40 sh 6769 1 6769 65 Ss select 0xfffff80004a606c0 dhclient 4891 1 4891 0 Ss select 0xfffff800045902c0 dhclient 4888 1 4888 0 Ss select 0xfffff80004580e40 dhclient 4869 1 4869 65 Ss select 0xfffff80004a60740 dhclient 1166 1 1166 0 Ss select 0xfffff80004590240 dhclient 1163 1 1163 0 Ss select 0xfffff80004590340 dhclient 812 762 812 0 Ss piperd 0xfffff80004cb58e8 syz-executor.3 777 762 777 0 Ss nanslp 0xffffffff82526d01 syz-executor.2 768 762 768 0 Ss piperd 0xfffff80004cc12f8 syz-executor.1 767 762 767 0 Ss piperd 0xfffff8001fb782f8 syz-executor.0 762 760 760 0 S (threaded) syz-fuzzer 100105 S kqread 0xfffff80004a06b00 syz-fuzzer 100108 S uwait 0xfffff80004b6a880 syz-fuzzer 100109 S uwait 0xfffff8000447d980 syz-fuzzer 100110 S uwait 0xfffff8000447db80 syz-fuzzer 100111 S uwait 0xfffff80004b6a980 syz-fuzzer 100112 S uwait 0xfffff80004592700 syz-fuzzer 100113 S uwait 0xfffff80004592800 syz-fuzzer 100114 S uwait 0xfffff80004592900 syz-fuzzer 100115 S uwait 0xfffff80004592a00 syz-fuzzer 760 758 760 0 Ss pause 0xfffff8001f6de5c8 csh 758 682 758 0 Ss select 0xfffff800045906c0 sshd 742 1 742 0 Ss+ ttyin 0xfffff800047e3cb0 getty 741 1 741 0 Ss+ ttyin 0xfffff80004b770b0 getty 740 1 740 0 Ss+ ttyin 0xfffff80004b774b0 getty 739 1 739 0 Ss+ ttyin 0xfffff80004b778b0 getty 738 1 738 0 Ss+ ttyin 0xfffff80004b77cb0 getty 737 1 737 0 Ss+ ttyin 0xfffff80004b760b0 getty 736 1 736 0 Ss+ ttyin 0xfffff80004b764b0 getty 735 1 735 0 Ss+ ttyin 0xfffff80004b768b0 getty 734 1 734 0 Ss+ ttyin 0xfffff80004b76cb0 getty 686 1 686 0 Ss nanslp 0xffffffff82526d00 cron 682 1 682 0 Ss select 0xfffff80004580840 sshd 495 1 495 0 Ss select 0xfffff80004580bc0 syslogd 424 1 424 0 Ss wait 0xfffff80004caca40 devd 423 1 423 65 Ss select 0xfffff80004580c40 dhclient 338 1 338 0 Ss select 0xfffff80004580ac0 dhclient 335 1 335 0 Ss select 0xfffff80004580d40 dhclient 23 0 0 0 DL vlruwt 0xfffff800043ee520 [vnlru] 22 0 0 0 DL syncer 0xffffffff82613de8 [syncer] 21 0 0 0 DL (threaded) [bufdaemon] 100070 D qsleep 0xffffffff82613150 [bufdaemon] 100077 D - 0xffffffff8200ae00 [bufspacedaemon-0] 100087 D sdflush 0xfffff80004b75ce8 [/ worker] 20 0 0 0 DL psleep 0xffffffff8263a3c8 [vmdaemon] 19 0 0 0 DL (threaded) [pagedaemon] 100068 D psleep 0xffffffff8262e7f8 [dom0] 100075 D launds 0xffffffff8262e804 [laundry: dom0] 100076 D umarcl 0xffffffff814d8370 [uma] 18 0 0 0 DL - 0xffffffff8235a888 [rand_harvestq] 17 0 0 0 DL waiting 0xffffffff82d0a1d8 [sctp_iterator] 16 0 0 0 DL pftm 0xffffffff82e45390 [pf purge] 15 0 0 0 DL - 0xffffffff8261272c [soaiod4] 9 0 0 0 DL - 0xffffffff8261272c [soaiod3] 8 0 0 0 DL - 0xffffffff8261272c [soaiod2] 7 0 0 0 DL - 0xffffffff8261272c [soaiod1] 6 0 0 0 DL (threaded) [cam] 100034 D - 0xffffffff822325c0 [doneq0] 100067 D - 0xffffffff82232490 [scanner] 5 0 0 0 DL crypto_ 0xfffff800043c3e90 [crypto returns 1] 4 0 0 0 DL crypto_ 0xfffff800043c3e30 [crypto returns 0] 3 0 0 0 DL crypto_ 0xffffffff8262bce0 [crypto] 14 0 0 0 DL seqstat 0xfffff800040d7888 [sequencer 00] 13 0 0 0 DL (threaded) [geom] 100025 D - 0xffffffff82505c70 [g_event] 100026 D - 0xffffffff82505c78 [g_up] 100027 D - 0xffffffff82505c80 [g_down] 2 0 0 0 DL (threaded) [KTLS] 100018 D - 0xfffff8000420dd00 [thr_0] 100019 D - 0xfffff8000420dd80 [thr_1] 12 0 0 0 RL (threaded) [intr] 100010 I [swi5: fast taskq] 100014 I [swi6: task queue] 100015 I [swi6: Giant taskq] 100020 I [swi3: vm] 100021 I [swi4: clock (0)] 100022 I [swi4: clock (1)] 100023 Run CPU 0 [swi1: netisr 0] 100035 I [irq24: virtio_pci0] 100036 I [irq25: virtio_pci0] 100037 I [irq26: virtio_pci0] 100038 I [irq27: virtio_pci0] 100039 I [irq28: virtio_pci1] 100040 I [irq29: virtio_pci1] 100041 I [irq30: virtio_pci1] 100042 I [irq31: virtio_pci1] 100043 I [irq32: virtio_pci1] 100048 I [irq10: virtio_pci2] 100050 I [irq1: atkbd0] 100051 I [irq12: psm0] 100052 I [swi0: uart uart++] 100060 I [swi1: pf send] 100073 I [swi1: hpts] 100074 I [swi1: hpts] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffff80004319000 [init] 10 0 0 0 DL audit_w 0xffffffff8262c200 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff82506210 [swapper] 100005 D - 0xfffff8000433f100 [if_config_tqg_0] 100006 D - 0xfffff8000433f000 [softirq_0] 100007 D - 0xfffff8000433ce00 [softirq_1] 100008 D - 0xfffff8000433cd00 [if_io_tqg_0] 100009 D - 0xfffff8000433cc00 [if_io_tqg_1] 100011 D - 0xfffff800040e1d00 [inm_free taskq] 100012 D - 0xfffff800040e1c00 [kqueue_ctx taskq] 100013 D - 0xfffff800040e1b00 [aiod_kick taskq] 100016 D - 0xfffff800040e1800 [in6m_free taskq] 100017 D - 0xfffff800040e1700 [thread taskq] 100024 D - 0xfffff800040e1600 [firmware taskq] 100029 D - 0xfffff800040e1400 [crypto_0] 100030 D - 0xfffff800040e1400 [crypto_1] 100044 D - 0xfffff800040de800 [vtnet0 rxq 0] 100045 D - 0xfffff800040de700 [vtnet0 txq 0] 100046 D - 0xfffff800040de600 [vtnet0 rxq 1] 100047 D - 0xfffff800040de500 [vtnet0 txq 1] 100049 D vtbslp 0xfffff80004583100 [virtio_balloon] 100053 D - 0xfffff800047f1d00 [mca taskq] 100058 D - 0xffffffff81d0e7e0 [deadlkres] 100062 D - 0xfffff80004a09400 [acpi_task_0] 100063 D - 0xfffff80004a09400 [acpi_task_1] 100064 D - 0xfffff80004a09400 [acpi_task_2] 100066 D - 0xfffff800040e1200 [CAM taskq] db> show all locks Process 7652 (syz-executor.2) thread 0xfffffe0025abac00 (100979) exclusive sleep mutex sctp-create (inp_create) r = 0 (0xfffff8001fd70e88) locked @ /syzkaller/managers/i386/kernel/sys/netinet/sctp_usrreq.c:7002 Process 12 (intr) thread 0xfffffe00049cd800 (100023) exclusive sleep mutex sctp-tcb (tcb) r = 0 (0xfffffe00258a4380) locked @ /syzkaller/managers/i386/kernel/sys/netinet/sctp_pcb.c:2143 db> show malloc Type InUse MemUse Requests pf_hash 5 11524K 5 devbuf 4214 4854K 4242 tcp_hpts 5 3201K 5 vtbuf 24 1968K 46 sysctloid 28170 1644K 28234 kobj 336 1344K 496 newblk 9 1026K 7282 vfscache 4 1025K 4 inodedep 78 551K 6810 pcb 35 546K 34807 ufs_quota 1 512K 1 vfs_hash 1 512K 1 callout 2 512K 2 intr 4 472K 4 subproc 140 273K 7742 acpica 1674 185K 52444 vnet_data 1 168K 1 sctp_timw 521 131K 521 pagedep 10 131K 6741 tfo_ccache 1 128K 1 sem 4 106K 4 DEVFS1 105 105K 122 filedesc 15 101K 13469 linker 254 99K 294 bus 979 79K 3032 mtx_pool 2 72K 2 BPF 38 71K 38 syncache 1 68K 1 acpitask 1 64K 1 ddb_capture 1 64K 1 module 507 64K 507 umtx 342 43K 342 kdtrace 183 35K 22399 temp 35 33K 2369 hostcache 1 32K 1 shm 1 32K 2 DEVFS3 124 31K 134 msg 4 30K 4 DEVFS_RULE 56 27K 56 vmem 3 26K 6 gtaskqueue 18 26K 18 ifaddr 75 26K 75 kbdmux 6 22K 6 dirrem 68 17K 6734 lltable 46 17K 48 ufs_mount 5 17K 6 proc 3 17K 3 tty 16 16K 16 tidhash 1 16K 1 ithread 99 16K 99 ether_multi 172 14K 182 bus-sc 30 14K 1413 KTRACE 100 13K 100 ifnet 7 13K 7 kenv 92 12K 92 eventhandler 132 12K 132 in6_multi 89 11K 89 pfs_nodes 20 10K 20 GEOM 60 10K 489 rman 82 10K 423 freefile 68 9K 6732 bmsafemap 2 9K 6780 UART 12 9K 12 devstat 4 9K 4 rpc 2 8K 2 shmfd 1 8K 1 pfs_vncache 1 8K 1 audit_evclass 233 8K 291 CAM DEV 3 6K 510 kqueue 60 6K 7666 plimit 23 6K 409 cred 23 6K 239 vt 11 6K 11 sglist 5 6K 5 CAM queue 5 6K 1528 taskqueue 48 6K 48 DEVFSP 76 5K 80 ufs_dirhash 24 5K 24 sctp_atcl 9 5K 42478 pf_ifnet 10 5K 19 UMA 251 5K 251 memdesc 1 4K 1 MCA 32 4K 32 evdev 4 4K 4 kcovinfo 64 4K 68 session 31 4K 46 pgrp 31 4K 46 routetbl 24 4K 24 select 26 4K 26 hhook 13 4K 13 lockf 28 3K 47 acpisem 22 3K 22 terminal 11 3K 11 proc-args 50 3K 628 uidinfo 3 3K 9 sctp_ifa 17 3K 19 sctp_stro 2 2K 10636 local_apic 1 2K 1 io_apic 1 2K 1 ipsec-saq 2 2K 2 ip6ndp 12 2K 21 Unitno 32 2K 12024 CAM XPT 22 2K 543 in_multi 6 2K 8 ipsecpolicy 2 2K 2 acpidev 20 2K 20 msi 9 2K 9 freework 5 2K 6731 clone 9 2K 9 tun 7 2K 7 sctp_stri 2 1K 9692 softdep 1 1K 1 mkdir 8 1K 13446 freeblks 4 1K 6730 sahead 1 1K 1 secasvar 1 1K 1 nhops 6 1K 8 vnodemarker 2 1K 26 NFSD session 1 1K 1 CAM periph 4 1K 271 ipsec 3 1K 3 sctp_ifn 6 1K 19 mld 6 1K 6 igmp 6 1K 6 toponodes 6 1K 6 isadev 6 1K 6 mount 16 1K 89 pci_link 10 1K 10 diradd 5 1K 6772 encap_export_host 12 1K 12 crypto 3 1K 3 newdirblk 4 1K 6723 pfil 4 1K 4 chacha20random 1 1K 1 CAM SIM 2 1K 2 cdev 2 1K 2 sctp_atky 11 1K 48271 inpcbpolicy 11 1K 351 osd 3 1K 9 vnodes 1 1K 1 ktls 1 1K 1 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 DEVFS 9 1K 10 feeder 7 1K 7 loginclass 3 1K 6 prison 6 1K 6 sctp_athm 9 1K 42481 CAM dev queue 2 1K 2 CAM I/O Scheduler 1 1K 1 apmdev 1 1K 1 atkbddev 2 1K 2 CAM path 4 1K 1034 tcpfunc 2 1K 2 soname 5 1K 16612 pmchooks 1 1K 1 nexusdev 5 1K 5 sctp_vrf 1 1K 1 sctp_map 4 1K 21272 entropy 2 1K 42 vnet 1 1K 1 acpiintr 1 1K 1 pmc 1 1K 1 filecaps 4 1K 105 cpus 2 1K 2 vnet_data_free 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 pf_table 0 0K 0 pf_rule 0 0K 0 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_temp 0 0K 0 sctp_mcore 0 0K 0 sctp_socko 0 0K 222 sctp_iter 0 0K 12 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 0 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 12 sctp_aadr 0 0K 0 vtfont 0 0K 0 madt_table 0 0K 2 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 amr 0 0K 0 scsi_da 0 0K 69 pvscsi 0 0K 0 smartpqi 0 0K 0 ata_da 0 0K 0 scsi_ch 0 0K 0 scsi_cd 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 AHCI driver 0 0K 0 agp 0 0K 0 iavf 0 0K 0 ixl 0 0K 0 nvme_da 0 0K 0 acpipwr 0 0K 0 twsbuf 0 0K 0 twe_commands 0 0K 0 twa_commands 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 axgbe 0 0K 0 fpukern_ctx 0 0K 0 tcp_log_dev 0 0K 0 xen_intr 0 0K 0 midi buffers 0 0K 0 mixer 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 qpidrv 0 0K 0 ac97 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 acpi_perf 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 vm_fictitious 0 0K 0 CAM CCB 0 0K 2146 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 tempbuff 0 0K 0 UMAHash 0 0K 0 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 4221 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 10 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 6 allocindir 0 0K 0 indirdep 0 0K 10 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 xform 0 0K 0 lDevFlags * malloc 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 6 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 LRO 0 0K 0 newreno data 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 0 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 0 ag_portal_data_t malloc 0 0K 0 statfs 0 0K 6905 export_host 0 0K 0 cl_savebuf 0 0K 2 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 biobuf 0 0K 0 aios 0 0K 0 lio 0 0K 0 acl 0 0K 0 MPSSAS 0 0K 0 mbuf_tag 0 0K 121 accf 0 0K 0 pts 0 0K 0 iov 0 0K 26432 ioctlops 0 0K 116 Witness 0 0K 0 stack 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 md_sectors 0 0K 0 sbuf 0 0K 288 md_disk 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 malodev 0 0K 0 SWAP 0 0K 0 LED 0 0K 0 sysctltmp 0 0K 669 sysctl 0 0K 1 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 ix_sriov 0 0K 0 aacraidcam 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 iirbuf 0 0K 0 cache 0 0K 0 aacraid_buf 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 3 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroffdiroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xenstore 0 0K 0 scsi_pass 0 0K 0 ciss_data 0 0K 0 xnb 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 sysmouse 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_cluster 2048 9780 126 9780 0 254 20287488 0 mbuf_jumbo_page 4096 0 762 19363 0 254 3121152 0 mbuf_packet 256 8214 1438 1197439 0 254 2470912 0 512 512 4152 24 4661 0 30 2138112 0 BUF TRIE 144 177 13291 751 0 62 1939392 0 RADIX NODE 144 12480 425 232689 0 63 1858320 0 sctp_asoc 2288 2 763 10636 0 254 1750320 0 4096 4096 339 3 566 0 2 1400832 0 128 128 9239 216 9251 0 126 1210240 0 512 512 107 2237 6882 0 30 1200128 0 sctp_ep 1280 5 757 17307 0 254 975360 0 UMA Slabs 0 112 7566 24 7566 0 126 850080 0 pbuf 832 0 969 0 0 2 806208 0 256 256 682 2183 14747 0 62 733440 0 sctp_raddr 736 12 758 59099 0 254 566720 0 tcpcb 1040 5 512 136 0 254 537680 0 socket 960 37 471 18798 0 254 487680 0 256 256 164 1576 22256 0 62 445440 0 256 Bucket 2048 181 19 2004 0 8 409600 0 65536 65536 6 0 6 0 1 393216 0 VM OBJECT 264 1206 114 109523 0 30 348480 0 4096 4096 76 8 7678 0 2 344064 0 VNODE 496 550 114 7285 0 30 329344 0 mbuf 256 502 781 1149488 0 254 328448 0 128 128 73 2345 6745 0 126 309504 0 THREAD 1792 144 27 7156 0 8 306432 0 ripcb 488 4 508 16 0 254 249856 0 udplite_inpcb 488 0 512 6 0 254 249856 0 tcp_inpcb 488 5 507 136 0 254 249856 0 udp_inpcb 488 2 510 193 0 254 249856 0 DEVCTL 1024 4 204 125 0 0 212992 0 16384 16384 6 7 6733 0 1 212992 0 64 64 2732 355 2735 0 254 197568 0 65536 65536 3 0 3 0 1 196608 0 128 128 1367 152 24843 0 126 194432 0 16 16 11835 165 11896 0 254 192000 0 MAP ENTRY 96 1438 452 407381 0 126 181440 0 UMA Zones 768 226 1 226 0 16 174336 0 32 32 4678 488 5118 0 254 165312 0 FFS2 dinode 256 514 116 7247 0 62 161280 0 256 256 77 553 7464 0 62 161280 0 vmem btag 56 2060 304 2060 0 254 132384 0 65536 65536 0 2 133 0 1 131072 0 65536 65536 0 2 8 0 1 131072 0 1024 1024 115 13 265 0 16 131072 0 unpcb 256 17 493 1107 0 254 130560 0 VMSPACE 2536 35 13 7643 0 4 121728 0 sctp_readq 152 0 780 8997 0 254 118560 0 sctp_chunk 152 10 770 48733 0 254 118560 0 128 Bucket 1024 73 42 2686 0 16 117760 0 ksiginfo 112 62 982 1508 0 126 116928 0 128 128 655 244 16977 0 126 115072 0 128 128 667 232 14811 0 126 115072 0 2048 2048 6 50 2916 0 8 114688 0 FFS inode 160 514 186 7247 0 62 112000 0 PROC 1312 57 24 7659 0 8 106272 0 S VFS Cache 104 617 397 7901 0 126 105456 0 64 Bucket 512 129 63 3490 0 30 98304 0 filedesc0 1080 58 26 7660 0 8 90720 0 4096 4096 17 5 6935 0 2 90112 0 g_bio 408 0 210 6238 0 30 85680 0 64 64 668 655 22265 0 254 84672 0 UMA Kegs 384 212 4 212 0 30 82944 0 8192 8192 8 2 99 0 1 81920 0 256 256 169 146 14783 0 62 80640 0 64 64 730 341 8560 0 254 68544 0 65536 65536 0 1 110 0 1 65536 0 32768 32768 2 0 2 0 1 65536 0 16384 16384 4 0 4 0 1 65536 0 2048 2048 9 23 17535 0 8 65536 0 32 Bucket 256 122 133 1369 0 62 65280 0 512 512 91 29 92 0 30 61440 0 sctp_stream_msg_out 112 0 540 9072 0 254 60480 0 DIRHASH 1024 34 14 34 0 16 49152 0 NAMEI 1024 0 48 40632 0 16 49152 0 8192 8192 3 3 31 0 1 49152 0 4096 4096 10 2 395 0 2 49152 0 2048 2048 10 14 12 0 8 49152 0 1024 1024 25 23 44 0 16 49152 0 1024 1024 4 44 10638 0 16 49152 0 512 512 80 16 86 0 30 49152 0 512 512 14 82 42818 0 30 49152 0 512 512 3 93 9693 0 30 49152 0 syncache 168 0 264 4 0 254 44352 0 Files 80 223 327 28917 0 126 44000 0 clpbuf 832 0 48 127 0 16 39936 0 pipe 760 22 28 376 0 16 38000 0 64 pcpu 8 4188 420 4192 0 254 36864 0 selfd 64 57 510 57362 0 254 36288 0 64 64 34 533 21642 0 254 36288 0 64 64 28 539 68 0 254 36288 0 64 64 195 372 231 0 254 36288 0 64 64 59 508 7685 0 254 36288 0 tcp_bbr_map 128 0 279 10 0 126 35712 0 128 128 12 267 2979 0 126 35712 0 128 128 114 165 455 0 126 35712 0 128 128 56 223 325 0 126 35712 0 routing nhops 256 28 107 39 0 62 34560 0 ttyoutq 256 72 63 160 0 62 34560 0 256 256 28 107 467 0 62 34560 0 256 256 12 123 13464 0 62 34560 0 256 256 58 77 7332 0 62 34560 0 256 256 31 104 41 0 62 34560 0 TURNSTILE 136 172 80 172 0 62 34272 0 32768 32768 1 0 1 0 1 32768 0 32768 32768 0 1 2 0 1 32768 0 32768 32768 0 1 112 0 1 32768 0 16384 16384 1 1 4 0 1 32768 0 16384 16384 1 1 137 0 1 32768 0 2048 2048 2 14 3 0 8 32768 0 2048 2048 8 8 10 0 8 32768 0 1024 1024 17 15 1173 0 16 32768 0 1024 1024 4 28 12 0 16 32768 0 512 512 14 50 14 0 30 32768 0 mt_stats_zone 64 446 66 446 0 254 32768 0 KNOTE 160 28 172 89672 0 62 32000 0 ttyinq 160 135 65 300 0 62 32000 0 tcp_bbr_pcb 832 0 36 2 0 16 29952 0 cpuset 104 7 272 7 0 126 29016 0 sctp_laddr 48 0 588 30 0 254 28224 0 hostcache 96 1 293 1 0 254 28224 0 PWD 32 18 864 6813 0 254 28224 0 32 32 118 764 31822 0 254 28224 0 4 Bucket 48 7 581 4683 0 254 28224 0 KMAP ENTRY 96 12 279 12 0 126 27936 0 8192 8192 3 0 3 0 1 24576 0 rtentry 176 35 103 39 0 62 24288 0 rl_entry 40 45 561 45 0 254 24240 0 vtnet_tx_hdr 24 0 1002 583952 0 254 24048 0 8 Bucket 80 42 258 17635 0 126 24000 0 Mountpoints 2816 2 6 2 0 4 22528 0 SLEEPQUEUE 88 172 84 172 0 126 22528 0 udpcb 32 2 628 199 0 254 20160 0 64 64 2 313 2 0 254 20160 0 32 32 39 591 239 0 254 20160 0 32 32 15 615 18784 0 254 20160 0 32 32 91 539 1872 0 254 20160 0 32 32 101 529 384 0 254 20160 0 32 32 45 585 55 0 254 20160 0 32 32 14 616 21 0 254 20160 0 16 Bucket 144 63 77 16979 0 62 20160 0 2 Bucket 32 56 574 8955 0 254 20160 0 procdesc 136 4 141 16 0 62 19720 0 epoch_record pcpu 256 4 60 4 0 62 16384 0 16384 16384 1 0 1 0 1 16384 0 8192 8192 2 0 2 0 1 16384 0 8192 8192 2 0 2 0 1 16384 0 8192 8192 2 0 2 0 1 16384 0 4096 4096 2 2 2 0 2 16384 0 4096 4096 0 4 4 0 2 16384 0 2048 2048 0 8 12 0 8 16384 0 2048 2048 1 7 2 0 8 16384 0 1024 1024 2 14 42 0 16 16384 0 1024 1024 0 16 1 0 16 16384 0 1024 1024 1 15 1 0 16 16384 0 512 512 0 32 119 0 30 16384 0 SMR CPU 32 2 509 2 0 254 16352 0 mt_zone 24 446 222 446 0 254 16032 0 16 16 467 533 15123 0 254 16000 0 16 16 41 959 45380 0 254 16000 0 16 16 16 984 42488 0 254 16000 0 kenv 258 3 57 1009 0 30 15480 0 FPU_save_area 832 1 17 1 0 16 14976 0 vmem 1856 1 7 1 0 8 14848 0 SMR SHARED 24 2 509 2 0 254 12264 0 16 16 16 734 2040 0 254 12000 0 16 16 9 741 20 0 254 12000 0 16 16 46 704 102 0 254 12000 0 16 16 0 750 2 0 254 12000 0 8192 8192 1 0 1 0 1 8192 0 4096 4096 0 2 2 0 2 8192 0 rentr 24 0 334 4 0 254 8016 0 UMA Slabs 1 176 9 13 9 0 62 3872 0 int pcpu 4 34 478 34 0 254 2048 0 FFS1 dinode 128 0 0 0 0 126 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 62 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 296 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 48 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 tcp_log_node 120 0 0 0 0 126 0 0 tcp_log_bucket 176 0 0 0 0 62 0 0 tcp_log 416 0 0 0 0 254 0 0 tcpreass 48 0 0 0 0 254 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 88 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 itimer 352 0 0 0 0 30 0 0 AIOLIO 280 0 0 0 0 30 0 0 AIOCB 752 0 0 0 0 16 0 0 AIOP 32 0 0 0 0 254 0 0 AIO 208 0 0 0 0 62 0 0 NCLNODE 592 0 0 0 0 16 0 0 TMPFS node 232 0 0 0 0 62 0 0 TMPFS dirent 64 0 0 0 0 254 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 VNODEPOLL 120 0 0 0 0 126 0 0 crypto_session 72 0 0 0 0 126 0 0 cryptop 280 0 0 0 0 30 0 0 nvme_request 128 0 0 0 0 126 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 ktls_session 192 0 0 0 0 62 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 domainset 40 0 0 0 0 254 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 832 0 0 0 0 62 0 0 mdpbuf 832 0 0 0 0 4 0 0 nfspbuf 832 0 0 0 0 16 0 0 swwbuf 832 0 0 0 0 8 0 0 swrbuf 832 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 65536 65536 0 0 0 0 1 0 0 65536 65536 0 0 0 0 1 0 0 65536 65536 0 0 0 0 1 0 0 32768 32768 0 0 0 0 1 0 0 32768 32768 0 0 0 0 1 0 0 32768 32768 0 0 0 0 1 0 0 32768 32768 0 0 0 0 1 0 0 16384 16384 0 0 0 0 1 0 0 16384 16384 0 0 0 0 1 0 0 16384 16384 0 0 0 0 1 0 0 8192 8192 0 0 0 0 1 0 0 4096 4096 0 0 0 0 2 0 0 2048 2048 0 0 0 0 8 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0