panic: mtx_lock() of spin mutex (null) @ /syzkaller/managers/i386/kernel/sys/netinet/sctputil.c:3275 cpuid = 1 time = 1602784864 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x47/frame 0xfffffe000365b0a0 vpanic() at vpanic+0x1c7/frame 0xfffffe000365b100 panic() at panic+0x43/frame 0xfffffe000365b160 __mtx_lock_flags() at __mtx_lock_flags+0x202/frame 0xfffffe000365b1c0 sctp_notify_assoc_change() at sctp_notify_assoc_change+0x5dd/frame 0xfffffe000365b230 sctp_ulp_notify() at sctp_ulp_notify+0x288/frame 0xfffffe000365b290 sctp_handle_cookie_ack() at sctp_handle_cookie_ack+0x252/frame 0xfffffe000365b2d0 sctp_process_control() at sctp_process_control+0x19b3/frame 0xfffffe000365b750 sctp_common_input_processing() at sctp_common_input_processing+0x7db/frame 0xfffffe000365b8e0 sctp_input_with_port() at sctp_input_with_port+0x308/frame 0xfffffe000365b9d0 sctp_input() at sctp_input+0x1f/frame 0xfffffe000365b9f0 ip_input() at ip_input+0x388/frame 0xfffffe000365ba90 swi_net() at swi_net+0x20d/frame 0xfffffe000365bb10 ithread_loop() at ithread_loop+0x33f/frame 0xfffffe000365bbb0 fork_exit() at fork_exit+0xb3/frame 0xfffffe000365bbf0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe000365bbf0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- KDB: enter: panic [ thread pid 12 tid 100023 ] Stopped at kdb_enter+0x67: movq $0,0x14814d6(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b ll+0x1a es 0x3b ll+0x1a fs 0x13 gs 0x1b ss 0 rax 0x12 rcx 0xffffffff810ddb20 vprintf+0x140 rdx 0x1 rbx 0 rsp 0xfffffe000365b080 rbp 0xfffffe000365b0a0 rsi 0 rdi 0xffffffff810ddb56 vprintf+0x176 r8 0 r9 0xffffffff r10 0x8bdb ll+0x8bba r11 0x234e ll+0x232d r12 0xffffffff82066b10 ddb_dbbe r13 0 r14 0xffffffff81967492 r15 0xffffffff81967492 rip 0xffffffff810d20a7 kdb_enter+0x67 rflags 0x82 ll+0x61 kdb_enter+0x67: movq $0,0x14814d6(%rip) db> show proc Process 12 (intr) at 0xfffff80004328520: state: NORMAL uid: 0 gids: 0 parent: pid 0 at 0xffffffff82506210 ABI: null reaper: 0xffffffff82506210 reapsubtree: 12 sigparent: 20 vmspace: 0xffffffff82506e60 (map 0xffffffff82506e60) (map.pmap 0xffffffff82506f20) (pmap 0xffffffff82506f80) threads: 23 100010 I [swi5: fast taskq] 100014 I [swi6: task queue] 100015 I [swi6: Giant taskq] 100020 I [swi3: vm] 100021 I [swi4: clock (0)] 100022 I [swi4: clock (1)] 100023 Run CPU 1 [swi1: netisr 0] 100035 I [irq24: virtio_pci0] 100036 I [irq25: virtio_pci0] 100037 I [irq26: virtio_pci0] 100038 I [irq27: virtio_pci0] 100039 I [irq28: virtio_pci1] 100040 I [irq29: virtio_pci1] 100041 I [irq30: virtio_pci1] 100042 I [irq31: virtio_pci1] 100043 I [irq32: virtio_pci1] 100048 I [irq10: virtio_pci2] 100050 I [irq1: atkbd0] 100051 I [irq12: psm0] 100052 I [swi0: uart uart++] 100060 I [swi1: pf send] 100073 I [swi1: hpts] 100074 I [swi1: hpts] db> ps pid ppid pgrp uid state wmesg wchan cmd 3440 766 766 0 R (threaded) syz-executor.1 100721 Run CPU 0 syz-executor.1 100681 S sbwait 0xfffff80004d8b6ac syz-executor.1 100682 S uwait 0xfffff80004c26700 syz-executor.1 3434 3091 3091 0 R (threaded) syz-executor.0 100119 RunQ syz-executor.0 100674 S connec 0xfffff80004d56870 syz-executor.0 100676 S uwait 0xfffff80004c27c80 syz-executor.0 100683 RunQ syz-executor.0 3290 3284 3290 0 Ss select 0xfffff80004ea0540 dhclient 3287 1 3287 0 Ss select 0xfffff8001fb6f6c0 dhclient 3284 3277 424 65 S select 0xfffff80004ea07c0 dhclient 3277 424 424 0 S wait 0xfffff8000712f000 sh 3267 1 3267 65 Ss select 0xfffff8001fb6f740 dhclient 3091 762 3091 0 Ss nanslp 0xffffffff82526d00 syz-executor.0 2588 1 2588 0 Ss select 0xfffff8001fbbdb40 dhclient 2585 1 2585 0 Ss select 0xfffff8001f0fd740 dhclient 2558 1 2558 65 Ss select 0xfffff80004b61dc0 dhclient 1909 1 1909 0 Ss select 0xfffff80004ea9140 dhclient 1906 1 1906 0 Ss select 0xfffff8001f0fd640 dhclient 832 762 832 0 Ss piperd 0xfffff80004cc5be0 syz-executor.3 776 762 776 0 Ss piperd 0xfffff80004cbf8e8 syz-executor.2 766 762 766 0 Ss nanslp 0xffffffff82526d00 syz-executor.1 762 760 760 0 S (threaded) syz-fuzzer 100109 S uwait 0xfffff80004c05e00 syz-fuzzer 100110 S uwait 0xfffff800047e1880 syz-fuzzer 100111 S uwait 0xfffff800047e1a80 syz-fuzzer 100112 S uwait 0xfffff800047e1b00 syz-fuzzer 100113 S uwait 0xfffff80004c29180 syz-fuzzer 100114 S uwait 0xfffff80004c05000 syz-fuzzer 100115 S kqread 0xfffff80004c3b800 syz-fuzzer 100116 S uwait 0xfffff80004c26b00 syz-fuzzer 100343 S uwait 0xfffff80004c06e80 syz-fuzzer 760 758 760 0 Ss pause 0xfffff80004cd00a8 csh 758 682 758 0 Ss select 0xfffff80004ea95c0 sshd 742 1 742 0 Ss+ ttyin 0xfffff800048090b0 getty 741 1 741 0 Ss+ ttyin 0xfffff80004b7e8b0 getty 740 1 740 0 Ss+ ttyin 0xfffff80004b810b0 getty 739 1 739 0 Ss+ ttyin 0xfffff80004b818b0 getty 738 1 738 0 Ss+ ttyin 0xfffff8000480f0b0 getty 737 1 737 0 Ss+ ttyin 0xfffff8000480f8b0 getty 736 1 736 0 Ss+ ttyin 0xfffff8000480e0b0 getty 735 1 735 0 Ss+ ttyin 0xfffff8000480e8b0 getty 734 1 734 0 Ss+ ttyin 0xfffff800048110b0 getty 686 1 686 0 Ss nanslp 0xffffffff82526d00 cron 682 1 682 0 Ss select 0xfffff80004ea96c0 sshd 495 1 495 0 Ss select 0xfffff80004ea9ec0 syslogd 424 1 424 0 Ss wait 0xfffff80004e4ea40 devd 423 1 423 65 Ss select 0xfffff80004bf7e40 dhclient 338 1 338 0 Ss select 0xfffff80004ea9dc0 dhclient 335 1 335 0 Ss select 0xfffff80004bf7dc0 dhclient 23 0 0 0 DL vlruwt 0xfffff80004b6ea40 [vnlru] 22 0 0 0 DL syncer 0xffffffff82613de8 [syncer] 21 0 0 0 DL (threaded) [bufdaemon] 100070 D qsleep 0xffffffff82613150 [bufdaemon] 100075 D - 0xffffffff8200ae00 [bufspacedaemon-0] 100088 D sdflush 0xfffff800045ce8e8 [/ worker] 20 0 0 0 DL psleep 0xffffffff8263a3c8 [vmdaemon] 19 0 0 0 DL (threaded) [pagedaemon] 100068 D psleep 0xffffffff8262e7f8 [dom0] 100076 D launds 0xffffffff8262e804 [laundry: dom0] 100077 D umarcl 0xffffffff814d83d0 [uma] 18 0 0 0 DL - 0xffffffff8235a888 [rand_harvestq] 17 0 0 0 DL waiting 0xffffffff82c941d8 [sctp_iterator] 16 0 0 0 DL pftm 0xffffffff82e45390 [pf purge] 15 0 0 0 DL - 0xffffffff8261272c [soaiod4] 9 0 0 0 DL - 0xffffffff8261272c [soaiod3] 8 0 0 0 DL - 0xffffffff8261272c [soaiod2] 7 0 0 0 DL - 0xffffffff8261272c [soaiod1] 6 0 0 0 DL (threaded) [cam] 100034 D - 0xffffffff822325c0 [doneq0] 100067 D - 0xffffffff82232490 [scanner] 5 0 0 0 DL crypto_ 0xfffff80004402090 [crypto returns 1] 4 0 0 0 DL crypto_ 0xfffff80004402030 [crypto returns 0] 3 0 0 0 DL crypto_ 0xffffffff8262bce0 [crypto] 14 0 0 0 DL seqstat 0xfffff800043d2c88 [sequencer 00] 13 0 0 0 DL (threaded) [geom] 100025 D - 0xffffffff82505c70 [g_event] 100026 D - 0xffffffff82505c78 [g_up] 100027 D - 0xffffffff82505c80 [g_down] 2 0 0 0 DL (threaded) [KTLS] 100018 D - 0xfffff8000422ea00 [thr_0] 100019 D - 0xfffff8000422ea80 [thr_1] 12 0 0 0 RL (threaded) [intr] 100010 I [swi5: fast taskq] 100014 I [swi6: task queue] 100015 I [swi6: Giant taskq] 100020 I [swi3: vm] 100021 I [swi4: clock (0)] 100022 I [swi4: clock (1)] 100023 Run CPU 1 [swi1: netisr 0] 100035 I [irq24: virtio_pci0] 100036 I [irq25: virtio_pci0] 100037 I [irq26: virtio_pci0] 100038 I [irq27: virtio_pci0] 100039 I [irq28: virtio_pci1] 100040 I [irq29: virtio_pci1] 100041 I [irq30: virtio_pci1] 100042 I [irq31: virtio_pci1] 100043 I [irq32: virtio_pci1] 100048 I [irq10: virtio_pci2] 100050 I [irq1: atkbd0] 100051 I [irq12: psm0] 100052 I [swi0: uart uart++] 100060 I [swi1: pf send] 100073 I [swi1: hpts] 100074 I [swi1: hpts] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffff8000432b000 [init] 10 0 0 0 DL audit_w 0xffffffff8262c200 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff82506210 [swapper] 100005 D - 0xfffff8000422f000 [if_config_tqg_0] 100006 D - 0xfffff8000422ee00 [softirq_0] 100007 D - 0xfffff8000422ed00 [softirq_1] 100008 D - 0xfffff8000422ec00 [if_io_tqg_0] 100009 D - 0xfffff8000422eb00 [if_io_tqg_1] 100011 D - 0xfffff80004349600 [inm_free taskq] 100012 D - 0xfffff80004349500 [kqueue_ctx taskq] 100013 D - 0xfffff80004349400 [aiod_kick taskq] 100016 D - 0xfffff80004346e00 [in6m_free taskq] 100017 D - 0xfffff80004346d00 [thread taskq] 100024 D - 0xfffff80004346700 [firmware taskq] 100029 D - 0xfffff80004347b00 [crypto_0] 100030 D - 0xfffff80004347b00 [crypto_1] 100044 D - 0xfffff80004347000 [vtnet0 rxq 0] 100045 D - 0xfffff800047f2300 [vtnet0 txq 0] 100046 D - 0xfffff800047f2200 [vtnet0 rxq 1] 100047 D - 0xfffff800047f2100 [vtnet0 txq 1] 100049 D vtbslp 0xfffff80004518180 [virtio_balloon] 100053 D - 0xfffff800047f3900 [mca taskq] 100058 D - 0xffffffff81d0e800 [deadlkres] 100062 D - 0xfffff800047f3700 [acpi_task_0] 100063 D - 0xfffff800047f3700 [acpi_task_1] 100064 D - 0xfffff800047f3700 [acpi_task_2] 100066 D - 0xfffff80004347500 [CAM taskq] db> show all locks Process 3440 (syz-executor.1) thread 0xfffffe0025953700 (100681) exclusive sx so_snd_sx (so_snd_sx) r = 0 (0xfffff80004d8b650) locked @ /syzkaller/managers/i386/kernel/sys/kern/uipc_sockbuf.c:464 Process 3434 (syz-executor.0) thread 0xfffffe00257a2100 (100683) exclusive sleep mutex sctp-create (inp_create) r = 0 (0xfffff8001feb7488) locked @ /syzkaller/managers/i386/kernel/sys/netinet/sctp_usrreq.c:7002 Process 12 (intr) thread 0xfffffe00049cd800 (100023) exclusive sleep mutex sctp-tcb (tcb) r = 0 (0xfffffe00258638b0) locked @ /syzkaller/managers/i386/kernel/sys/netinet/sctp_pcb.c:2143 db> show malloc Type InUse MemUse Requests pf_hash 5 11524K 5 devbuf 4214 4854K 4242 tcp_hpts 5 3201K 5 vtbuf 24 1968K 46 sysctloid 28170 1644K 28234 kobj 336 1344K 496 newblk 62 1040K 9098 vfscache 4 1025K 4 inodedep 118 571K 3689 pcb 46 560K 8177 ufs_quota 1 512K 1 vfs_hash 1 512K 1 callout 2 512K 2 intr 4 472K 4 subproc 141 277K 3523 acpica 1674 185K 52444 vnet_data 1 168K 1 pagedep 14 132K 2497 tfo_ccache 1 128K 1 filedesc 18 121K 4970 sem 4 106K 4 DEVFS1 105 105K 122 linker 254 99K 297 bus 979 79K 3032 mtx_pool 2 72K 2 BPF 38 71K 46 syncache 1 68K 1 acpitask 1 64K 1 ddb_capture 1 64K 1 module 507 64K 507 umtx 342 43K 342 kdtrace 186 36K 10602 temp 35 33K 2437 hostcache 1 32K 1 shm 1 32K 4 DEVFS3 124 31K 134 msg 4 30K 4 ifaddr 78 27K 79 DEVFS_RULE 56 27K 56 gtaskqueue 18 26K 18 dirrem 98 25K 3273 sctp_timw 91 23K 91 vmem 3 22K 5 kbdmux 6 22K 6 lltable 46 17K 54 ufs_mount 5 17K 6 proc 3 17K 3 tty 16 16K 16 tidhash 1 16K 1 ithread 99 16K 99 ether_multi 172 14K 215 bus-sc 30 14K 1413 freefile 100 13K 3269 KTRACE 100 13K 100 ifnet 7 13K 7 kenv 92 12K 92 eventhandler 132 12K 132 in6_multi 89 11K 105 pfs_nodes 20 10K 20 GEOM 60 10K 488 rman 82 10K 423 sctp_atcl 18 9K 8311 bmsafemap 2 9K 3742 UART 12 9K 12 devstat 4 9K 4 rpc 2 8K 2 shmfd 1 8K 15 pfs_vncache 1 8K 1 freework 31 8K 5944 audit_evclass 233 8K 291 sctp_stro 6 6K 2070 CAM DEV 3 6K 510 kqueue 61 6K 3447 plimit 23 6K 448 cred 23 6K 261 vt 11 6K 11 sglist 5 6K 5 CAM queue 5 6K 1528 taskqueue 48 6K 48 select 40 5K 40 DEVFSP 76 5K 100 ufs_dirhash 24 5K 24 pf_ifnet 10 5K 19 UMA 251 5K 251 memdesc 1 4K 1 MCA 32 4K 32 evdev 4 4K 4 kcovinfo 64 4K 85 session 31 4K 51 pgrp 31 4K 53 routetbl 23 4K 33 hhook 13 4K 13 lockf 29 4K 88 acpisem 22 3K 22 terminal 11 3K 11 proc-args 50 3K 673 sctp_ifa 18 3K 22 uidinfo 3 3K 11 local_apic 1 2K 1 io_apic 1 2K 1 ipsec-saq 2 2K 2 ip6ndp 12 2K 23 Unitno 33 2K 5304 CAM XPT 22 2K 543 mkdir 12 2K 4944 freeblks 6 2K 3371 in_multi 6 2K 9 diradd 10 2K 3318 ipsecpolicy 2 2K 2 acpidev 20 2K 20 msi 9 2K 9 clone 9 2K 9 tun 7 2K 7 sctp_stri 2 1K 1509 softdep 1 1K 1 indirdep 4 1K 1279 sahead 1 1K 1 secasvar 1 1K 1 nhops 6 1K 8 vnodemarker 2 1K 44 NFSD session 1 1K 1 sctp_atky 23 1K 10606 CAM periph 4 1K 271 newdirblk 7 1K 2472 ipsec 3 1K 3 sctp_ifn 6 1K 22 mld 6 1K 6 igmp 6 1K 6 toponodes 6 1K 6 isadev 6 1K 6 mount 16 1K 89 pci_link 10 1K 10 encap_export_host 12 1K 12 crypto 3 1K 3 pfil 4 1K 4 chacha20random 1 1K 1 CAM SIM 2 1K 2 cdev 2 1K 2 inpcbpolicy 10 1K 2147 osd 3 1K 9 sctp_athm 18 1K 8313 vnodes 1 1K 1 ktls 1 1K 1 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 DEVFS 9 1K 10 feeder 7 1K 7 sctp_map 12 1K 4140 loginclass 3 1K 6 prison 6 1K 6 soname 6 1K 6523 CAM dev queue 2 1K 2 CAM I/O Scheduler 1 1K 1 apmdev 1 1K 1 atkbddev 2 1K 2 CAM path 4 1K 1034 tcpfunc 2 1K 2 pmchooks 1 1K 1 nexusdev 5 1K 5 sctp_vrf 1 1K 1 entropy 2 1K 45 vnet 1 1K 1 acpiintr 1 1K 1 pmc 1 1K 1 filecaps 4 1K 112 cpus 2 1K 2 vnet_data_free 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 pf_table 0 0K 2 pf_rule 0 0K 2 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_temp 0 0K 0 sctp_mcore 0 0K 0 sctp_socko 0 0K 1543 sctp_iter 0 0K 13 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 0 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 13 sctp_aadr 0 0K 102 vtfont 0 0K 0 madt_table 0 0K 2 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 amr 0 0K 0 scsi_da 0 0K 69 pvscsi 0 0K 0 smartpqi 0 0K 0 ata_da 0 0K 0 scsi_ch 0 0K 0 scsi_cd 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 AHCI driver 0 0K 0 agp 0 0K 0 iavf 0 0K 0 ixl 0 0K 0 nvme_da 0 0K 0 acpipwr 0 0K 0 twsbuf 0 0K 0 twe_commands 0 0K 0 twa_commands 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 axgbe 0 0K 0 fpukern_ctx 0 0K 0 tcp_log_dev 0 0K 0 xen_intr 0 0K 0 midi buffers 0 0K 0 mixer 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 qpidrv 0 0K 0 ac97 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 acpi_perf 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 vm_fictitious 0 0K 0 CAM CCB 0 0K 5547 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 tempbuff 0 0K 0 UMAHash 0 0K 0 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 229 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 19 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 495 allocindir 0 0K 0 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 xform 0 0K 0 lDevFlags * malloc 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 8 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 LRO 0 0K 0 newreno data 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 0 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 0 ag_portal_data_t malloc 0 0K 0 statfs 0 0K 2653 export_host 0 0K 0 cl_savebuf 0 0K 239 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 biobuf 0 0K 0 aios 0 0K 0 lio 0 0K 0 acl 0 0K 0 MPSSAS 0 0K 0 mbuf_tag 0 0K 165 accf 0 0K 0 pts 0 0K 0 iov 0 0K 17104 ioctlops 0 0K 186 Witness 0 0K 0 stack 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 md_sectors 0 0K 0 sbuf 0 0K 288 md_disk 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 malodev 0 0K 0 SWAP 0 0K 0 LED 0 0K 0 sysctltmp 0 0K 701 sysctl 0 0K 1 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 431 rctl 0 0K 0 ix_sriov 0 0K 0 aacraidcam 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 iirbuf 0 0K 0 cache 0 0K 0 aacraid_buf 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 5 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroffdiroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xenstore 0 0K 0 scsi_pass 0 0K 0 ciss_data 0 0K 0 xnb 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 sysmouse 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_cluster 2048 9780 126 9780 0 254 20287488 0 mbuf_packet 256 8212 1440 1104928 0 254 2470912 0 512 512 4243 53 8453 0 30 2199552 0 mbuf_jumbo_page 4096 0 508 196 0 254 2080768 0 BUF TRIE 144 195 13273 5933 0 62 1939392 0 RADIX NODE 144 12284 257 139441 0 62 1805904 0 4096 4096 337 3 503 0 2 1392640 0 128 128 9341 114 12673 0 126 1210240 0 pbuf 832 0 969 0 0 2 806208 0 UMA Slabs 0 112 7060 26 7060 0 126 793632 0 sctp_ep 1280 11 499 3984 0 254 652800 0 sctp_asoc 2288 5 250 2070 0 254 583440 0 sctp_raddr 736 15 755 9587 0 254 566720 0 tcpcb 1040 4 513 146 0 254 537680 0 socket 960 44 464 10014 0 254 487680 0 65536 65536 6 0 6 0 1 393216 0 256 Bucket 2048 174 10 1888 0 8 376832 0 VM OBJECT 264 1234 116 52526 0 30 356400 0 VNODE 496 558 106 3830 0 30 329344 0 mbuf 256 509 774 611935 0 254 328448 0 4096 4096 66 10 3450 0 2 311296 0 THREAD 1792 147 24 3799 0 8 306432 0 ripcb 488 4 508 75 0 254 249856 0 udplite_inpcb 488 0 512 1697 0 254 249856 0 tcp_inpcb 488 4 508 146 0 254 249856 0 udp_inpcb 488 2 510 229 0 254 249856 0 DEVCTL 1024 2 206 127 0 0 212992 0 16384 16384 7 6 2483 0 1 212992 0 64 64 2795 292 3054 0 254 197568 0 65536 65536 1 2 135 0 1 196608 0 16 16 11839 411 11907 0 254 196000 0 256 256 245 505 9943 0 62 192000 0 MAP ENTRY 96 1494 396 184098 0 126 181440 0 128 128 1197 198 24741 0 126 178560 0 UMA Zones 768 226 1 226 0 16 174336 0 32 32 4708 458 4787 0 254 165312 0 32768 32768 0 5 47 0 1 163840 0 1024 1024 131 13 148 0 16 147456 0 128 128 1075 72 5468 0 126 146816 0 FFS2 dinode 256 521 49 3791 0 62 145920 0 65536 65536 0 2 183 0 1 131072 0 32768 32768 2 2 5 0 1 131072 0 unpcb 256 21 489 3844 0 254 130560 0 VMSPACE 2536 36 12 3425 0 4 121728 0 sctp_chunk 152 12 768 7516 0 254 118560 0 ksiginfo 112 64 980 1350 0 126 116928 0 vmem btag 56 1965 111 1965 0 254 116256 0 FFS inode 160 521 179 3791 0 62 112000 0 PROC 1312 58 23 3440 0 8 106272 0 S VFS Cache 104 620 394 5239 0 126 105456 0 128 Bucket 1024 54 45 453 0 16 101376 0 256 256 165 210 7739 0 62 96000 0 512 512 148 36 158 0 30 94208 0 filedesc0 1080 59 25 3441 0 8 90720 0 64 64 578 745 8660 0 254 84672 0 128 128 366 285 7942 0 126 83328 0 UMA Kegs 384 212 4 212 0 30 82944 0 2048 2048 0 40 5579 0 8 81920 0 2048 2048 16 24 3990 0 8 81920 0 64 Bucket 512 80 80 2980 0 30 81920 0 256 256 193 122 3555 0 62 80640 0 sctp_readq 152 0 520 54 0 254 79040 0 4096 4096 17 1 21 0 2 73728 0 g_bio 408 0 180 19842 0 30 73440 0 64 64 540 531 1088 0 254 68544 0 65536 65536 1 0 1 0 1 65536 0 65536 65536 1 0 1 0 1 65536 0 65536 65536 0 1 8 0 1 65536 0 4096 4096 14 2 26 0 2 65536 0 256 256 116 139 13621 0 62 65280 0 512 512 31 89 8324 0 30 61440 0 sctp_stream_msg_out 112 0 540 6 0 254 60480 0 128 128 116 287 3940 0 126 51584 0 256 256 94 101 232 0 62 49920 0 256 256 31 164 1625 0 62 49920 0 256 256 28 167 4455 0 62 49920 0 32 Bucket 256 50 145 863 0 62 49920 0 DIRHASH 1024 34 14 34 0 16 49152 0 NAMEI 1024 0 48 28345 0 16 49152 0 16384 16384 3 0 3 0 1 49152 0 8192 8192 3 3 31 0 1 49152 0 8192 8192 6 0 6 0 1 49152 0 8192 8192 5 1 7 0 1 49152 0 2048 2048 10 14 16 0 8 49152 0 2048 2048 5 19 512 0 8 49152 0 1024 1024 8 40 2082 0 16 49152 0 512 512 56 40 733 0 30 49152 0 512 512 3 93 1508 0 30 49152 0 syncache 168 0 264 4 0 254 44352 0 Files 80 229 321 21040 0 126 44000 0 clpbuf 832 0 48 1039 0 16 39936 0 pipe 760 22 28 435 0 16 38000 0 64 pcpu 8 4199 409 4260 0 254 36864 0 selfd 64 81 486 22134 0 254 36288 0 PWD 32 19 1115 2560 0 254 36288 0 64 64 50 517 132 0 254 36288 0 64 64 33 534 15443 0 254 36288 0 64 64 86 481 3614 0 254 36288 0 64 64 201 366 461 0 254 36288 0 64 64 169 398 198 0 254 36288 0 128 128 65 214 122 0 126 35712 0 128 128 29 250 199 0 126 35712 0 128 128 54 225 310 0 126 35712 0 routing nhops 256 27 108 44 0 62 34560 0 ttyoutq 256 72 63 160 0 62 34560 0 256 256 46 89 2956 0 62 34560 0 TURNSTILE 136 172 80 172 0 62 34272 0 SLEEPQUEUE 88 172 212 172 0 126 33792 0 32768 32768 0 1 112 0 1 32768 0 32768 32768 1 0 1 0 1 32768 0 16384 16384 2 0 5 0 1 32768 0 2048 2048 2 14 27 0 8 32768 0 2048 2048 3 13 24 0 8 32768 0 2048 2048 3 13 196 0 8 32768 0 2048 2048 3 13 259 0 8 32768 0 1024 1024 3 29 1296 0 16 32768 0 1024 1024 3 29 3 0 16 32768 0 1024 1024 4 28 137 0 16 32768 0 1024 1024 11 21 11 0 16 32768 0 1024 1024 1 31 30 0 16 32768 0 512 512 1 63 61 0 30 32768 0 512 512 30 34 341 0 30 32768 0 mt_stats_zone 64 446 66 446 0 254 32768 0 16 Bucket 144 57 167 8326 0 62 32256 0 KNOTE 160 28 172 63912 0 62 32000 0 ttyinq 160 135 65 300 0 62 32000 0 cpuset 104 7 272 7 0 126 29016 0 sctp_asconf_ack 48 0 588 35 0 254 28224 0 sctp_laddr 48 8 580 58 0 254 28224 0 hostcache 96 1 293 1 0 254 28224 0 32 32 110 772 12581 0 254 28224 0 4 Bucket 48 6 582 15056 0 254 28224 0 8 Bucket 80 73 277 20803 0 126 28000 0 KMAP ENTRY 96 12 279 12 0 126 27936 0 8192 8192 3 0 3 0 1 24576 0 4096 4096 5 1 7 0 2 24576 0 4096 4096 2 4 3104 0 2 24576 0 4096 4096 6 0 7 0 2 24576 0 rtentry 176 34 104 44 0 62 24288 0 rl_entry 40 83 523 83 0 254 24240 0 vtnet_tx_hdr 24 0 1002 543442 0 254 24048 0 Mountpoints 2816 2 6 2 0 4 22528 0 udpcb 32 2 628 1926 0 254 20160 0 32 32 27 603 5345 0 254 20160 0 32 32 20 610 47 0 254 20160 0 32 32 74 556 3010 0 254 20160 0 32 32 32 598 270 0 254 20160 0 2 Bucket 32 64 566 5068 0 254 20160 0 tcp_bbr_map 128 0 155 6 0 126 19840 0 procdesc 136 4 141 18 0 62 19720 0 epoch_record pcpu 256 4 60 4 0 62 16384 0 16384 16384 1 0 1 0 1 16384 0 16384 16384 0 1 136 0 1 16384 0 16384 16384 1 0 1 0 1 16384 0 8192 8192 1 1 90 0 1 16384 0 1024 1024 11 5 12 0 16 16384 0 512 512 1 31 2 0 30 16384 0 SMR CPU 32 2 509 2 0 254 16352 0 pf tags 104 0 156 4 0 126 16224 0 rentr 24 0 668 8 0 254 16032 0 mt_zone 24 446 222 446 0 254 16032 0 16 16 95 905 11930 0 254 16000 0 16 16 208 792 6333 0 254 16000 0 VNODEPOLL 120 0 132 1 0 126 15840 0 kenv 258 3 57 1012 0 30 15480 0 tcp_bbr_pcb 832 0 18 2 0 16 14976 0 FPU_save_area 832 1 17 1 0 16 14976 0 vmem 1856 1 7 1 0 8 14848 0 SMR SHARED 24 2 509 2 0 254 12264 0 32 32 126 252 1345 0 254 12096 0 32 32 14 364 19 0 254 12096 0 16 16 10 740 50 0 254 12000 0 16 16 6 744 545 0 254 12000 0 16 16 20 730 155 0 254 12000 0 16 16 265 485 24396 0 254 12000 0 8192 8192 1 0 1 0 1 8192 0 8192 8192 1 0 1 0 1 8192 0 16 16 4 496 6 0 254 8000 0 UMA Slabs 1 176 9 13 9 0 62 3872 0 int pcpu 4 34 478 34 0 254 2048 0 FFS1 dinode 128 0 0 0 0 126 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 sctp_asconf 40 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 62 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 296 0 0 0 0 254 0 0 pf mtags 48 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 tcp_log_node 120 0 0 0 0 126 0 0 tcp_log_bucket 176 0 0 0 0 62 0 0 tcp_log 416 0 0 0 0 254 0 0 tcpreass 48 0 0 0 0 254 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 88 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 itimer 352 0 0 0 0 30 0 0 AIOLIO 280 0 0 0 0 30 0 0 AIOCB 752 0 0 0 0 16 0 0 AIOP 32 0 0 0 0 254 0 0 AIO 208 0 0 0 0 62 0 0 NCLNODE 592 0 0 0 0 16 0 0 TMPFS node 232 0 0 0 0 62 0 0 TMPFS dirent 64 0 0 0 0 254 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 crypto_session 72 0 0 0 0 126 0 0 cryptop 280 0 0 0 0 30 0 0 nvme_request 128 0 0 0 0 126 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 ktls_session 192 0 0 0 0 62 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 domainset 40 0 0 0 0 254 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 832 0 0 0 0 62 0 0 mdpbuf 832 0 0 0 0 4 0 0 nfspbuf 832 0 0 0 0 16 0 0 swwbuf 832 0 0 0 0 8 0 0 swrbuf 832 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 65536 65536 0 0 0 0 1 0 0 65536 65536 0 0 0 0 1 0 0 32768 32768 0 0 0 0 1 0 0 32768 32768 0 0 0 0 1 0 0 32768 32768 0 0 0 0 1 0 0 32768 32768 0 0 0 0 1 0 0 16384 16384 0 0 0 0 1 0 0 16384 16384 0 0 0 0 1 0 0 8192 8192 0 0 0 0 1 0 0 4096 4096 0 0 0 0 2 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0