panic: sctp_inpcb_free: inp 0xfffffe009b2a6a78 still has socket cpuid = 0 time = 1651282420 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0xc7/frame 0xfffffe009b5172f0 kdb_backtrace() at kdb_backtrace+0xd3/frame 0xfffffe009b517450 vpanic() at vpanic+0x2b8/frame 0xfffffe009b517530 panic() at panic+0xb5/frame 0xfffffe009b517600 sctp_inpcb_free() at sctp_inpcb_free+0x1fd3/frame 0xfffffe009b517690 sctp_close() at sctp_close+0x1fb/frame 0xfffffe009b517770 soclose() at soclose+0x3fc/frame 0xfffffe009b517880 _fdrop() at _fdrop+0x58/frame 0xfffffe009b5178b0 closef() at closef+0x689/frame 0xfffffe009b517a90 fdescfree() at fdescfree+0xaa4/frame 0xfffffe009b517c70 exit1() at exit1+0x882/frame 0xfffffe009b517d10 sys_exit() at sys_exit+0x28/frame 0xfffffe009b517d30 amd64_syscall() at amd64_syscall+0x40c/frame 0xfffffe009b517f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe009b517f30 --- syscall (1, FreeBSD ELF64, sys_exit), rip = 0x2ebdaa, rsp = 0x8213306a8, rbp = 0x8213306b0 --- KDB: enter: panic [ thread pid 77444 tid 186676 ] Stopped at kdb_enter+0x6b: movq $0,0x27048aa(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0x28 rax 0x12 rcx 0x60924fa7b0b35fa5 rdx 0x1 rbx 0 rsp 0xfffffe009b517430 rbp 0xfffffe009b517450 rsi 0 rdi 0xffffffff817825fa vprintf+0x35a r8 0 r9 0xffffffff r10 0 r11 0 r12 0xfffffe009b8351e0 r13 0xfffffe009b517401 r14 0xffffffff82bc9b00 .str.26 r15 0xffffffff82bc9b00 .str.26 rip 0xffffffff81775b6b kdb_enter+0x6b rflags 0x46 kdb_enter+0x6b: movq $0,0x27048aa(%rip) db> show proc Process 77444 (syz-executor.3) at 0xfffffe009b84da90: state: NORMAL uid: 0 gids: 0, 0, 5 parent: pid 781 at 0xfffffe0058d7da90 ABI: FreeBSD ELF64 flag: 0x10002000 flag2: 0 arguments: /root/syz-executor.3 exec reaper: 0xfffffe0053df0000 reapsubtree: 1 sigparent: 20 vmspace: 0xfffffe009e492000 (map 0xfffffe009e492000) (map.pmap 0xfffffe009e4920c0) (pmap 0xfffffe009e492128) threads: 1 186676 Run CPU 0 syz-executor.3 db> ps pid ppid pgrp uid state wmesg wchan cmd 77444 781 781 0 RE CPU 0 syz-executor.3 77443 782 782 0 R syz-executor.1 77439 780 780 0 R (threaded) syz-executor.2 186172 RunQ syz-executor.2 186895 RunQ syz-executor.2 77438 779 779 0 RE syz-executor.0 75690 1 781 0 SV uwait 0xfffffe0057895900 syz-executor.3 75689 1 781 0 SV uwait 0xfffffe0099bdb300 syz-executor.3 75680 1 781 0 SV uwait 0xfffffe009b863d00 syz-executor.3 75679 1 781 0 SV uwait 0xfffffe009b863c00 syz-executor.3 75669 1 781 0 SV uwait 0xfffffe009b863f00 syz-executor.3 73630 1 781 0 SV uwait 0xfffffe0099bda880 syz-executor.3 73603 0 0 0 DL - 0xffffffff83f677c0 [soaiod4] 73602 0 0 0 DL - 0xffffffff83f677c0 [soaiod3] 73601 0 0 0 DL - 0xffffffff83f677c0 [soaiod2] 73600 0 0 0 DL - 0xffffffff83f677c0 [soaiod1] 61528 1 782 0 SV uwait 0xfffffe0057892a00 syz-executor.1 61520 1 782 0 SV uwait 0xfffffe0056fe4980 syz-executor.1 61510 1 782 0 SV uwait 0xfffffe0057892e80 syz-executor.1 51514 1 779 0 S uwait 0xfffffe0057895c00 syz-executor.0 51512 1 782 0 S uwait 0xfffffe0057895f00 syz-executor.1 42531 1 42528 0 S uwait 0xfffffe0099bdab80 syz-executor.1 42524 1 42521 0 S uwait 0xfffffe0099bdb000 syz-executor.1 42517 1 42516 0 S uwait 0xfffffe0056fe4c80 syz-executor.1 32295 1 32290 0 S uwait 0xfffffe0099bdb980 syz-executor.2 29884 1 780 0 SV uwait 0xfffffe0099bdb200 syz-executor.2 29874 1 780 0 SV uwait 0xfffffe0056fe4b80 syz-executor.2 29865 1 780 0 SV uwait 0xfffffe0057893800 syz-executor.2 29857 1 780 0 SV uwait 0xfffffe0057893000 syz-executor.2 29848 1 780 0 SV uwait 0xfffffe0057892280 syz-executor.2 29841 1 780 0 SV uwait 0xfffffe0099bdae80 syz-executor.2 29831 1 780 0 SV uwait 0xfffffe0057892c80 syz-executor.2 29821 1 780 0 SV uwait 0xfffffe0057892480 syz-executor.2 29811 1 780 0 SV uwait 0xfffffe0099bdb100 syz-executor.2 29802 1 780 0 SV uwait 0xfffffe0057893b00 syz-executor.2 29793 1 780 0 SV uwait 0xfffffe0099bdac80 syz-executor.2 29784 1 780 0 SV uwait 0xfffffe0053df9b00 syz-executor.2 29779 1 780 0 SV uwait 0xfffffe0099bdad80 syz-executor.2 29766 1 780 0 SV uwait 0xfffffe0056fe2800 syz-executor.2 29757 1 780 0 SV uwait 0xfffffe0057893400 syz-executor.2 29753 1 780 0 SV uwait 0xfffffe0057893900 syz-executor.2 29744 1 780 0 SV uwait 0xfffffe0057893500 syz-executor.2 29735 1 780 0 SV uwait 0xfffffe0057892580 syz-executor.2 29725 1 780 0 SV uwait 0xfffffe0057892380 syz-executor.2 29713 1 780 0 SV uwait 0xfffffe0058c88780 syz-executor.2 29704 1 780 0 SV uwait 0xfffffe0057894600 syz-executor.2 29695 1 780 0 SV uwait 0xfffffe0057892680 syz-executor.2 29682 1 782 0 SV uwait 0xfffffe0057892780 syz-executor.1 29676 29672 782 0 SV uwait 0xfffffe0056fe4880 syz-executor.1 29672 1 782 0 DV ppwait 0xfffffe0099bf9f88 syz-executor.1 29665 1 782 0 SV uwait 0xfffffe0057892d80 syz-executor.1 13793 1 13793 65 Ss select 0xfffffe0058b3de40 dhclient 11082 1 11082 0 Ss select 0xfffffe0058b3f6c0 dhclient 11079 1 11079 0 Ss select 0xfffffe009238a940 dhclient 11060 1 11060 65 Ss select 0xfffffe0058b3f040 dhclient 8387 1 8387 0 Ss select 0xfffffe009238a140 dhclient 8380 1 8380 0 Ss select 0xfffffe009238be40 dhclient 8328 1 8328 65 Ss select 0xfffffe009238a740 dhclient 4723 1 4723 0 Ss select 0xfffffe0058b3ee40 dhclient 4720 1 4720 0 Ss select 0xfffffe0058b3f140 dhclient 4701 1 4701 65 Ss select 0xfffffe0058b3eac0 dhclient 2474 0 0 0 DL aiordy 0xfffffe0092e95000 [aiod4] 2472 0 0 0 DL aiordy 0xfffffe0092e48000 [aiod3] 2471 0 0 0 DL aiordy 0xfffffe0058c68548 [aiod2] 2470 0 0 0 DL aiordy 0xfffffe0058c68000 [aiod1] 1277 1 1277 0 Ss select 0xfffffe0058b3edc0 dhclient 1269 1 1269 0 Ss select 0xfffffe009238b1c0 dhclient 782 774 782 0 Rs syz-executor.1 781 774 781 0 Ss nanslp 0xffffffff83e47a81 syz-executor.3 780 774 780 0 Rs syz-executor.2 779 774 779 0 Rs syz-executor.0 774 772 772 0 S (threaded) syz-fuzzer 100114 S uwait 0xfffffe0057894900 syz-fuzzer 100115 S uwait 0xfffffe0057894a00 syz-fuzzer 100116 S uwait 0xfffffe0057894b00 syz-fuzzer 100117 S uwait 0xfffffe0057893c00 syz-fuzzer 100118 S uwait 0xfffffe0057893d00 syz-fuzzer 100119 S uwait 0xfffffe0057893e00 syz-fuzzer 100120 S uwait 0xfffffe0057893f00 syz-fuzzer 100121 S uwait 0xfffffe0053df9900 syz-fuzzer 100122 S kqread 0xfffffe000818f700 syz-fuzzer 772 770 772 0 Ss pause 0xfffffe0058c690b0 csh 770 688 770 0 Ss select 0xfffffe009238b640 sshd 756 1 756 0 Ss+ ttyin 0xfffffe00574774b0 getty 755 1 755 0 Ss+ ttyin 0xfffffe00586ef0b0 getty 754 1 754 0 Ss+ ttyin 0xfffffe00586ef8b0 getty 753 1 753 0 Ss+ ttyin 0xfffffe005748d0b0 getty 752 1 752 0 Ss+ ttyin 0xfffffe005748d8b0 getty 751 1 751 0 Ss+ ttyin 0xfffffe005748e0b0 getty 750 1 750 0 Ss+ ttyin 0xfffffe005748e8b0 getty 749 1 749 0 Ss+ ttyin 0xfffffe0053f41cb0 getty 748 1 748 0 Ss+ ttyin 0xfffffe0053f424b0 getty 692 1 692 0 Ss nanslp 0xffffffff83e47a81 cron 688 1 688 0 Ss select 0xfffffe0058b3f7c0 sshd 501 1 501 0 Ss select 0xfffffe009238b7c0 syslogd 430 1 430 0 Ss select 0xfffffe009238bac0 devd 429 1 429 65 Ss select 0xfffffe009238b8c0 dhclient 344 1 344 0 Ss select 0xfffffe009238b9c0 dhclient 341 1 341 0 Ss select 0xfffffe0058b3f940 dhclient 17 0 0 0 DL vlruwt 0xfffffe0056f9e548 [vnlru] 16 0 0 0 DL syncer 0xffffffff83f6d2e0 [syncer] 15 0 0 0 DL (threaded) [bufdaemon] 100079 D psleep 0xffffffff83f6b8e0 [bufdaemon] 100082 D - 0xffffffff83211f80 [bufspacedaemon-0] 100091 D sdflush 0xfffffe0058c99ce8 [/ worker] 9 0 0 0 DL psleep 0xffffffff83f9f400 [vmdaemon] 8 0 0 0 DL (threaded) [pagedaemon] 100077 D psleep 0xffffffff83f932b8 [dom0] 100080 D launds 0xffffffff83f932c4 [laundry: dom0] 100081 D umarcl 0xffffffff81eb1ca0 [uma] 7 0 0 0 DL - 0xffffffff83c03788 [rand_harvestq] 6 0 0 0 DL pftm 0xffffffff849c5530 [pf purge] 5 0 0 0 DL waiting 0xffffffff847f84a0 [sctp_iterator] 4 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff83aa56c0 [doneq0] 100045 D - 0xffffffff83aa5640 [async] 100076 D - 0xffffffff83aa54c0 [scanner] 14 0 0 0 DL seqstat 0xfffffe0056f01c88 [sequencer 00] 3 0 0 0 DL (threaded) [crypto] 100040 D crypto_ 0xffffffff83f8eac0 [crypto] 100041 D crypto_ 0xfffffe0053f73030 [crypto returns 0] 100042 D crypto_ 0xfffffe0053f73080 [crypto returns 1] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff83e1d000 [g_event] 100036 D - 0xffffffff83e1d020 [g_up] 100037 D - 0xffffffff83e1d040 [g_down] 2 0 0 0 WL (threaded) [clock] 100029 I [clock (0)] 100030 I [clock (1)] 12 0 0 0 RL (threaded) [intr] 100015 I [swi5: fast taskq] 100018 I [swi6: task queue] 100019 I [swi6: Giant taskq] 100031 Run CPU 1 [swi1: netisr 0] 100032 I [swi1: hpts] 100033 I [swi1: hpts] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq33: virtio_pci2] 100060 I [irq34: virtio_pci2] 100061 I [irq35: virtio_pci2] 100063 I [irq1: atkbd0] 100064 I [irq12: psm0] 100065 I [swi0: uart uart++] 100069 I [swi1: pf send] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffffe0053df0000 [init] 10 0 0 0 DL audit_w 0xffffffff83f8f5c0 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff83e1da40 [swapper] 100005 D - 0xfffffe0008191000 [if_config_tqg_0] 100006 D - 0xfffffe0008190e00 [softirq_0] 100007 D - 0xfffffe0008190d00 [softirq_1] 100008 D - 0xfffffe0008190c00 [if_io_tqg_0] 100009 D - 0xfffffe0008190b00 [if_io_tqg_1] 100010 D - 0xfffffe0008190a00 [inm_free taskq] 100011 D - 0xfffffe0008190900 [linuxkpi_irq_wq] 100012 D - 0xfffffe0008190800 [in6m_free taskq] 100013 D - 0xfffffe0008190700 [deferred_unmount ta] 100014 D - 0xfffffe0008190600 [thread taskq] 100016 D - 0xfffffe0008190400 [kqueue_ctx taskq] 100017 D - 0xfffffe0008190300 [pci_hp taskq] 100020 D - 0xfffffe0008190000 [aiod_kick taskq] 100021 D - 0xfffffe000818fe00 [linuxkpi_short_wq_0] 100022 D - 0xfffffe000818fe00 [linuxkpi_short_wq_1] 100023 D - 0xfffffe000818fe00 [linuxkpi_short_wq_2] 100024 D - 0xfffffe000818fe00 [linuxkpi_short_wq_3] 100025 D - 0xfffffe000818fd00 [linuxkpi_long_wq_0] 100026 D - 0xfffffe000818fd00 [linuxkpi_long_wq_1] 100027 D - 0xfffffe000818fd00 [linuxkpi_long_wq_2] 100028 D - 0xfffffe000818fd00 [linuxkpi_long_wq_3] 100034 D - 0xfffffe000818fc00 [firmware taskq] 100038 D - 0xfffffe000818f200 [crypto_0] 100039 D - 0xfffffe000818f200 [crypto_1] 100055 D - 0xfffffe000818e900 [vtnet0 rxq 0] 100056 D - 0xfffffe000818e800 [vtnet0 txq 0] 100057 D - 0xfffffe000818e700 [vtnet0 rxq 1] 100058 D - 0xfffffe000818e600 [vtnet0 txq 1] 100062 D vtbslp 0xfffffe0056f4be00 [virtio_balloon] 100066 D - 0xffffffff82bcf980 [deadlkres] 100070 D - 0xfffffe0008191100 [mca taskq] 100072 D - 0xfffffe000818eb00 [acpi_task_0] 100073 D - 0xfffffe000818eb00 [acpi_task_1] 100074 D - 0xfffffe000818eb00 [acpi_task_2] 100075 D - 0xfffffe000818ea00 [CAM taskq] db> show all locks Process 77444 (syz-executor.3) thread 0xfffffe009b8351e0 (186676) exclusive sleep mutex sctp-inp (inp) r = 0 (0xfffffe009b2a6e98) locked @ /syzkaller/managers/main/kernel/sys/netinet/sctp_pcb.c:3340 exclusive rw sctp-info (sctp-info) r = 0 (0xfffffe0053c40f38) locked @ /syzkaller/managers/main/kernel/sys/netinet/sctp_pcb.c:3339 exclusive sleep mutex sctp-create (inp_create) r = 0 (0xfffffe009b2a6eb8) locked @ /syzkaller/managers/main/kernel/sys/netinet/sctp_pcb.c:3338 Process 12 (intr) thread 0xfffffe0053ea8560 (100031) exclusive sleep mutex sctp-tcb (tcb) r = 0 (0xfffffe009aefbbf0) locked @ /syzkaller/managers/main/kernel/sys/netinet/sctp_pcb.c:2138 db> show malloc Type InUse MemUse Requests pf_hash 5 11524K 5 pcb 4013 5026K 370609 tcp_hpts 7 4801K 7 devbuf 4217 4323K 4245 sysctloid 35322 2081K 35393 vtbuf 24 1968K 46 sctp_stro 1960 1960K 45144 sctp_atcl 3959 1485K 236119 kobj 328 1312K 489 newblk 12 1027K 79093 vfscache 3 1025K 3 filedesc 98 778K 152787 inodedep 504 701K 78262 linker 380 572K 437 ufs_quota 1 512K 1 vfs_hash 1 512K 1 callout 2 512K 2 intr 4 472K 4 subproc 227 470K 77569 vmem 3 274K 6 sctp_atky 5925 247K 285733 acpica 1674 184K 57552 vnet_data 1 168K 1 tidhash 3 141K 3 pagedep 13 131K 76404 tfo_ccache 1 128K 1 dirrem 491 123K 78126 DEVFS1 109 109K 126 sem 4 106K 4 BPF 46 88K 334 bus 994 81K 5207 sctp_timw 319 80K 319 mtx_pool 2 72K 2 syncache 1 68K 1 module 514 65K 514 acpitask 1 64K 1 ddb_capture 1 64K 1 sctp_athm 3959 62K 237541 sctp_map 3920 62K 90288 freefile 489 62K 78083 umtx 462 58K 462 kdtrace 305 57K 164342 temp 34 33K 3456 DEVFS3 128 32K 138 hostcache 1 32K 1 shm 1 32K 40 msg 4 30K 4 gtaskqueue 18 26K 18 kbdmux 6 22K 6 ifaddr 70 20K 72 DEVFS_RULE 56 20K 56 ufs_mount 4 17K 5 proc 3 17K 3 tty 16 16K 16 routetbl 130 16K 530 ithread 97 16K 97 lltable 46 15K 285 bus-sc 34 15K 1681 KTRACE 100 13K 1991 ifnet 7 13K 7 ether_multi 152 13K 162 kenv 95 12K 95 eventhandler 134 12K 134 rman 88 11K 431 GEOM 61 11K 490 CAM queue 5 11K 1528 cred 38 10K 708 kqueue 108 9K 77601 in6_multi 65 9K 65 bmsafemap 2 9K 78221 ksem 2 9K 154 UART 12 9K 12 devstat 4 9K 4 rpc 2 8K 2 shmfd 1 8K 85 pfs_vncache 1 8K 1 pfs_nodes 20 8K 20 audit_evclass 237 8K 296 taskqueue 63 7K 63 pwddesc 104 7K 77446 sglist 5 7K 5 CAM DEV 3 6K 510 plimit 24 6K 571 DEVFSP 91 6K 1020 session 38 5K 190 ufs_dirhash 24 5K 24 pf_ifnet 11 5K 47 UMA 272 5K 272 vt 11 5K 11 proc-args 133 5K 78949 memdesc 1 4K 1 MCA 32 4K 32 evdev 4 4K 4 lockf 33 4K 89 acpisem 28 4K 28 selfd 55 4K 1026293 hhook 15 4K 17 kcovinfo 52 4K 52 terminal 11 3K 11 ip6opt 10 3K 216 select 19 3K 171 uidinfo 4 3K 43 sctp_stri 4 2K 7160 local_apic 1 2K 1 io_apic 1 2K 1 fpukern_ctx 2 2K 2 ipsec-saq 2 2K 2 ip6ndp 12 2K 13 Unitno 30 2K 47 sctp_ifa 13 2K 14 CAM XPT 22 2K 543 cryptodev 25 2K 802 msi 12 2K 12 in_multi 6 2K 8 ipsecpolicy 2 2K 2 acpidev 20 2K 20 clone 9 2K 9 tun 7 2K 7 softdep 1 1K 1 freework 4 1K 77509 mkdir 8 1K 152764 sahead 1 1K 1 secasvar 1 1K 1 nhops 6 1K 8 vnodemarker 2 1K 140 NFSD session 1 1K 1 newdirblk 7 1K 76382 diradd 7 1K 78168 CAM periph 4 1K 271 sctp_ifn 6 1K 14 ipsec 3 1K 3 freeblks 3 1K 77479 mld 6 1K 6 igmp 6 1K 6 toponodes 6 1K 6 isadev 6 1K 6 mount 16 1K 113 pci_link 10 1K 10 crypto 4 1K 224 encap_export_host 12 1K 12 procdesc 5 1K 18 inpcbpolicy 16 1K 12368 pfil 4 1K 4 cdev 2 1K 2 osd 9 1K 9154 chacha20random 1 1K 1 CC Mem 5 1K 9186 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 DEVFS 9 1K 10 vnodes 1 1K 1 CAM SIM 2 1K 2 feeder 7 1K 7 tcpfunc 3 1K 3 loginclass 3 1K 6 prison 6 1K 6 lkpikmalloc 5 1K 6 aesni_data 2 1K 2 pf_rule 1 1K 102 nexusdev 8 1K 8 apmdev 1 1K 1 atkbddev 2 1K 2 CAM dev queue 2 1K 2 CAM I/O Scheduler 1 1K 1 CAM path 4 1K 1034 pmchooks 1 1K 1 soname 4 1K 132469 filecaps 5 1K 127 sctp_vrf 1 1K 1 vnet 1 1K 1 entropy 2 1K 49 pmc 1 1K 1 acpiintr 1 1K 1 sigio 1 1K 4 cpus 2 1K 2 vnet_data_free 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 ext2_mount 0 0K 0 ext2_node 0 0K 0 ext2_extents 0 0K 0 mqdata 0 0K 0 tcp_do 0 0K 0 tcp_fsb 0 0K 3060 filemon 0 0K 192 pf_table 0 0K 30 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_temp 0 0K 0 sctp_mcore 0 0K 0 sctp_socko 0 0K 100012 sctp_iter 0 0K 60 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 48 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 12 sctp_aadr 0 0K 82 ipcomp 0 0K 0 esp 0 0K 0 ah 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 ixl 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 NFSCL lckown 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 iavf 0 0K 0 axgbe 0 0K 0 NFSCL client 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 NFSD V4lock 0 0K 0 xen_intr 0 0K 0 NFSD V4state 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 bounce 0 0K 0 busdma 0 0K 0 qpidrv 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 DEVFS4 0 0K 0 vm_fictitious 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xenstore 0 0K 0 scsi_pass 0 0K 0 ciss_data 0 0K 0 xnb 0 0K 0 xen_acpi 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 UMAHash 0 0K 0 Balloon 0 0K 0 sysmouse 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 63819 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 67 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 306 allocindir 0 0K 0 indirdep 0 0K 30 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 vtfont 0 0K 0 BACKLIGHT 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 pvscsi 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 scsi_cd 0 0K 0 ktls_ocf 0 0K 0 AHCI driver 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS_RX 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 lkpiskb 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpindev 0 0K 0 lkpifw 0 0K 0 lkpi80211 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6_msource 0 0K 0 ip6_moptions 0 0K 3 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 LRO 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 2 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 6 VN POLL 0 0K 0 agp 0 0K 0 statfs 0 0K 76913 namei_tracker 0 0K 8 export_host 0 0K 0 cl_savebuf 0 0K 3 nvme_da 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 twsbuf 0 0K 0 twe_commands 0 0K 0 tcp_log_dev 0 0K 1813 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 523 biobuf 0 0K 0 aios 0 0K 9 lio 0 0K 36 acl 0 0K 0 mbuf_tag 0 0K 198 ktls 0 0K 0 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 tempbuff 0 0K 0 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 accf 0 0K 0 pts 0 0K 0 iov 0 0K 53867 ioctlops 0 0K 1333 eventfd 0 0K 50 Witness 0 0K 0 stack 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 sbuf 0 0K 288 mpr_user 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 MPRSAS 0 0K 0 SWAP 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 sysctltmp 0 0K 1122 sysctl 0 0K 3 md_sectors 0 0K 0 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 md_disk 0 0K 0 malodev 0 0K 0 LED 0 0K 0 ix_sriov 0 0K 0 cache 0 0K 0 aacraidcam 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 aacraid_buf 0 0K 0 aaccam 0 0K 0 boottrace 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 9392 260 1479446 0 254 39534592 0 sctp_asoc 2256 1960 580 45144 0 254 5730240 0 malloc-2048 2048 2004 212 184901 0 8 4538368 0 mbuf 256 13308 422 2918003 0 254 3514880 0 sctp_ep 1208 1995 545 184892 0 254 3068320 0 malloc-384 384 4620 2640 82379 0 30 2787840 0 RADIX NODE 144 17508 1023 1482209 0 63 2668464 0 mbuf_cluster 2048 1270 0 1270 0 254 2600960 0 tcp_log 416 0 5589 241482 0 254 2325024 0 pbuf 2624 0 778 0 0 2 2041472 0 malloc-1024 1024 1963 21 45108 0 16 2031616 0 sctp_raddr 736 1960 581 45402 0 254 1870176 0 BUF TRIE 144 187 11601 2164 0 62 1697472 0 malloc-384 384 3992 118 236391 0 30 1578240 0 malloc-128 128 11644 353 242787 0 126 1535616 0 UMA Slabs 0 112 13154 16 13154 0 126 1475040 0 malloc-4096 4096 330 4 514 0 2 1368064 0 vmem btag 56 23876 67 23876 0 254 1340808 0 VM OBJECT 264 3289 101 1649235 0 30 894960 0 malloc-16384 16384 49 5 76392 0 1 884736 0 malloc-256 256 50 3250 141355 0 62 844800 0 malloc-256 256 554 2671 78640 0 62 825600 0 socket 960 78 686 199112 0 254 733440 0 256 Bucket 2048 251 85 8963 0 8 688128 0 FFS inode 1160 559 29 78648 0 8 682080 0 tcp_bbr_map 128 763 4197 65248 0 126 634880 0 malloc-256 256 2073 297 340686 0 62 606720 0 tcpcb 1104 5 506 9143 0 254 564144 0 MAP ENTRY 96 4664 544 3852595 0 127 499968 0 ertt_txseginfo 40 7809 4412 220094 0 254 488840 0 malloc-4096 4096 111 5 77529 0 2 475136 0 malloc-64 64 4004 3115 1107586 0 254 455616 0 malloc-128 128 524 2762 78577 0 126 420608 0 THREAD 1808 198 33 86896 0 8 417648 0 sctp_stream_msg_out 112 3105 459 21780 0 254 399168 0 lkpicurr 168 2 2350 2 0 62 395136 0 malloc-65536 65536 6 0 6 0 1 393216 0 malloc-65536 65536 1 5 631 0 1 393216 0 lkpimm 168 1 2327 1 0 62 391104 0 sctp_chunk 152 1136 1152 20173 0 254 347776 0 mbuf_packet 256 814 456 185541 0 254 325120 0 VNODE 448 600 93 78691 0 30 310464 0 malloc-16 16 14614 386 23822 0 254 240000 0 VMSPACE 2552 78 12 77386 0 4 229680 0 DEVCTL 1024 0 220 154 0 0 225280 0 FPU_save_area 832 200 43 92111 0 16 202176 0 malloc-64 64 2537 550 54306 0 254 197568 0 malloc-65536 65536 1 2 659 0 1 196608 0 malloc-128 128 1241 278 27303 0 126 194432 0 malloc-32 32 5564 358 5707 0 254 189504 0 UMA Zones 768 244 0 244 0 16 187392 0 malloc-128 128 1219 176 6093 0 126 178560 0 FFS2 dinode 256 559 131 78648 0 62 176640 0 malloc-256 256 210 480 95762 0 62 176640 0 128 Bucket 1024 123 40 17841 0 16 166912 0 malloc-32768 32768 0 5 19 0 1 163840 0 malloc-1024 1024 137 23 154 0 16 163840 0 PROC 1352 105 15 77444 0 8 162240 0 S VFS Cache 104 1036 485 81023 0 126 158184 0 malloc-32 32 4138 524 324765 0 254 149184 0 filedesc0 1072 105 28 77446 0 8 142576 0 64 Bucket 512 178 86 34733 0 30 135168 0 malloc-65536 65536 0 2 10 0 1 131072 0 unpcb 256 22 488 1766 0 254 130560 0 ksiginfo 112 111 933 6007 0 126 116928 0 malloc-4096 4096 21 7 53 0 2 114688 0 malloc-256 256 305 130 87731 0 62 111360 0 32 Bucket 256 192 243 28193 0 62 111360 0 malloc-512 512 4 212 7158 0 30 110592 0 malloc-32768 32768 0 3 120 0 1 98304 0 malloc-2048 2048 5 43 1231 0 8 98304 0 malloc-256 256 340 35 794 0 62 96000 0 malloc-8192 8192 9 2 38 0 1 90112 0 malloc-4096 4096 20 2 34 0 2 90112 0 UMA Kegs 384 230 3 230 0 30 89472 0 syncache 168 0 528 19 0 254 88704 0 g_bio 408 0 210 13121 0 30 85680 0 pipe 744 23 92 1521 0 16 85560 0 clpbuf 2624 0 32 108 0 16 83968 0 malloc-128 128 377 274 80476 0 126 83328 0 malloc-8192 8192 1 9 447 0 1 81920 0 malloc-384 384 173 37 174 0 30 80640 0 sctp_readq 152 0 520 360 0 254 79040 0 malloc-16 16 4046 704 297990 0 254 76000 0 malloc-16 16 4199 551 247161 0 254 76000 0 malloc-8192 8192 8 1 10 0 1 73728 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-32768 32768 2 0 2 0 1 65536 0 malloc-32768 32768 2 0 2 0 1 65536 0 malloc-16384 16384 3 1 7 0 1 65536 0 udp_inpcb 424 6 147 1250 0 30 64872 0 tcp_inpcb 424 5 148 9143 0 30 64872 0 ripcb 424 5 148 1371 0 30 64872 0 sctp_laddr 48 12 1332 79922 0 254 64512 0 Files 80 282 518 304410 0 126 64000 0 malloc-512 512 12 108 286 0 30 61440 0 udplite_inpcb 424 0 126 604 0 30 53424 0 malloc-64 64 147 672 77565 0 254 52416 0 malloc-64 64 137 682 87252 0 254 52416 0 malloc-64 64 566 253 1551 0 254 52416 0 DIRHASH 1024 35 13 35 0 16 49152 0 NAMEI 1024 0 48 339296 0 16 49152 0 malloc-16384 16384 2 1 18 0 1 49152 0 malloc-16384 16384 0 3 160 0 1 49152 0 malloc-2048 2048 10 14 47 0 8 49152 0 malloc-1024 1024 5 43 2088 0 16 49152 0 tcp_rack_pcb 896 0 54 3060 0 16 48384 0 pcpu-8 8 4720 912 7486 0 254 45056 0 tcp_bbr_pcb 832 1 53 3393 0 16 44928 0 TURNSTILE 136 232 83 232 0 62 42840 0 malloc-8192 8192 2 3 393 0 1 40960 0 Mountpoints 2752 2 12 2 0 4 38528 0 da_ccb 544 0 70 3385 0 16 38080 0 PWD 32 57 1077 76447 0 254 36288 0 malloc-64 64 60 507 21417 0 254 36288 0 malloc-64 64 224 343 329 0 254 36288 0 malloc-64 64 188 379 1122 0 254 36288 0 16 Bucket 144 72 180 1264 0 62 36288 0 8 Bucket 80 80 370 19326 0 126 36000 0 malloc-128 128 36 243 267 0 126 35712 0 malloc-128 128 30 249 5802 0 126 35712 0 malloc-128 128 47 232 47 0 126 35712 0 routing nhops 256 27 108 34 0 62 34560 0 ttyoutq 256 72 63 160 0 62 34560 0 malloc-384 384 53 37 55 0 30 34560 0 malloc-256 256 29 106 3405 0 62 34560 0 malloc-256 256 7 128 79321 0 62 34560 0 SLEEPQUEUE 88 232 152 232 0 126 33792 0 malloc-16384 16384 2 0 2 0 1 32768 0 malloc-4096 4096 2 6 77538 0 2 32768 0 malloc-2048 2048 3 13 1034 0 8 32768 0 malloc-2048 2048 3 13 236 0 8 32768 0 malloc-2048 2048 0 16 50 0 8 32768 0 malloc-1024 1024 8 24 208 0 16 32768 0 malloc-1024 1024 5 27 5 0 16 32768 0 malloc-1024 1024 3 29 455 0 16 32768 0 malloc-1024 1024 9 23 9 0 16 32768 0 malloc-1024 1024 3 29 20 0 16 32768 0 malloc-512 512 0 64 134 0 30 32768 0 malloc-512 512 2 62 51 0 30 32768 0 malloc-512 512 5 59 143 0 30 32768 0 malloc-512 512 8 56 8 0 30 32768 0 pcpu-64 64 496 16 496 0 254 32768 0 KNOTE 160 28 172 557398 0 62 32000 0 ttyinq 160 135 65 300 0 62 32000 0 tcp_rack_map 120 0 264 7044 0 126 31680 0 cpuset 104 7 272 7 0 126 29016 0 tcp_inpcb ports 32 2 880 5699 0 254 28224 0 ertt 72 5 387 9143 0 126 28224 0 4 Bucket 48 7 581 1717 0 254 28224 0 2 Bucket 32 69 813 7651 0 254 28224 0 AIO 208 0 133 280 0 62 27664 0 malloc-8192 8192 3 0 3 0 1 24576 0 malloc-4096 4096 2 4 5 0 2 24576 0 rtentry 176 30 108 34 0 62 24288 0 PGRP 88 39 237 198 0 126 24288 0 rl_entry 40 86 520 86 0 254 24240 0 malloc-384 384 3 57 132 0 30 23040 0 malloc-384 384 20 40 363 0 30 23040 0 hostcache 64 2 313 2 0 254 20160 0 udpcb 32 6 624 1854 0 254 20160 0 udp_inpcb ports 32 3 627 43 0 254 20160 0 udplite_inpcb ports 32 0 630 9 0 254 20160 0 AIOP 32 4 626 4 0 254 20160 0 malloc-32 32 107 523 128 0 254 20160 0 malloc-32 32 26 604 1576 0 254 20160 0 malloc-32 32 121 509 71205 0 254 20160 0 malloc-32 32 59 571 7432 0 254 20160 0 malloc-32 32 56 574 1106 0 254 20160 0 malloc-32 32 5 625 9 0 254 20160 0 cryptop 280 0 70 76 0 30 19600 0 AIOCB 552 0 35 515 0 16 19320 0 AIOLIO 272 0 70 36 0 30 19040 0 epoch_record pcpu 256 4 60 4 0 62 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-2048 2048 3 5 3 0 8 16384 0 malloc-2048 2048 3 5 3 0 8 16384 0 malloc-512 512 0 32 20 0 30 16384 0 malloc-512 512 1 31 1 0 30 16384 0 SMR CPU 32 7 504 7 0 254 16352 0 tcpreass 48 0 336 2 0 254 16128 0 vtnet_tx_hdr 24 0 668 816464 0 254 16032 0 kenv 258 15 45 1053 0 30 15480 0 mqnode 416 3 33 3 0 30 14976 0 vmem 1856 1 7 1 0 8 14848 0 SMR SHARED 24 7 504 7 0 254 12264 0 malloc-16 16 0 750 18 0 254 12000 0 malloc-16 16 14 736 24462 0 254 12000 0 malloc-16 16 40 710 167 0 254 12000 0 malloc-16 16 291 459 26740 0 254 12000 0 itimer 352 0 33 3 0 30 11616 0 malloc-384 384 1 29 1 0 30 11520 0 malloc-384 384 2 28 2 0 30 11520 0 malloc-8192 8192 1 0 1 0 1 8192 0 malloc-8192 8192 1 0 1 0 1 8192 0 malloc-4096 4096 0 2 1 0 2 8192 0 pcpu-16 16 14 498 14 0 254 8192 0 malloc-16 16 10 490 12 0 254 8000 0 UMA Slabs 1 176 11 11 11 0 62 3872 0 KMAP ENTRY 96 15 24 20 0 0 3744 0 pcpu-4 4 1 511 1 0 254 2048 0 FFS1 dinode 128 0 0 0 0 126 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 254 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 312 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 56 0 0 0 0 254 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 72 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 tcp_log_id_node 120 0 0 0 0 126 0 0 tcp_log_id_bucket 176 0 0 0 0 62 0 0 ripcb ports 32 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 NCLNODE 608 0 0 0 0 16 0 0 TMPFS node 224 0 0 0 0 62 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 linux_dma_object 32 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 domainset 40 0 0 0 0 254 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-4096 4096 0 0 0 0 2 0 0 pcpu-32 32 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0