panic: Warning: Last msg marked incomplete, yet nothing left? cpuid = 1 time = 1641801081 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0xc7/frame 0xfffffe009a0d8030 kdb_backtrace() at kdb_backtrace+0xd3/frame 0xfffffe009a0d8190 vpanic() at vpanic+0x2b8/frame 0xfffffe009a0d8270 panic() at panic+0xb5/frame 0xfffffe009a0d8330 sctp_lower_sosend() at sctp_lower_sosend+0x58e7/frame 0xfffffe009a0d8700 sctp_sosend() at sctp_sosend+0x72c/frame 0xfffffe009a0d89a0 sosend() at sosend+0xfc/frame 0xfffffe009a0d8a10 kern_sendit() at kern_sendit+0x58a/frame 0xfffffe009a0d8b70 freebsd32_sendmsg() at freebsd32_sendmsg+0x4e6/frame 0xfffffe009a0d8d30 ia32_syscall() at ia32_syscall+0x41d/frame 0xfffffe009a0d8f30 int0x80_syscall_common() at int0x80_syscall_common+0x9c/frame 0xfbffcf78 KDB: enter: panic [ thread pid 37000 tid 141217 ] Stopped at kdb_enter+0x6b: movq $0,0x2510c4a(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0 rax 0x12 rcx 0x122c33ca2ec30524 rdx 0xdffff7c000000000 rbx 0 rsp 0xfffffe009a0d8170 rbp 0xfffffe009a0d8190 rsi 0x1 rdi 0 r8 0x3 r9 0xffffffff r10 0 r11 0xfffffe009000fc50 r12 0xfffffe00962523a0 r13 0xfffffe009a0d8101 r14 0xffffffff82b557e0 .str.18 r15 0xffffffff82b557e0 .str.18 rip 0xffffffff817556cb kdb_enter+0x6b rflags 0x200046 kernload+0x46 kdb_enter+0x6b: movq $0,0x2510c4a(%rip) db> show proc Process 37000 (syz-executor.2) at 0xfffffe0096208548: state: NORMAL uid: 0 gids: 0, 0, 5 parent: pid 779 at 0xfffffe0090089000 ABI: FreeBSD ELF32 flag: 0x10000080 flag2: 0 arguments: /root/syz-executor.2 exec reaper: 0xfffffe0053df6000 reapsubtree: 1 sigparent: 20 vmspace: 0xfffffe009a4723f0 (map 0xfffffe009a4723f0) (map.pmap 0xfffffe009a4724b0) (pmap 0xfffffe009a472518) threads: 3 139430 S nanslp 0xffffffff83c339c1 syz-executor.2 141217 Run CPU 1 syz-executor.2 141218 S uwait 0xfffffe009637ea00 syz-executor.2 db> ps pid ppid pgrp uid state wmesg wchan cmd 37000 779 779 0 R (threaded) syz-executor.2 139430 S nanslp 0xffffffff83c339c1 syz-executor.2 141217 Run CPU 1 syz-executor.2 141218 S uwait 0xfffffe009637ea00 syz-executor.2 26160 1 786 0 SV uwait 0xfffffe0096329380 syz-executor.3 26152 26150 786 0 SV uwait 0xfffffe0090026500 syz-executor.3 26150 1 786 0 DV ppwait 0xfffffe00962064f8 syz-executor.3 26138 1 786 0 SV uwait 0xfffffe0090010c00 syz-executor.3 26090 1 786 0 S uwait 0xfffffe009637de00 syz-executor.3 26089 1 786 0 S uwait 0xfffffe0090029080 syz-executor.3 26085 1 786 0 S uwait 0xfffffe0099f9d180 syz-executor.3 26077 1 786 0 S umtxn 0xfffffe009637e080 syz-executor.3 14324 1 14324 65 Ss select 0xfffffe00928b28c0 dhclient 13234 1 13234 0 Ss select 0xfffffe00928b2740 dhclient 13231 1 13231 0 Ss select 0xfffffe00928b1e40 dhclient 13212 1 13212 65 Ss select 0xfffffe00928b2f40 dhclient 8354 1 8354 0 Ss select 0xfffffe00571d77c0 dhclient 8351 1 8351 0 Ss select 0xfffffe00928b29c0 dhclient 8332 1 8332 65 Ss select 0xfffffe00571d76c0 dhclient 5643 1 779 0 S umtxn 0xfffffe0090011480 syz-executor.2 5639 1 779 0 S umtxn 0xfffffe0090012400 syz-executor.2 5632 1 779 0 S uwait 0xfffffe0090028880 syz-executor.2 5626 1 779 0 S umtxn 0xfffffe0090010900 syz-executor.2 5622 1 779 0 S umtxn 0xfffffe0007979100 syz-executor.2 5616 1 779 0 S umtxn 0xfffffe0090012e80 syz-executor.2 4541 1 4541 0 Ss select 0xfffffe00571d6e40 dhclient 4538 1 4538 0 Ss select 0xfffffe00928b2ac0 dhclient 4510 1 4510 65 Ss select 0xfffffe00928b3940 dhclient 1859 1 1859 0 Ss select 0xfffffe00571d7740 dhclient 1856 1 1856 0 Ss select 0xfffffe00571d7640 dhclient 1622 1 777 0 S umtxn 0xfffffe0090013200 syz-executor.0 1619 1 777 0 S uwait 0xfffffe0090012280 syz-executor.0 1615 0 0 0 DL aiordy 0xfffffe0058c2a548 [aiod4] 1614 0 0 0 DL aiordy 0xfffffe0096229000 [aiod3] 1613 0 0 0 DL aiordy 0xfffffe009008b000 [aiod2] 1611 0 0 0 DL aiordy 0xfffffe0090089548 [aiod1] 1608 1 777 0 S uwait 0xfffffe0090028580 syz-executor.0 786 774 786 0 Ss piperd 0xfffffe0058b6a2e8 syz-executor.3 779 774 779 0 Ss nanslp 0xffffffff83c339c1 syz-executor.2 778 774 778 0 Ss piperd 0xfffffe0058b52ba0 syz-executor.1 777 774 777 0 Ss piperd 0xfffffe0058b522e8 syz-executor.0 774 772 772 0 S (threaded) syz-fuzzer 100109 S uwait 0xfffffe0007975300 syz-fuzzer 100115 S uwait 0xfffffe0007979700 syz-fuzzer 100116 S uwait 0xfffffe0007979800 syz-fuzzer 100117 S uwait 0xfffffe0007979900 syz-fuzzer 100118 S uwait 0xfffffe0090028980 syz-fuzzer 100119 S uwait 0xfffffe0090028a80 syz-fuzzer 100120 S uwait 0xfffffe0090028b80 syz-fuzzer 100121 S kqread 0xfffffe0090006000 syz-fuzzer 100122 S uwait 0xfffffe0090028d80 syz-fuzzer 772 770 772 0 Ss pause 0xfffffe009008ab40 csh 770 688 770 0 Ss select 0xfffffe00081e5140 sshd 754 1 754 0 Ss+ ttyin 0xfffffe00572330b0 getty 753 1 753 0 Ss+ ttyin 0xfffffe00543f5cb0 getty 752 1 752 0 Ss+ ttyin 0xfffffe00543f64b0 getty 751 1 751 0 Ss+ ttyin 0xfffffe00543f6cb0 getty 750 1 750 0 Ss+ ttyin 0xfffffe00543f74b0 getty 749 1 749 0 Ss+ ttyin 0xfffffe00543f7cb0 getty 748 1 748 0 Ss+ ttyin 0xfffffe00543f84b0 getty 747 1 747 0 Ss+ ttyin 0xfffffe00543f8cb0 getty 746 1 746 0 Ss+ ttyin 0xfffffe00576da4b0 getty 692 1 692 0 Ss nanslp 0xffffffff83c339c1 cron 688 1 688 0 Ss select 0xfffffe00571d7840 sshd 501 1 501 0 Ss select 0xfffffe00081e5240 syslogd 430 1 430 0 Ss select 0xfffffe00081e54c0 devd 429 1 429 65 Ss select 0xfffffe00081e5440 dhclient 344 1 344 0 Ss select 0xfffffe00081e5540 dhclient 341 1 341 0 Ss select 0xfffffe00081e52c0 dhclient 17 0 0 0 DL syncer 0xffffffff83d591c0 [syncer] 16 0 0 0 DL vlruwt 0xfffffe00587bfa90 [vnlru] 15 0 0 0 DL (threaded) [bufdaemon] 100078 D qsleep 0xffffffff83d577e0 [bufdaemon] 100081 D - 0xffffffff83011f80 [bufspacedaemon-0] 100093 D sdflush 0xfffffe00079640e8 [/ worker] 9 0 0 0 DL psleep 0xffffffff83d8acc0 [vmdaemon] 8 0 0 0 DL (threaded) [pagedaemon] 100076 D psleep 0xffffffff83d7eb78 [dom0] 100079 D launds 0xffffffff83d7eb84 [laundry: dom0] 100080 D umarcl 0xffffffff81e54280 [uma] 7 0 0 0 DL - 0xffffffff839ef908 [rand_harvestq] 6 0 0 0 DL pftm 0xffffffff8495c530 [pf purge] 5 0 0 0 DL waiting 0xffffffff845824a0 [sctp_iterator] 4 0 0 0 DL (threaded) [cam] 100045 D - 0xffffffff838918c0 [doneq0] 100046 D - 0xffffffff83891840 [async] 100075 D - 0xffffffff838916c0 [scanner] 14 0 0 0 DL seqstat 0xfffffe00570d4488 [sequencer 00] 3 0 0 0 DL (threaded) [crypto] 100041 D crypto_ 0xffffffff83d7a3c0 [crypto] 100042 D crypto_ 0xfffffe0053fae030 [crypto returns 0] 100043 D crypto_ 0xfffffe0053fae080 [crypto returns 1] 13 0 0 0 DL (threaded) [geom] 100036 D - 0xffffffff83c090a0 [g_event] 100037 D - 0xffffffff83c090c0 [g_up] 100038 D - 0xffffffff83c090e0 [g_down] 2 0 0 0 WL (threaded) [clock] 100030 I [clock (0)] 100031 I [clock (1)] 12 0 0 0 WL (threaded) [intr] 100015 I [swi5: fast taskq] 100018 I [swi6: task queue] 100019 I [swi6: Giant taskq] 100029 I [swi1: netisr 0] 100032 I [swi1: hpts] 100033 I [swi1: hpts] 100034 I [swi3: busdma] 100047 I [irq24: virtio_pci0] 100048 I [irq25: virtio_pci0] 100049 I [irq26: virtio_pci0] 100050 I [irq27: virtio_pci0] 100051 I [irq28: virtio_pci1] 100052 I [irq29: virtio_pci1] 100053 I [irq30: virtio_pci1] 100054 I [irq31: virtio_pci1] 100055 I [irq32: virtio_pci1] 100060 I [irq10: virtio_pci2] 100062 I [irq1: atkbd0] 100063 I [irq12: psm0] 100064 I [swi0: uart uart++] 100068 I [swi1: pf send] 11 0 0 0 RL (threaded) [idle] 100003 Run CPU 0 [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffffe0053df6000 [init] 10 0 0 0 DL audit_w 0xffffffff83d7aea0 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff83c09b40 [swapper] 100005 D - 0xfffffe000819c500 [softirq_0] 100006 D - 0xfffffe000819c400 [softirq_1] 100007 D - 0xfffffe000819c300 [if_io_tqg_0] 100008 D - 0xfffffe000819c200 [if_io_tqg_1] 100009 D - 0xfffffe000819c100 [if_config_tqg_0] 100010 D - 0xfffffe000819c000 [linuxkpi_irq_wq] 100011 D - 0xfffffe000819be00 [inm_free taskq] 100012 D - 0xfffffe000819bd00 [in6m_free taskq] 100013 D - 0xfffffe000819bc00 [deferred_unmount ta] 100014 D - 0xfffffe000819bb00 [thread taskq] 100016 D - 0xfffffe000819b800 [kqueue_ctx taskq] 100017 D - 0xfffffe000819b700 [pci_hp taskq] 100020 D - 0xfffffe000819b200 [aiod_kick taskq] 100021 D - 0xfffffe000819b100 [linuxkpi_short_wq_0] 100022 D - 0xfffffe000819b100 [linuxkpi_short_wq_1] 100023 D - 0xfffffe000819b100 [linuxkpi_short_wq_2] 100024 D - 0xfffffe000819b100 [linuxkpi_short_wq_3] 100025 D - 0xfffffe000819b000 [linuxkpi_long_wq_0] 100026 D - 0xfffffe000819b000 [linuxkpi_long_wq_1] 100027 D - 0xfffffe000819b000 [linuxkpi_long_wq_2] 100028 D - 0xfffffe000819b000 [linuxkpi_long_wq_3] 100035 D - 0xfffffe0053edad00 [firmware taskq] 100039 D - 0xfffffe0053edac00 [crypto_0] 100040 D - 0xfffffe0053edac00 [crypto_1] 100056 D - 0xfffffe0053eda400 [vtnet0 rxq 0] 100057 D - 0xfffffe0053eda300 [vtnet0 txq 0] 100058 D - 0xfffffe0053eda200 [vtnet0 rxq 1] 100059 D - 0xfffffe0053eda100 [vtnet0 txq 1] 100061 D vtbslp 0xfffffe00571b4580 [virtio_balloon] 100065 D - 0xffffffff82b5b2c1 [deadlkres] 100069 D - 0xfffffe0053ed9300 [acpi_task_0] 100070 D - 0xfffffe0053ed9300 [acpi_task_1] 100071 D - 0xfffffe0053ed9300 [acpi_task_2] 100072 D - 0xfffffe000819e100 [mca taskq] 100074 D - 0xfffffe0053eda900 [CAM taskq] db> show all locks Process 37000 (syz-executor.2) thread 0xfffffe00962523a0 (141217) exclusive sleep mutex sctp-send-tcb (tcbs) r = 0 (0xfffffe009a3fec90) locked @ /syzkaller/managers/i386/kernel/sys/netinet/sctp_output.c:13132 db> show malloc Type InUse MemUse Requests pf_hash 5 11524K 5 tcp_hpts 6 4801K 6 devbuf 4213 4323K 4241 sysctloid 35454 2089K 35525 vtbuf 24 1968K 46 pcb 781 1390K 184051 kobj 322 1288K 480 newblk 9 1026K 37854 vfscache 3 1025K 3 inodedep 364 649K 36147 ufs_quota 1 512K 1 vfs_hash 1 512K 1 callout 2 512K 2 intr 4 472K 4 sctp_stro 370 370K 20512 filedesc 46 361K 72037 subproc 172 337K 37144 sctp_atcl 749 281K 111934 vmem 3 274K 6 acpica 1674 184K 56282 vnet_data 1 168K 1 tidhash 3 141K 3 linker 354 134K 382 pagedep 9 130K 36010 tfo_ccache 1 128K 1 DEVFS1 109 109K 126 sem 4 106K 4 dirrem 355 89K 36057 BPF 46 88K 55 bus 973 80K 3426 mtx_pool 2 72K 2 syncache 1 68K 1 sctp_timw 268 67K 268 acpitask 1 64K 1 ddb_capture 1 64K 1 module 505 64K 505 umtx 418 53K 418 kdtrace 249 49K 78236 sctp_atky 1135 48K 133067 freefile 355 45K 36040 temp 34 33K 8284 DEVFS3 128 32K 138 hostcache 1 32K 1 shm 1 32K 10 msg 4 30K 4 gtaskqueue 18 26K 18 kbdmux 6 22K 6 ifaddr 71 21K 73 DEVFS_RULE 56 20K 56 ufs_mount 4 17K 5 proc 3 17K 3 tty 16 16K 16 routetbl 135 16K 427 ithread 93 15K 93 bus-sc 34 15K 1651 lltable 43 14K 129 ether_multi 157 13K 167 KTRACE 100 13K 100 ifnet 7 13K 7 sctp_athm 749 12K 111959 sctp_map 740 12K 40030 kenv 95 12K 95 eventhandler 133 12K 133 CAM queue 5 11K 1528 GEOM 61 10K 488 rman 84 10K 425 in6_multi 71 9K 71 bmsafemap 2 9K 36146 UART 12 9K 12 devstat 4 9K 4 ksem 1 8K 116 rpc 2 8K 2 shmfd 1 8K 151 pfs_vncache 1 8K 1 pfs_nodes 20 8K 20 audit_evclass 237 8K 296 kqueue 77 7K 37051 taskqueue 63 7K 63 cred 26 7K 335 sglist 5 7K 5 CAM DEV 3 6K 510 plimit 24 6K 454 pwddesc 75 5K 37300 ufs_dirhash 24 5K 24 UMA 272 5K 272 session 35 5K 433 pf_ifnet 10 5K 19 DEVFSP 68 5K 6667 vt 11 5K 11 memdesc 1 4K 1 MCA 32 4K 32 evdev 4 4K 4 proc-args 106 4K 38382 acpisem 28 4K 28 selfd 55 4K 489001 lockf 32 4K 60 hhook 15 4K 17 kcovinfo 52 4K 52 terminal 11 3K 11 select 19 3K 90 uidinfo 3 3K 15 local_apic 1 2K 1 io_apic 1 2K 1 fpukern_ctx 2 2K 2 ipsec-saq 2 2K 2 ip6ndp 12 2K 15 sctp_ifa 14 2K 15 Unitno 30 2K 117 CAM XPT 22 2K 543 in_multi 6 2K 8 ipsecpolicy 2 2K 2 acpidev 20 2K 20 msi 9 2K 9 clone 9 2K 9 tun 7 2K 7 freework 5 2K 36564 softdep 1 1K 1 mkdir 8 1K 71980 freeblks 4 1K 36049 sahead 1 1K 1 secasvar 1 1K 1 nhops 6 1K 8 vnodemarker 2 1K 68 NFSD session 1 1K 1 CAM periph 4 1K 271 sctp_ifn 6 1K 15 ipsec 3 1K 3 mld 6 1K 6 igmp 6 1K 6 toponodes 6 1K 6 isadev 6 1K 6 mount 16 1K 89 pci_link 10 1K 10 crypto 4 1K 28 encap_export_host 12 1K 12 procdesc 5 1K 18 newdirblk 4 1K 35990 diradd 4 1K 36097 pfil 4 1K 4 cdev 2 1K 2 inpcbpolicy 15 1K 5724 osd 8 1K 4362 chacha20random 1 1K 1 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 DEVFS 9 1K 10 CC Mem 4 1K 4351 vnodes 1 1K 1 CAM SIM 2 1K 2 feeder 7 1K 7 tcpfunc 3 1K 3 loginclass 3 1K 6 prison 6 1K 6 lkpikmalloc 5 1K 6 aesni_data 2 1K 2 soname 5 1K 48056 nexusdev 8 1K 8 apmdev 1 1K 1 atkbddev 2 1K 2 xform 2 1K 6320 CAM dev queue 2 1K 2 CAM I/O Scheduler 1 1K 1 CAM path 4 1K 1034 pmchooks 1 1K 1 filecaps 5 1K 116 sctp_vrf 1 1K 1 vnet 1 1K 1 entropy 2 1K 47 acpiintr 1 1K 1 pmc 1 1K 1 cpus 2 1K 2 vnet_data_free 1 1K 1 Per-cpu 1 1K 1 iov 1 1K 26224 p1003.1b 1 1K 1 pf_table 0 0K 0 pf_rule 0 0K 0 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_temp 0 0K 0 tcp_do 0 0K 0 tcp_fsb 0 0K 2546 sctp_mcore 0 0K 0 sctp_socko 0 0K 37868 sctp_iter 0 0K 11 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 0 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 11 sctp_aadr 0 0K 0 sctp_stri 0 0K 0 mqdata 0 0K 0 filemon 0 0K 1891 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 ixl 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 iavf 0 0K 0 axgbe 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 xen_intr 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 bounce 0 0K 0 busdma 0 0K 0 qpidrv 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 DEVFS4 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 DEVFS2 0 0K 0 xenbus 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 vm_fictitious 0 0K 0 xenstore 0 0K 0 ciss_data 0 0K 0 BACKLIGHT 0 0K 0 xnb 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 sysmouse 0 0K 0 vtfont 0 0K 0 UMAHash 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 29658 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 32 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 2 allocindir 0 0K 0 indirdep 0 0K 747 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 scsi_da 0 0K 69 pvscsi 0 0K 0 ata_da 0 0K 0 scsi_ch 0 0K 0 scsi_cd 0 0K 0 AHCI driver 0 0K 0 agp 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 ktls_ocf 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 lkpiskb 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpindev 0 0K 0 lkpifw 0 0K 0 lkpi80211 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 417 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 nvme_da 0 0K 0 LRO 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 0 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 0 VN POLL 0 0K 0 statfs 0 0K 36255 namei_tracker 0 0K 0 export_host 0 0K 0 cl_savebuf 0 0K 3 acpipwr 0 0K 0 acpi_perf 0 0K 0 acpicmbat 0 0K 0 twsbuf 0 0K 0 twe_commands 0 0K 0 tcp_log_dev 0 0K 3 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 523 PUC 0 0K 0 biobuf 0 0K 0 aios 0 0K 0 lio 0 0K 78 acl 0 0K 0 ppbusdev 0 0K 0 mbuf_tag 0 0K 139 ktls 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 tempbuff 0 0K 0 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 accf 0 0K 0 pts 0 0K 0 ioctlops 0 0K 215 eventfd 0 0K 89 Witness 0 0K 0 stack 0 0K 0 mps_user 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 sbuf 0 0K 288 mfibuf 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 md_sectors 0 0K 0 SWAP 0 0K 0 md_disk 0 0K 0 malodev 0 0K 0 sysctltmp 0 0K 925 sysctl 0 0K 3 LED 0 0K 0 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 ix_sriov 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 ix 0 0K 0 cache 0 0K 0 ipsbuf 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 1 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8338 1060 1103572 0 254 38494208 0 mbuf 256 9266 652 1936041 0 254 2539008 0 pbuf 2624 0 794 0 0 2 2083456 0 RADIX NODE 144 13264 451 907597 0 65 1974960 0 malloc-384 384 4913 127 116104 0 30 1935360 0 malloc-4096 4096 420 8 38136 0 2 1753088 0 sctp_asoc 2288 370 395 20015 0 254 1750320 0 BUF TRIE 144 191 11597 2077 0 62 1697472 0 mbuf_cluster 2048 762 0 762 0 254 1560576 0 malloc-128 128 11641 77 12037 0 126 1499904 0 UMA Slabs 0 112 11363 7 11363 0 126 1273440 0 sctp_ep 1208 379 641 91919 0 254 1232160 0 malloc-8192 8192 12 135 1906 0 1 1204224 0 malloc-384 384 363 2457 36221 0 30 1082880 0 vmem btag 56 19088 103 19088 0 254 1074696 0 malloc-2048 2048 381 83 91980 0 8 950272 0 malloc-256 256 555 2490 73359 0 62 779520 0 socket 960 49 715 99101 0 254 733440 0 malloc-256 256 10 2855 67525 0 62 733440 0 FFS inode 1160 531 29 36572 0 8 649600 0 VM OBJECT 264 2284 116 539361 0 30 633600 0 malloc-1024 1024 506 70 20186 0 16 589824 0 sctp_raddr 736 370 400 20388 0 254 566720 0 tcpcb 1096 4 507 4351 0 254 560056 0 malloc-16384 16384 23 7 36017 0 1 491520 0 malloc-128 128 817 2593 72857 0 126 436480 0 256 Bucket 2048 188 12 4859 0 8 409600 0 lkpicurr 168 2 2350 2 0 62 395136 0 malloc-65536 65536 6 0 6 0 1 393216 0 lkpimm 168 1 2327 1 0 62 391104 0 THREAD 1808 173 36 41226 0 8 377872 0 MAP ENTRY 96 3007 395 1155880 0 126 326592 0 VNODE 448 572 121 36615 0 30 310464 0 malloc-2048 2048 10 134 517 0 8 294912 0 malloc-64 64 3894 201 4181 0 254 262080 0 malloc-16 16 14667 583 47529 0 254 244000 0 DEVCTL 1024 0 220 155 0 0 225280 0 tcp_log 416 0 513 12 0 254 213408 0 malloc-65536 65536 1 2 2536 0 1 196608 0 malloc-65536 65536 1 2 299 0 1 196608 0 malloc-32768 32768 0 6 695 0 1 196608 0 mbuf_packet 256 260 502 63440 0 254 195072 0 malloc-128 128 1364 155 29103 0 126 194432 0 UMA Zones 768 244 0 244 0 16 187392 0 FPU_save_area 832 175 50 45804 0 16 187200 0 malloc-32 32 5346 324 6806 0 254 181440 0 VMSPACE 2552 53 16 36990 0 4 176088 0 FFS2 dinode 256 531 99 36571 0 62 161280 0 malloc-256 256 473 157 42723 0 62 161280 0 S VFS Cache 104 1021 500 37404 0 126 158184 0 malloc-256 256 424 146 92730 0 62 145920 0 128 Bucket 1024 93 38 3755 0 16 134144 0 malloc-65536 65536 0 2 8 0 1 131072 0 unpcb 256 20 490 1410 0 254 130560 0 PROC 1352 74 22 37008 0 8 129792 0 64 Bucket 512 165 67 22596 0 30 118784 0 ksiginfo 112 87 957 4741 0 126 116928 0 malloc-16384 16384 3 4 276 0 1 114688 0 filedesc0 1072 75 23 37300 0 8 105056 0 malloc-128 128 591 184 3463 0 126 99200 0 pipe 744 23 107 1095 0 16 96720 0 malloc-4096 4096 21 1 32 0 2 90112 0 UMA Kegs 384 230 3 230 0 30 89472 0 g_bio 408 0 210 18495 0 30 85680 0 malloc-384 384 171 39 619 0 30 80640 0 malloc-256 256 105 210 37128 0 62 80640 0 32 Bucket 256 167 148 12805 0 62 80640 0 sctp_chunk 152 358 162 947 0 254 79040 0 malloc-64 64 709 362 27018 0 254 68544 0 malloc-64 64 750 321 56348 0 254 68544 0 malloc-128 128 163 364 75613 0 126 67456 0 malloc-128 128 312 215 36605 0 126 67456 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-65536 65536 0 1 136 0 1 65536 0 malloc-32768 32768 2 0 2 0 1 65536 0 malloc-16384 16384 3 1 18 0 1 65536 0 tcp_inpcb 424 4 149 4351 0 30 64872 0 ripcb 424 5 148 896 0 30 64872 0 sctp_stream_msg_out 112 258 282 1377 0 254 60480 0 malloc-8192 8192 3 4 167 0 1 57344 0 malloc-64 64 166 653 527822 0 254 52416 0 malloc-64 64 476 343 1081 0 254 52416 0 Files 80 229 421 157299 0 126 52000 0 malloc-256 256 108 87 37541 0 62 49920 0 malloc-256 256 8 187 36100 0 62 49920 0 DIRHASH 1024 34 14 34 0 16 49152 0 NAMEI 1024 0 48 167895 0 16 49152 0 malloc-2048 2048 7 17 550 0 8 49152 0 tcp_rack_pcb 896 0 54 2546 0 16 48384 0 da_ccb 544 0 84 4734 0 16 45696 0 syncache 168 0 264 4 0 254 44352 0 malloc-32 32 865 521 113865 0 254 44352 0 clpbuf 2624 0 16 20 0 16 41984 0 malloc-4096 4096 4 6 36262 0 2 40960 0 pcpu-8 8 4738 382 4957 0 254 40960 0 sctp_laddr 48 12 828 55962 0 254 40320 0 sctp_readq 152 0 260 4 0 254 39520 0 Mountpoints 2752 2 12 2 0 4 38528 0 udplite_inpcb 424 0 90 163 0 30 38160 0 udp_inpcb 424 6 84 314 0 30 38160 0 PWD 32 34 1100 36087 0 254 36288 0 malloc-64 64 62 505 4788 0 254 36288 0 malloc-64 64 83 484 41371 0 254 36288 0 malloc-64 64 3 564 47 0 254 36288 0 16 Bucket 144 86 166 696 0 62 36288 0 8 Bucket 80 73 377 8789 0 126 36000 0 tcp_bbr_map 128 0 279 724 0 126 35712 0 malloc-128 128 31 248 2717 0 126 35712 0 malloc-128 128 12 267 268 0 126 35712 0 routing nhops 256 27 108 35 0 62 34560 0 ttyoutq 256 72 63 160 0 62 34560 0 malloc-256 256 24 111 5015 0 62 34560 0 TURNSTILE 136 210 42 210 0 62 34272 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 0 1 120 0 1 32768 0 malloc-16384 16384 0 2 160 0 1 32768 0 malloc-8192 8192 4 0 4 0 1 32768 0 malloc-2048 2048 7 9 7 0 8 32768 0 malloc-2048 2048 3 13 83 0 8 32768 0 malloc-2048 2048 1 15 75 0 8 32768 0 malloc-2048 2048 5 11 236 0 8 32768 0 malloc-1024 1024 2 30 48 0 16 32768 0 malloc-1024 1024 3 29 2348 0 16 32768 0 malloc-1024 1024 4 28 10 0 16 32768 0 malloc-1024 1024 12 20 169 0 16 32768 0 malloc-1024 1024 3 29 3 0 16 32768 0 malloc-1024 1024 8 24 167 0 16 32768 0 malloc-512 512 1 63 185 0 30 32768 0 malloc-512 512 1 63 44 0 30 32768 0 malloc-512 512 14 50 139 0 30 32768 0 malloc-512 512 9 55 30 0 30 32768 0 pcpu-64 64 484 28 484 0 254 32768 0 ertt_txseginfo 40 0 808 49363 0 254 32320 0 KNOTE 160 28 172 286241 0 62 32000 0 ttyinq 160 135 65 300 0 62 32000 0 tcp_rack_map 120 0 264 3416 0 126 31680 0 tcp_bbr_pcb 832 0 36 458 0 16 29952 0 cpuset 104 7 272 148 0 126 29016 0 tcp_inpcb ports 32 2 880 2595 0 254 28224 0 malloc-32 32 266 616 45446 0 254 28224 0 4 Bucket 48 7 581 803 0 254 28224 0 AIO 208 0 133 61 0 62 27664 0 malloc-8192 8192 1 2 102 0 1 24576 0 malloc-8192 8192 1 2 4 0 1 24576 0 malloc-8192 8192 3 0 3 0 1 24576 0 rtentry 176 31 107 35 0 62 24288 0 PGRP 88 35 241 437 0 126 24288 0 rl_entry 40 84 522 84 0 254 24240 0 malloc-16 16 787 713 112036 0 254 24000 0 malloc-16 16 766 734 71043 0 254 24000 0 itimer 352 0 66 83 0 30 23232 0 malloc-384 384 0 60 3 0 30 23040 0 malloc-384 384 25 35 28 0 30 23040 0 malloc-384 384 1 59 42 0 30 23040 0 SLEEPQUEUE 88 210 46 210 0 126 22528 0 hostcache 64 1 314 1 0 254 20160 0 udpcb 32 6 624 477 0 254 20160 0 udplite_inpcb ports 32 0 630 14 0 254 20160 0 udp_inpcb ports 32 3 627 33 0 254 20160 0 ertt 72 4 276 4351 0 126 20160 0 malloc-32 32 80 550 240 0 254 20160 0 malloc-32 32 70 560 1502 0 254 20160 0 malloc-32 32 161 469 1051 0 254 20160 0 malloc-32 32 81 549 9183 0 254 20160 0 malloc-32 32 32 598 276 0 254 20160 0 2 Bucket 32 128 502 4347 0 254 20160 0 AIOCB 552 0 35 371 0 16 19320 0 AIOLIO 272 0 70 39 0 30 19040 0 epoch_record pcpu 256 4 60 4 0 62 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-4096 4096 0 4 6 0 2 16384 0 malloc-4096 4096 2 2 5 0 2 16384 0 malloc-4096 4096 1 3 2 0 2 16384 0 malloc-4096 4096 0 4 7 0 2 16384 0 malloc-2048 2048 0 8 12 0 8 16384 0 malloc-1024 1024 4 12 4 0 16 16384 0 malloc-512 512 1 31 1 0 30 16384 0 SMR CPU 32 7 504 7 0 254 16352 0 tcpreass 48 0 336 1 0 254 16128 0 vtnet_tx_hdr 24 0 668 579931 0 254 16032 0 kenv 258 15 45 1045 0 30 15480 0 mqnode 416 3 33 3 0 30 14976 0 vmem 1856 1 7 1 0 8 14848 0 domainset 40 0 315 106 0 254 12600 0 SMR SHARED 24 7 504 7 0 254 12264 0 AIOP 32 4 374 4 0 254 12096 0 malloc-16 16 15 735 1320 0 254 12000 0 malloc-16 16 326 424 1025 0 254 12000 0 malloc-16 16 5 745 35532 0 254 12000 0 malloc-16 16 244 506 39079 0 254 12000 0 malloc-16 16 17 733 33 0 254 12000 0 malloc-384 384 11 19 11 0 30 11520 0 malloc-384 384 1 29 1 0 30 11520 0 malloc-8192 8192 1 0 1 0 1 8192 0 pcpu-16 16 14 498 14 0 254 8192 0 UMA Slabs 1 176 9 13 9 0 62 3872 0 KMAP ENTRY 96 12 27 14 0 0 3744 0 FFS1 dinode 128 0 0 0 0 126 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 62 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 312 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 48 0 0 0 0 254 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 72 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 tcp_log_node 120 0 0 0 0 126 0 0 tcp_log_bucket 176 0 0 0 0 62 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 ripcb ports 32 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 NCLNODE 608 0 0 0 0 16 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 TMPFS node 224 0 0 0 0 62 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 cryptop 280 0 0 0 0 30 0 0 linux_dma_object 32 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-4096 4096 0 0 0 0 2 0 0 malloc-512 512 0 0 0 0 30 0 0 malloc-512 512 0 0 0 0 30 0 0 malloc-512 512 0 0 0 0 30 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0