panic: Bad tailq NEXT(0xfffffe00a65687f0->tqh_last) != NULL cpuid = 0 time = 1625285956 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x47/frame 0xfffffe009c7444a0 vpanic() at vpanic+0x1cc/frame 0xfffffe009c744500 panic() at panic+0x43/frame 0xfffffe009c744560 sctp_ss_default_add() at sctp_ss_default_add+0x132/frame 0xfffffe009c7445a0 sctp_lower_sosend() at sctp_lower_sosend+0x5522/frame 0xfffffe009c744790 sctp_sosend() at sctp_sosend+0x49b/frame 0xfffffe009c7448d0 sosend() at sosend+0xc6/frame 0xfffffe009c744940 kern_sendit() at kern_sendit+0x33c/frame 0xfffffe009c7449f0 sendit() at sendit+0x23f/frame 0xfffffe009c744a50 sys_sendmsg() at sys_sendmsg+0x8b/frame 0xfffffe009c744ab0 amd64_syscall() at amd64_syscall+0x247/frame 0xfffffe009c744bf0 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe009c744bf0 --- syscall (198, FreeBSD ELF64, nosys), rip = 0x287eda, rsp = 0x7fffdfffdf08, rbp = 0x7fffdfffdf70 --- KDB: enter: panic [ thread pid 15793 tid 122299 ] Stopped at kdb_enter+0x67: movq $0,0x164025e(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0x28 rax 0x12 rcx 0xfffffe0095600000 rdx 0x3ffff rbx 0 rsp 0xfffffe009c744480 rbp 0xfffffe009c7444a0 rsi 0x40001 rdi 0xffffffff81132646 vprintf+0x176 r8 0 r9 0x8080808080808080 r10 0xfffffe009c744370 r11 0x1ff63fff59c r12 0xffffffff82267d50 ddb_dbbe r13 0xffffffff830ce900 _refcount_update_saturated+0xae0 r14 0xffffffff81a74fb1 r15 0xffffffff81a74fb1 rip 0xffffffff811296a7 kdb_enter+0x67 rflags 0x82 kdb_enter+0x67: movq $0,0x164025e(%rip) db> show proc Process 15793 (syz-executor.0) at 0xfffff800655d4538: state: NORMAL uid: 0 gids: 0, 0, 5 parent: pid 783 at 0xfffff80015f62538 ABI: FreeBSD ELF64 flag: 0x10080480 flag2: 0 arguments: /root/syz-executor.0 reaper: 0xfffff80004bda538 reapsubtree: 1 sigparent: 20 vmspace: 0xfffffe009ead39f0 (map 0xfffffe009ead39f0) (map.pmap 0xfffffe009ead3ab0) (pmap 0xfffffe009ead3b10) threads: 2 121808 s syz-executor.0 122299 Run CPU 0 syz-executor.0 db> ps pid ppid pgrp uid state wmesg wchan cmd 15794 814 814 0 R (threaded) syz-executor.3 121851 RunQ syz-executor.3 122301 RunQ syz-executor.3 122303 S connec 0xfffff8003cbbcbf0 syz-executor.3 122304 S uwait 0xfffff80054105f00 syz-executor.3 122306 S uwait 0xfffff8003cf3b500 syz-executor.3 15793 783 783 0 T (threaded) syz-executor.0 121808 s syz-executor.0 122299 Run CPU 0 syz-executor.0 15792 821 821 0 R (threaded) syz-executor.2 119881 RunQ syz-executor.2 122296 S connec 0xfffff80065301840 syz-executor.2 122297 S uwait 0xfffff8006565a100 syz-executor.2 15566 1 783 0 S uwait 0xfffff8003c375300 syz-executor.0 15487 1 783 0 S uwait 0xfffff80065daf500 syz-executor.0 14433 1 814 0 S uwait 0xfffff800652aeb00 syz-executor.3 13159 1 788 0 S uwait 0xfffff8003cc98b00 syz-executor.1 12887 1 821 0 S uwait 0xfffff8003cfc1b00 syz-executor.2 12436 1 783 0 S uwait 0xfffff8003cc98900 syz-executor.0 11276 11275 11275 0 SV uwait 0xfffff80065905b00 syz-executor.1 11275 1 11275 0 RVs syz-executor.1 9832 1 788 0 S uwait 0xfffff8003cfaff00 syz-executor.1 9246 9245 814 0 SV uwait 0xfffff80015d54100 syz-executor.3 9245 1 814 0 DV ppwait 0xfffff8003cbb4a20 syz-executor.3 9242 9234 783 0 SV uwait 0xfffff80065579900 syz-executor.0 9234 1 783 0 RV syz-executor.0 8906 1 788 0 S uwait 0xfffff8003cfc1f00 syz-executor.1 8385 1 783 0 S uwait 0xfffff8006565a700 syz-executor.0 8290 1 8283 0 S uwait 0xfffff8003cec3f00 syz-executor.3 7849 7847 821 0 SV uwait 0xfffff8003cec3100 syz-executor.2 7847 1 821 0 RV syz-executor.2 7616 1 783 0 S uwait 0xfffff80015d57300 syz-executor.0 7363 1 814 0 S uwait 0xfffff8003c552700 syz-executor.3 7362 1 814 0 S uwait 0xfffff80065092700 syz-executor.3 7197 1 814 0 S uwait 0xfffff8003cfaf300 syz-executor.3 6347 1 6347 65 Ss select 0xfffff800652e8bc0 dhclient 5313 1 5313 0 Ss select 0xfffff8006554b140 dhclient 5303 1 5303 0 Ss select 0xfffff800652e8440 dhclient 5268 1 5268 65 Ss select 0xfffff80065295bc0 dhclient 3964 1 3964 0 Ss select 0xfffff8006540e440 dhclient 3957 1 3957 0 Ss select 0xfffff800655a12c0 dhclient 3938 1 3938 65 Ss select 0xfffff800653c0740 dhclient 2820 1 2820 0 Ss select 0xfffff80065252740 dhclient 2817 1 2817 0 Ss select 0xfffff80065270bc0 dhclient 2798 1 2798 65 Ss select 0xfffff8003cebb440 dhclient 2728 1 788 0 S uwait 0xfffff8003cfc1500 syz-executor.1 2727 1 788 0 S uwait 0xfffff8003c552300 syz-executor.1 1916 1 788 0 S uwait 0xfffff8003cec3d00 syz-executor.1 1913 1 788 0 S uwait 0xfffff80015d54300 syz-executor.1 1603 1 788 0 S uwait 0xfffff8003cec3900 syz-executor.1 1600 1 788 0 S uwait 0xfffff8003cfafd00 syz-executor.1 1593 1 783 0 S uwait 0xfffff8003cec3500 syz-executor.0 1591 1 783 0 S umtxn 0xfffff8003cfc1300 syz-executor.0 1479 0 0 0 DL aiordy 0xfffff8003cfed000 [aiod4] 1478 0 0 0 DL aiordy 0xfffff80015e1b538 [aiod3] 1477 0 0 0 DL aiordy 0xfffff8003cd23a70 [aiod2] 1476 0 0 0 DL aiordy 0xfffff8003cd23000 [aiod1] 1291 1 783 0 S uwait 0xfffff8003c375100 syz-executor.0 1290 1 783 0 S uwait 0xfffff8003cfc1700 syz-executor.0 821 780 821 0 Ss nanslp 0xffffffff8273d0a0 syz-executor.2 814 780 814 0 Ss nanslp 0xffffffff8273d0a0 syz-executor.3 812 1 812 0 Ss select 0xfffff8003cabb440 dhclient 809 1 809 0 Ss select 0xfffff8003c3cfec0 dhclient 788 780 788 0 Rs syz-executor.1 783 780 783 0 Ss nanslp 0xffffffff8273d0a0 syz-executor.0 780 778 778 0 R (threaded) syz-fuzzer 100117 S uwait 0xfffff8003c555900 syz-fuzzer 100118 RunQ syz-fuzzer 100119 RunQ syz-fuzzer 100120 S uwait 0xfffff80015d54b00 syz-fuzzer 100121 S uwait 0xfffff80015d54d00 syz-fuzzer 100122 S uwait 0xfffff80015d54f00 syz-fuzzer 100123 S kqread 0xfffff8003c552b00 syz-fuzzer 100124 S uwait 0xfffff80015ac3300 syz-fuzzer 100125 S uwait 0xfffff8003c552f00 syz-fuzzer 778 776 778 0 Ss pause 0xfffff80015e745e8 csh 776 694 776 0 Ss select 0xfffff80015e5b5c0 sshd 760 1 760 0 Ss+ ttyin 0xfffff800154a5cb0 getty 759 1 759 0 Ss+ ttyin 0xfffff80015b2e4b0 getty 758 1 758 0 Ss+ ttyin 0xfffff80015b2ecb0 getty 757 1 757 0 Ss+ ttyin 0xfffff80015ab64b0 getty 756 1 756 0 Ss+ ttyin 0xfffff80015ab6cb0 getty 755 1 755 0 Ss+ ttyin 0xfffff80015ab84b0 getty 754 1 754 0 Ss+ ttyin 0xfffff80015ab8cb0 getty 753 1 753 0 Ss+ ttyin 0xfffff80015abf4b0 getty 752 1 752 0 Ss+ ttyin 0xfffff80015abfcb0 getty 698 1 698 0 Ss nanslp 0xffffffff8273d0a1 cron 694 1 694 0 Ss select 0xfffff8003c44b8c0 sshd 507 1 507 0 Ss select 0xfffff80015d58440 syslogd 436 1 436 0 Ss select 0xfffff80015f17440 devd 435 1 435 65 Ss select 0xfffff8003c44b140 dhclient 350 1 350 0 Ss select 0xfffff80015e5bec0 dhclient 347 1 347 0 Ss select 0xfffff8003c3cfd40 dhclient 23 0 0 0 DL vlruwt 0xfffff80015cd1538 [vnlru] 22 0 0 0 DL syncer 0xffffffff8282c450 [syncer] 21 0 0 0 RL (threaded) [bufdaemon] 100081 RunQ [bufdaemon] 100086 D - 0xffffffff8220aec0 [bufspacedaemon-0] 100098 D sdflush 0xfffff80004ddc4e8 [/ worker] 20 0 0 0 DL psleep 0xffffffff82853388 [vmdaemon] 19 0 0 0 DL (threaded) [pagedaemon] 100079 D psleep 0xffffffff828477f8 [dom0] 100087 D launds 0xffffffff82847804 [laundry: dom0] 100088 D umarcl 0xffffffff815c5980 [uma] 18 0 0 0 DL - 0xffffffff82571458 [rand_harvestq] 17 0 0 0 DL waiting 0xffffffff830df8b8 [sctp_iterator] 16 0 0 0 DL pftm 0xffffffff82d79410 [pf purge] 15 0 0 0 DL - 0xffffffff82828bdc [soaiod4] 9 0 0 0 DL - 0xffffffff82828bdc [soaiod3] 8 0 0 0 DL - 0xffffffff82828bdc [soaiod2] 7 0 0 0 DL - 0xffffffff82828bdc [soaiod1] 6 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff82448b40 [doneq0] 100045 D - 0xffffffff82448ac0 [async] 100078 D - 0xffffffff82448990 [scanner] 14 0 0 0 DL seqstat 0xfffff80004d8a488 [sequencer 00] 5 0 0 0 DL crypto_ 0xfffff80004d53d80 [crypto returns 1] 4 0 0 0 DL crypto_ 0xfffff80004d53d30 [crypto returns 0] 3 0 0 0 DL crypto_ 0xffffffff82844ce0 [crypto] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff8271c8e0 [g_event] 100036 D - 0xffffffff8271c8e8 [g_up] 100037 D - 0xffffffff8271c8f0 [g_down] 2 0 0 0 DL (threaded) [KTLS] 100028 D - 0xfffff80004c69100 [thr_0] 100029 D - 0xfffff80004c69180 [thr_1] 12 0 0 0 WL (threaded) [intr] 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 I [swi3: vm] 100031 I [swi4: clock (0)] 100032 I [swi4: clock (1)] 100033 I [swi1: netisr 0] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100084 I [swi1: hpts] 100085 I [swi1: hpts] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffff80004bda538 [init] 10 0 0 0 DL audit_w 0xffffffff828451f0 [audit] 0 0 0 0 RLs (threaded) [kernel] 100000 D swapin 0xffffffff8271ce70 [swapper] 100005 D - 0xfffff8000443b500 [if_io_tqg_0] 100006 D - 0xfffff8000443b100 [if_io_tqg_1] 100007 D - 0xfffff80004c78d00 [if_config_tqg_0] 100008 D - 0xfffff80004c78900 [softirq_0] 100009 D - 0xfffff80004c78500 [softirq_1] 100010 D - 0xfffff80004c78100 [aiod_kick taskq] 100012 D - 0xfffff80004c75900 [kqueue_ctx taskq] 100013 D - 0xfffff80004c75500 [pci_hp taskq] 100015 D - 0xfffff80004c6ed00 [inm_free taskq] 100017 D - 0xfffff80004c6e500 [linuxkpi_irq_wq] 100018 Run CPU 1 [thread taskq] 100019 D - 0xfffff80004c69d00 [in6m_free taskq] 100020 D - 0xfffff80004c69900 [linuxkpi_short_wq_0] 100021 D - 0xfffff80004c69900 [linuxkpi_short_wq_1] 100022 D - 0xfffff80004c69900 [linuxkpi_short_wq_2] 100023 D - 0xfffff80004c69900 [linuxkpi_short_wq_3] 100024 D - 0xfffff80004c69500 [linuxkpi_long_wq_0] 100025 D - 0xfffff80004c69500 [linuxkpi_long_wq_1] 100026 D - 0xfffff80004c69500 [linuxkpi_long_wq_2] 100027 D - 0xfffff80004c69500 [linuxkpi_long_wq_3] 100034 D - 0xfffff80004cd3d00 [firmware taskq] 100038 D - 0xfffff80004cd3900 [crypto_0] 100039 D - 0xfffff80004cd3900 [crypto_1] 100055 D - 0xfffff800153ebd00 [vtnet0 rxq 0] 100056 D - 0xfffff800153eb900 [vtnet0 txq 0] 100057 D - 0xfffff800153eb500 [vtnet0 rxq 1] 100058 D - 0xfffff800153eb100 [vtnet0 txq 1] 100060 D vtbslp 0xfffff80015463700 [virtio_balloon] 100064 D - 0xfffff80015468900 [mca taskq] 100066 D - 0xffffffff81e218d0 [deadlkres] 100074 D - 0xfffff80015a47100 [acpi_task_0] 100075 D - 0xfffff80015a47100 [acpi_task_1] 100076 D - 0xfffff80015a47100 [acpi_task_2] 100077 D - 0xfffff80004cd3500 [CAM taskq] 7848 7847 821 0 Z syz-executor.2 db> show all locks Process 15793 (syz-executor.0) thread 0xfffffe00a19a1c80 (122299) exclusive sleep mutex sctp-send-tcb (tcbs) r = 0 (0xfffffe00a6568c90) locked @ /syzkaller/managers/main/kernel/sys/netinet/sctp_output.c:13125 Process 0 (kernel) thread 0xfffffe00042b6720 (100018) exclusive sleep mutex kernel arena (kernel arena) r = 0 (0xffffffff82786cc0) locked @ /syzkaller/managers/main/kernel/sys/kern/subr_vmem.c:789 exclusive sleep mutex vmem list lock (vmem list lock) r = 0 (0xffffffff82202b40) locked @ /syzkaller/managers/main/kernel/sys/kern/subr_vmem.c:784 db> show malloc Type InUse MemUse Requests sctp_stro 302 92386K 2996 sysctloid 34288 12858K 34355 pf_hash 5 11560K 5 devbuf 4216 6982K 4244 pcb 646 3362K 31058 tcp_hpts 5 3219K 5 kobj 326 2608K 486 vtbuf 24 2064K 46 filedesc 80 1279K 29483 newblk 11 1037K 16600 vfscache 3 1035K 3 subproc 226 919K 15928 acpica 1674 649K 54806 sctp_atcl 607 607K 11870 inodedep 72 591K 14868 callout 2 528K 2 ufs_quota 1 520K 1 vfs_hash 1 520K 1 intr 4 480K 4 bus 986 377K 3469 sctp_atky 911 342K 13972 BPF 46 337K 63 linker 348 270K 425 kdtrace 307 246K 38102 sctp_athm 607 228K 11874 sctp_map 604 227K 4120 DEVFS1 107 214K 124 umtx 462 203K 462 module 510 192K 510 vnet_data 1 176K 1 vmem 3 168K 9 tidhash 3 164K 3 pagedep 11 141K 14689 tfo_ccache 1 136K 1 sem 4 120K 4 UMA 270 102K 270 audit_evclass 236 89K 294 temp 36 81K 3167 mtx_pool 2 80K 2 syncache 1 76K 1 routetbl 138 76K 430 msg 4 68K 4 ufs_dirhash 105 68K 105 ifaddr 70 66K 72 acpitask 1 64K 1 ddb_capture 1 64K 1 DEVFS3 126 63K 136 ether_multi 152 57K 162 gtaskqueue 18 57K 18 DEVFS_RULE 56 54K 56 kenv 95 52K 95 eventhandler 133 50K 133 kqueue 101 46K 15838 ithread 99 43K 99 rman 84 42K 425 KTRACE 101 38K 1624 lltable 43 37K 162 pwddesc 98 37K 15795 taskqueue 60 36K 60 proc 3 34K 3 ufs_mount 5 34K 6 bus-sc 33 34K 1684 devstat 4 33K 4 hostcache 1 32K 1 tty 16 32K 16 shm 1 32K 20 dirrem 63 32K 14751 DEVFSP 80 30K 330 GEOM 60 29K 489 in6_multi 65 29K 65 cred 28 28K 402 kbdmux 6 28K 6 ksem 7 26K 1704 ifnet 7 25K 7 plimit 24 24K 537 kcovinfo 64 24K 68 freefile 61 23K 14720 inpcbpolicy 59 23K 22345 CAM queue 5 21K 1528 selfd 55 21K 232337 pfs_nodes 20 20K 20 proc-args 52 20K 801 session 37 19K 64 UART 12 18K 12 sctp_timw 33 17K 33 bmsafemap 2 17K 14846 rpc 2 16K 2 shmfd 1 16K 35 pfs_vncache 1 16K 1 osd 40 16K 19788 sglist 5 13K 5 lockf 34 13K 83 MCA 32 12K 32 CAM DEV 3 12K 510 vt 11 11K 11 Unitno 29 11K 47 acpisem 28 11K 28 CAM XPT 22 11K 543 pf_ifnet 13 10K 23 uidinfo 4 10K 35 pf_table 2 8K 3 memdesc 1 8K 1 ipsec-saq 2 8K 2 evdev 4 8K 4 acpidev 20 8K 20 hhook 15 8K 17 select 19 8K 120 mount 16 7K 108 fpukern_ctx 3 6K 3 terminal 11 6K 11 ip6ndp 12 6K 13 sctp_ifa 13 5K 14 tcp_fsb 12 5K 4118 ipsecpolicy 2 5K 2 encap_export_host 12 5K 12 clone 9 5K 9 local_apic 1 4K 1 io_apic 1 4K 1 sahead 1 4K 1 secasvar 1 4K 1 nhops 6 4K 8 pci_link 10 4K 10 msi 9 4K 9 DEVFS 9 4K 10 tun 7 4K 7 ipsec 3 3K 3 mkdir 8 3K 29320 in_multi 6 3K 8 soname 8 3K 51553 nexusdev 7 3K 7 feeder 7 3K 7 procdesc 5 3K 14 freework 5 3K 14712 sctp_ifn 6 3K 14 newdirblk 6 3K 14660 mld 6 3K 6 igmp 6 3K 6 toponodes 6 3K 6 prison 6 3K 6 isadev 6 3K 6 softdep 1 2K 1 freeblks 4 2K 14675 vnodemarker 2 2K 102 NFSD session 1 2K 1 lkpikmalloc 5 2K 6 CAM periph 4 2K 271 filecaps 5 2K 113 crypto 4 2K 58 ip6opt 4 2K 1003 diradd 4 2K 14795 pfil 4 2K 4 CAM path 4 2K 1034 cubic data 3 2K 859 tcpfunc 3 2K 3 loginclass 3 2K 6 chacha20random 1 1K 1 vnodes 1 1K 1 ktls 1 1K 1 cdev 2 1K 2 aesni_data 2 1K 2 cdg data 2 1K 327 pf_rule 2 1K 4 pf_osfp 2 1K 2 cpus 2 1K 2 atkbddev 2 1K 2 CAM dev queue 2 1K 2 xform 2 1K 456 entropy 2 1K 55 CAM SIM 2 1K 2 iov 2 1K 47721 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 pmchooks 1 1K 1 sctp_vrf 1 1K 1 chd data 1 1K 406 apmdev 1 1K 1 CAM I/O Scheduler 1 1K 1 vnet_data_free 1 1K 1 vnet 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 acpiintr 1 1K 1 pmc 1 1K 1 sctp_mcore 0 0K 0 sctp_socko 0 0K 2190 sctp_iter 0 0K 10 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 0 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 10 sctp_aadr 0 0K 1 sctp_stri 0 0K 12 mqdata 0 0K 0 dctcp data 0 0K 382 vegas data 0 0K 313 htcp data 0 0K 401 tcp_do 0 0K 0 pf_altq 0 0K 0 pf_temp 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 iavf 0 0K 0 ixl 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 axgbe 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xen_intr 0 0K 0 xenstore 0 0K 0 ciss_data 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 qpidrv 0 0K 0 BACKLIGHT 0 0K 0 xnb 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 sysmouse 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 vtfont 0 0K 0 vm_fictitious 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 amr 0 0K 0 pvscsi 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 UMAHash 0 0K 0 scsi_cd 0 0K 0 AHCI driver 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 13311 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 48 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 20 allocindir 0 0K 0 indirdep 0 0K 14 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 agp 0 0K 0 nvme_da 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 ktls_ocf 0 0K 0 twsbuf 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpifw 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 twe_commands 0 0K 0 LRO 0 0K 0 newreno data 0 0K 2 ip_msource 0 0K 0 ip_moptions 0 0K 6 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 0 VN POLL 0 0K 0 twa_commands 0 0K 0 statfs 0 0K 16949 namei_tracker 0 0K 0 export_host 0 0K 0 cl_savebuf 0 0K 8 tcp_log_dev 0 0K 1084 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 9953 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 biobuf 0 0K 0 aios 0 0K 2 lio 0 0K 6 acl 0 0K 0 tempbuff 0 0K 0 mbuf_tag 0 0K 157 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 accf 0 0K 0 pts 0 0K 0 ioctlops 0 0K 291 eventfd 0 0K 43 Witness 0 0K 0 stack 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 sbuf 0 0K 304 md_sectors 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 md_disk 0 0K 0 SWAP 0 0K 0 malodev 0 0K 0 LED 0 0K 0 sysctltmp 0 0K 974 sysctl 0 0K 3 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 ix_sriov 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 cache 0 0K 0 iirbuf 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 1 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8324 981 1372292 0 254 38113280 0 FFS inode 1160 22436 27 37159 0 8 26057080 0 malloc-384 384 36109 401 45513 0 30 14019840 0 malloc-32768 32768 236 141 819 0 1 12353536 0 VNODE 448 22475 70 37200 0 30 10100160 0 FFS2 dinode 256 22436 19 37157 0 62 5748480 0 malloc-1024 1024 4174 10 4211 0 16 4284416 0 malloc-8192 8192 307 196 9802 0 1 4120576 0 S VFS Cache 104 33552 339 52115 0 126 3524664 0 RADIX NODE 144 21896 305 418619 0 63 3196944 0 malloc-8192 8192 328 2 1099 0 1 2703360 0 pbuf 2624 0 973 0 0 2 2553152 0 tcp_log 416 165 5946 195698 0 254 2542176 0 mbuf 256 9201 714 3467716 0 254 2538240 0 VM OBJECT 264 8114 136 236594 0 30 2178000 0 BUF TRIE 144 335 13133 1847 0 62 1939392 0 UMA Slabs 0 112 14141 4 14141 0 126 1584240 0 malloc-32768 32768 41 5 14745 0 1 1507328 0 malloc-384 384 2662 968 295895 0 30 1393920 0 malloc-1024 1024 149 1099 15646 0 16 1277952 0 sctp_asoc 2288 302 233 2060 0 254 1224080 0 mbuf_cluster 2048 570 2 570 0 254 1171456 0 malloc-1024 1024 612 396 11907 0 16 1032192 0 malloc-8192 8192 101 16 15797 0 1 958464 0 malloc-1024 1024 135 705 15169 0 16 860160 0 sctp_ep 1280 305 277 9797 0 254 744960 0 malloc-1024 1024 526 194 32286 0 16 737280 0 malloc-384 384 1775 15 54266 0 30 687360 0 malloc-512 512 205 1075 15349 0 30 655360 0 malloc-384 384 639 1031 125914 0 30 641280 0 vmem btag 56 9208 140 9208 0 254 523488 0 malloc-4096 4096 9 114 10917 0 3 503808 0 tcp_bbr_map 128 15 3395 145216 0 126 436480 0 malloc-16384 16384 24 2 29 0 1 425984 0 THREAD 1808 207 24 22306 0 8 417648 0 sctp_raddr 736 302 237 2060 0 254 396704 0 malloc-65536 65536 5 1 6 0 1 393216 0 lkpimm 160 1 2324 1 0 62 372000 0 lkpicurr 160 2 2323 2 0 62 372000 0 MAP ENTRY 96 3166 572 859295 0 126 358848 0 256 Bucket 2048 149 17 22577 0 8 339968 0 DIRHASH 1024 309 3 309 0 16 319488 0 malloc-512 512 33 583 17026 0 30 315392 0 ertt_txseginfo 40 1 7574 910468 0 254 303000 0 malloc-384 384 659 71 23092 0 30 280320 0 tcp_inpcb 488 36 508 19776 0 254 265472 0 malloc-65536 65536 0 4 6 0 1 262144 0 malloc-2048 2048 125 3 146 0 8 262144 0 socket 944 102 146 33793 0 254 234112 0 DEVCTL 1024 0 220 153 0 0 225280 0 VMSPACE 2544 67 20 15761 0 4 221328 0 malloc-8192 8192 22 4 46 0 1 212992 0 malloc-65536 65536 3 0 3 0 1 196608 0 malloc-16384 16384 11 1 247 0 1 196608 0 tcpcb 1080 36 146 19776 0 254 196560 0 g_bio 408 0 470 36884 0 30 191760 0 UMA Zones 768 242 2 242 0 16 187392 0 malloc-384 384 403 67 73278 0 30 180480 0 malloc-1024 1024 168 4 179 0 16 176128 0 PROC 1336 98 28 15794 0 8 168336 0 malloc-384 384 361 69 17645 0 30 165120 0 malloc-16384 16384 4 5 19 0 1 147456 0 malloc-2048 2048 31 41 3514 0 8 147456 0 mbuf_packet 256 45 525 42128 0 254 145920 0 malloc-512 512 251 29 14999 0 30 143360 0 malloc-512 512 198 66 35454 0 30 135168 0 filedesc0 1072 98 28 15795 0 8 135072 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-65536 65536 0 2 176 0 1 131072 0 ksiginfo 112 111 933 2805 0 126 116928 0 128 Bucket 1024 67 36 4434 0 16 105472 0 sctp_chunk 152 383 293 1361 0 254 102752 0 32 Bucket 256 230 160 20442 0 62 99840 0 malloc-32768 32768 1 2 121 0 1 98304 0 malloc-16384 16384 2 4 32 0 1 98304 0 malloc-512 512 123 53 15672 0 30 90112 0 UMA Kegs 384 227 6 227 0 30 89472 0 clpbuf 2624 0 32 60 0 16 83968 0 malloc-384 384 108 72 21629 0 30 69120 0 malloc-16384 16384 2 2 7 0 1 65536 0 malloc-4096 4096 13 3 91 0 2 65536 0 malloc-2048 2048 10 20 523 0 8 61440 0 malloc-8192 8192 3 4 26 0 1 57344 0 malloc-8192 8192 2 5 16984 0 1 57344 0 malloc-4096 4096 7 6 33 0 2 53248 0 16 Bucket 144 199 165 4713 0 62 52416 0 malloc-16384 16384 3 0 3 0 1 49152 0 malloc-16384 16384 3 0 3 0 1 49152 0 malloc-8192 8192 1 5 107 0 1 49152 0 64 Bucket 512 48 48 11845 0 30 49152 0 malloc-4096 4096 6 5 110 0 2 45056 0 tcp_rack_pcb 832 12 42 4118 0 16 44928 0 malloc-2048 2048 16 4 24 0 8 40960 0 malloc-512 512 25 55 1443 0 30 40960 0 pcpu-8 8 4745 375 5050 0 254 40960 0 Files 80 294 206 70007 0 126 40000 0 tcp_bbr_pcb 832 6 39 7101 0 16 37440 0 pipe 744 23 27 665 0 16 37200 0 NAMEI 1024 0 36 141299 0 16 36864 0 malloc-512 512 34 38 239 0 30 36864 0 TURNSTILE 136 232 20 232 0 62 34272 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-512 512 48 16 240 0 30 32768 0 pcpu-64 64 487 25 487 0 254 32768 0 udplite_inpcb 488 12 52 1782 0 254 31232 0 malloc-4096 4096 5 2 1232 0 2 28672 0 malloc-2048 2048 2 12 152 0 8 28672 0 4 Bucket 48 36 552 1663 0 254 28224 0 KNOTE 160 28 147 303341 0 62 28000 0 L VFS Cache 320 71 13 71 0 30 26880 0 malloc-8192 8192 0 3 13 0 1 24576 0 malloc-4096 4096 1 5 10 0 2 24576 0 PWD 32 43 713 18822 0 254 24192 0 ttyinq 160 135 15 300 0 62 24000 0 8 Bucket 80 52 248 6866 0 126 24000 0 ttyoutq 256 72 18 160 0 62 23040 0 SLEEPQUEUE 88 232 24 232 0 126 22528 0 tcp_rack_map 112 28 152 7207 0 126 20160 0 ertt 72 36 244 19776 0 126 20160 0 2 Bucket 32 256 374 4471 0 254 20160 0 Mountpoints 2752 2 5 2 0 4 19264 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-4096 4096 2 2 113 0 2 16384 0 malloc-2048 2048 7 1 7 0 8 16384 0 vtnet_tx_hdr 24 0 668 1447294 0 254 16032 0 ripcb 488 5 27 397 0 254 15616 0 malloc-2048 2048 3 3 163 0 8 12288 0 malloc-2048 2048 2 4 6 0 8 12288 0 sctp_stream_msg_out 112 9 99 1370 0 254 12096 0 udp_inpcb 488 6 18 390 0 254 11712 0 kenv 258 15 30 1038 0 30 11610 0 routing nhops 256 27 18 34 0 62 11520 0 unpcb 256 25 20 1599 0 254 11520 0 malloc-1024 1024 1 7 11 0 16 8192 0 rtentry 176 30 16 34 0 62 8096 0 tcptw 88 0 92 9 0 254 8096 0 PGRP 88 37 55 65 0 126 8096 0 rl_entry 40 84 118 84 0 254 8080 0 sctp_laddr 48 0 168 815 0 254 8064 0 udpcb 32 18 234 2172 0 254 8064 0 AIOP 32 4 248 4 0 254 8064 0 sctp_readq 152 0 52 7 0 254 7904 0 AIO 208 2 36 165 0 62 7904 0 itimer 352 0 22 19 0 30 7744 0 AIOCB 552 0 14 190 0 16 7728 0 AIOLIO 272 0 28 6 0 30 7616 0 FPU_save_area 832 1 8 1 0 16 7488 0 cpuset 104 7 55 101 0 126 6448 0 domainset 40 0 126 94 0 254 5040 0 epoch_record pcpu 256 4 12 4 0 62 4096 0 malloc-4096 4096 1 0 1 0 2 4096 0 pcpu-16 16 7 249 7 0 254 4096 0 tcp_log_bucket 176 0 23 1 0 62 4048 0 hostcache 64 1 62 1 0 254 4032 0 syncache 168 0 24 4 0 254 4032 0 ipq 56 0 72 1 0 254 4032 0 tcp_log_node 120 0 33 5 0 126 3960 0 UMA Slabs 1 176 9 13 9 0 62 3872 0 mqnode 416 3 6 3 0 30 3744 0 KMAP ENTRY 96 12 27 12 0 0 3744 0 vmem 1856 1 1 1 0 8 3712 0 SMR CPU 32 3 60 3 0 254 2016 0 SMR SHARED 24 3 60 3 0 254 1512 0 FFS1 dinode 128 0 0 0 0 126 0 0 da_ccb 544 0 0 0 0 16 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 62 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 304 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 48 0 0 0 0 254 0 0 tcpreass 48 0 0 0 0 254 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 NCLNODE 584 0 0 0 0 16 0 0 TMPFS node 224 0 0 0 0 62 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 cryptop 280 0 0 0 0 30 0 0 linux_dma_object 24 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 ktls_session 128 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-1024 1024 0 0 0 0 16 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0