panic: sx_xlock() of destroyed sx @ /syzkaller/managers/main/kernel/sys/kern/uipc_sockbuf.c:467 cpuid = 0 time = 267 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x47/frame 0xfffffe00517816d0 vpanic() at vpanic+0x1cc/frame 0xfffffe0051781730 panic() at panic+0x43/frame 0xfffffe0051781790 _sx_xlock() at _sx_xlock+0x19a/frame 0xfffffe00517817e0 sosend_generic() at sosend_generic+0x1e0/frame 0xfffffe00517818d0 sosend() at sosend+0xc6/frame 0xfffffe0051781940 kern_sendit() at kern_sendit+0x33c/frame 0xfffffe00517819f0 sendit() at sendit+0x23f/frame 0xfffffe0051781a50 sys_sendto() at sys_sendto+0x5c/frame 0xfffffe0051781ab0 amd64_syscall() at amd64_syscall+0x247/frame 0xfffffe0051781bf0 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0051781bf0 --- syscall (198, FreeBSD ELF64, nosys), rip = 0x286eda, rsp = 0x7fffdff58f08, rbp = 0x7fffdff58f70 --- KDB: enter: panic [ thread pid 16359 tid 120456 ] Stopped at kdb_enter+0x67: movq $0,0x164006e(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0x28 rax 0x12 rcx 0xfffffe009c000000 rdx 0x3ffff rbx 0 rsp 0xfffffe00517816b0 rbp 0xfffffe00517816d0 rsi 0x40001 rdi 0xffffffff81132836 vprintf+0x176 r8 0 r9 0x8080808080808080 r10 0xfffffe00517815a0 r11 0x1ffaefff59c r12 0xffffffff82267d30 ddb_dbbe r13 0xffffffff81a54f00 r14 0xffffffff81a73f45 r15 0xffffffff81a73f45 rip 0xffffffff81129897 kdb_enter+0x67 rflags 0x82 kdb_enter+0x67: movq $0,0x164006e(%rip) db> show proc Process 16359 (syz-executor.3) at 0xfffff80026273a60: state: NORMAL uid: 0 gids: 0, 0, 5 parent: pid 823 at 0xfffff80015e6ca60 ABI: FreeBSD ELF64 flag: 0x10000080 flag2: 0 arguments: /root/syz-executor.3 reaper: 0xfffff80004bd7530 reapsubtree: 1 sigparent: 20 vmspace: 0xfffffe00950dc000 (map 0xfffffe00950dc000) (map.pmap 0xfffffe00950dc0c0) (pmap 0xfffffe00950dc120) threads: 7 116513 RunQ syz-executor.3 120444 S sbwait 0xfffff80026674924 syz-executor.3 120452 S sbwait 0xfffff8002662d924 syz-executor.3 120453 S sbwait 0xfffff80026c001c4 syz-executor.3 120454 S sbwait 0xfffff80026750cd4 syz-executor.3 120455 Run CPU 1 syz-executor.3 120456 Run CPU 0 syz-executor.3 db> ps pid ppid pgrp uid state wmesg wchan cmd 16359 823 823 0 R (threaded) syz-executor.3 116513 RunQ syz-executor.3 120444 S sbwait 0xfffff80026674924 syz-executor.3 120452 S sbwait 0xfffff8002662d924 syz-executor.3 120453 S sbwait 0xfffff80026c001c4 syz-executor.3 120454 S sbwait 0xfffff80026750cd4 syz-executor.3 120455 Run CPU 1 syz-executor.3 120456 Run CPU 0 syz-executor.3 14347 1 823 0 S uwait 0xfffff8002641eb00 syz-executor.3 14344 1 823 0 S uwait 0xfffff80036cae500 syz-executor.3 8737 0 0 0 DL aiordy 0xfffff8002639b000 [aiod4] 8736 0 0 0 DL aiordy 0xfffff80036dcba60 [aiod3] 8735 0 0 0 DL aiordy 0xfffff80036ff2530 [aiod2] 8734 0 0 0 DL aiordy 0xfffff80036da9000 [aiod1] 2859 1 2859 65 Ss select 0xfffff80026198ec0 dhclient 2338 1 2338 0 Ss select 0xfffff8002620dec0 dhclient 2335 1 2335 0 Ss select 0xfffff800261dc2c0 dhclient 2315 1 2315 65 Ss select 0xfffff80036a665c0 dhclient 1973 1 1973 0 Ss select 0xfffff800261c6740 dhclient 1970 1 1970 0 Ss select 0xfffff80036a662c0 dhclient 1951 1 1951 65 Ss select 0xfffff80026212740 dhclient 1498 1 1498 0 Ss select 0xfffff8002615e140 dhclient 1495 1 1495 0 Ss select 0xfffff8002626aec0 dhclient 1473 1 1473 65 Ss select 0xfffff8002629d2c0 dhclient 823 780 823 0 Ss nanslp 0xffffffff8273d0a1 syz-executor.3 821 1 821 0 Ss select 0xfffff80036d4f2c0 dhclient 818 1 818 0 Ss select 0xfffff80036d4f8c0 dhclient 806 780 806 0 Ss piperd 0xfffff80036c04ba0 syz-executor.2 786 780 786 0 Ss piperd 0xfffff800362f38b8 syz-executor.1 785 780 785 0 Ss piperd 0xfffff800361f1000 syz-executor.0 780 778 778 0 S (threaded) syz-fuzzer 100095 S uwait 0xfffff80015e71500 syz-fuzzer 100115 S uwait 0xfffff800364b0300 syz-fuzzer 100116 S uwait 0xfffff800364b0500 syz-fuzzer 100117 S uwait 0xfffff800364b0700 syz-fuzzer 100118 S uwait 0xfffff80036a25900 syz-fuzzer 100119 S uwait 0xfffff80015909100 syz-fuzzer 100120 S uwait 0xfffff80015909300 syz-fuzzer 100121 S uwait 0xfffff80036a25b00 syz-fuzzer 103223 S uwait 0xfffff80036e1a100 syz-fuzzer 117975 S kqread 0xfffff8003643bf00 syz-fuzzer 778 776 778 0 Ss pause 0xfffff80015e00b10 csh 776 694 776 0 Ss select 0xfffff80015dec140 sshd 760 1 760 0 Ss+ ttyin 0xfffff800154804b0 getty 759 1 759 0 Ss+ ttyin 0xfffff80015aa2cb0 getty 758 1 758 0 Ss+ ttyin 0xfffff80015aa44b0 getty 757 1 757 0 Ss+ ttyin 0xfffff80015aa4cb0 getty 756 1 756 0 Ss+ ttyin 0xfffff80015aa94b0 getty 755 1 755 0 Ss+ ttyin 0xfffff80015aa9cb0 getty 754 1 754 0 Ss+ ttyin 0xfffff80015aac4b0 getty 753 1 753 0 Ss+ ttyin 0xfffff80015aaccb0 getty 752 1 752 0 Ss+ ttyin 0xfffff800154644b0 getty 698 1 698 0 Ss nanslp 0xffffffff8273d0a1 cron 694 1 694 0 Ss select 0xfffff800363eb5c0 sshd 507 1 507 0 Ss select 0xfffff80015dabbc0 syslogd 436 1 436 0 Ss select 0xfffff80015dec8c0 devd 435 1 435 65 Ss select 0xfffff80015dec2c0 dhclient 350 1 350 0 Ss select 0xfffff80015dec5c0 dhclient 347 1 347 0 Ss select 0xfffff80015daf440 dhclient 23 0 0 0 DL vlruwt 0xfffff80015cc0530 [vnlru] 22 0 0 0 DL syncer 0xffffffff8282c450 [syncer] 21 0 0 0 DL (threaded) [bufdaemon] 100081 D qsleep 0xffffffff8282b520 [bufdaemon] 100086 D - 0xffffffff8220aec0 [bufspacedaemon-0] 100097 D sdflush 0xfffff80015e48ce8 [/ worker] 20 0 0 0 DL psleep 0xffffffff82853388 [vmdaemon] 19 0 0 0 DL (threaded) [pagedaemon] 100079 D psleep 0xffffffff828477f8 [dom0] 100087 D launds 0xffffffff82847804 [laundry: dom0] 100088 D umarcl 0xffffffff815c42c0 [uma] 18 0 0 0 DL - 0xffffffff82571458 [rand_harvestq] 17 0 0 0 DL waiting 0xffffffff82fed8b8 [sctp_iterator] 16 0 0 0 DL pftm 0xffffffff830e7410 [pf purge] 15 0 0 0 DL - 0xffffffff82828bdc [soaiod4] 9 0 0 0 DL - 0xffffffff82828bdc [soaiod3] 8 0 0 0 DL - 0xffffffff82828bdc [soaiod2] 7 0 0 0 DL - 0xffffffff82828bdc [soaiod1] 6 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff82448b40 [doneq0] 100045 D - 0xffffffff82448ac0 [async] 100078 D - 0xffffffff82448990 [scanner] 14 0 0 0 DL seqstat 0xfffff80004d84c88 [sequencer 00] 5 0 0 0 DL crypto_ 0xfffff80004d4dd80 [crypto returns 1] 4 0 0 0 DL crypto_ 0xfffff80004d4dd30 [crypto returns 0] 3 0 0 0 DL crypto_ 0xffffffff82844ce0 [crypto] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff8271c8e0 [g_event] 100036 D - 0xffffffff8271c8e8 [g_up] 100037 D - 0xffffffff8271c8f0 [g_down] 2 0 0 0 DL (threaded) [KTLS] 100028 D - 0xfffff80004c71500 [thr_0] 100029 D - 0xfffff80004c71580 [thr_1] 12 0 0 0 WL (threaded) [intr] 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 I [swi3: vm] 100031 I [swi4: clock (0)] 100032 I [swi4: clock (1)] 100033 I [swi1: netisr 0] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100084 I [swi1: hpts] 100085 I [swi1: hpts] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffff80004bd7530 [init] 10 0 0 0 DL audit_w 0xffffffff828451f0 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff8271ce70 [swapper] 100005 D - 0xfffff80004c7a900 [if_io_tqg_0] 100006 D - 0xfffff80004c7a500 [if_io_tqg_1] 100007 D - 0xfffff80004c7a100 [if_config_tqg_0] 100008 D - 0xfffff80004c71d00 [softirq_0] 100009 D - 0xfffff80004c71900 [softirq_1] 100010 D - 0xfffff8000414f900 [aiod_kick taskq] 100012 D - 0xfffff8000414f100 [kqueue_ctx taskq] 100013 D - 0xfffff80004c69d00 [pci_hp taskq] 100015 D - 0xfffff80004c69500 [inm_free taskq] 100017 D - 0xfffff80004c66d00 [linuxkpi_irq_wq] 100018 D - 0xfffff80004c66900 [thread taskq] 100019 D - 0xfffff80004c66500 [in6m_free taskq] 100020 D - 0xfffff80004c66100 [linuxkpi_short_wq_0] 100021 D - 0xfffff80004c66100 [linuxkpi_short_wq_1] 100022 D - 0xfffff80004c66100 [linuxkpi_short_wq_2] 100023 D - 0xfffff80004c66100 [linuxkpi_short_wq_3] 100024 D - 0xfffff80004c5dd00 [linuxkpi_long_wq_0] 100025 D - 0xfffff80004c5dd00 [linuxkpi_long_wq_1] 100026 D - 0xfffff80004c5dd00 [linuxkpi_long_wq_2] 100027 D - 0xfffff80004c5dd00 [linuxkpi_long_wq_3] 100034 D - 0xfffff80004c5d900 [firmware taskq] 100038 D - 0xfffff80004c5d500 [crypto_0] 100039 D - 0xfffff80004c5d500 [crypto_1] 100055 D - 0xfffff800153e2d00 [vtnet0 rxq 0] 100056 D - 0xfffff800153e2900 [vtnet0 txq 0] 100057 D - 0xfffff800153e2500 [vtnet0 rxq 1] 100058 D - 0xfffff800153e2100 [vtnet0 txq 1] 100060 D vtbslp 0xfffff8001543c900 [virtio_balloon] 100064 D - 0xfffff800154bfd00 [mca taskq] 100066 D - 0xffffffff81e206e0 [deadlkres] 100074 D - 0xfffff800154bf900 [acpi_task_0] 100075 D - 0xfffff800154bf900 [acpi_task_1] 100076 D - 0xfffff800154bf900 [acpi_task_2] 100077 D - 0xfffff80004c5d100 [CAM taskq] db> show all locks Process 16359 (syz-executor.3) thread 0xfffffe0095033720 (120455) exclusive sleep mutex socket (socket) r = 0 (0xfffff800266b63b0) locked @ /syzkaller/managers/main/kernel/sys/netinet/tcp_usrreq.c:457 exclusive rw tcpinp (tcpinp) r = 0 (0xfffff800265aa3f0) locked @ /syzkaller/managers/main/kernel/sys/netinet/tcp_usrreq.c:450 db> show malloc Type InUse MemUse Requests sysctloid 34288 12858K 34355 pf_hash 5 11560K 5 devbuf 4216 6982K 4244 pcb 1252 5932K 32605 tcp_hpts 5 3219K 5 kobj 326 2608K 486 vtbuf 24 2064K 46 sctp_atcl 1178 1178K 15698 sctp_stro 589 1178K 3823 newblk 9 1036K 16065 vfscache 3 1035K 3 inodedep 152 671K 15526 sctp_atky 1775 669K 19705 acpica 1674 649K 54806 subproc 151 613K 16452 callout 2 528K 2 ufs_quota 1 520K 1 vfs_hash 1 520K 1 intr 4 480K 4 sctp_athm 1178 442K 15822 sctp_map 1178 442K 7646 bus 986 377K 3469 BPF 46 337K 89 linker 348 270K 409 filedesc 16 243K 30879 DEVFS1 107 214K 124 module 510 192K 510 kdtrace 232 191K 36823 vnet_data 1 176K 1 tidhash 3 164K 3 umtx 374 164K 374 pagedep 9 140K 15446 tfo_ccache 1 136K 1 sem 4 120K 4 vmem 3 104K 6 UMA 270 102K 270 audit_evclass 236 89K 294 mtx_pool 2 80K 2 temp 35 77K 2542 syncache 1 76K 1 dirrem 143 72K 15442 routetbl 127 70K 406 msg 4 68K 4 ifaddr 70 66K 72 acpitask 1 64K 1 ddb_capture 1 64K 1 DEVFS3 126 63K 136 ether_multi 152 57K 162 gtaskqueue 18 57K 18 sctp_timw 112 56K 112 freefile 143 54K 15439 DEVFS_RULE 56 54K 56 kenv 95 52K 95 eventhandler 133 50K 133 ithread 99 43K 99 rman 84 42K 425 lltable 47 41K 106 KTRACE 100 38K 100 taskqueue 60 36K 60 proc 3 34K 3 ufs_mount 5 34K 6 bus-sc 33 34K 1684 kqueue 68 34K 16388 devstat 4 33K 4 hostcache 1 32K 1 tty 16 32K 16 shm 1 32K 4 DEVFSP 80 30K 550 GEOM 60 29K 489 in6_multi 65 29K 65 kbdmux 6 28K 6 cred 26 26K 280 pwddesc 66 25K 16366 ifnet 7 25K 7 inpcbpolicy 65 25K 18446 plimit 24 24K 486 kcovinfo 64 24K 68 CAM queue 5 21K 1528 selfd 55 21K 230233 osd 54 21K 14620 pfs_nodes 20 20K 20 proc-args 52 20K 682 UART 12 18K 12 session 35 18K 51 bmsafemap 2 17K 15518 ksem 1 16K 2153 rpc 2 16K 2 shmfd 1 16K 13 pfs_vncache 1 16K 1 ufs_dirhash 24 14K 24 sglist 5 13K 5 MCA 32 12K 32 CAM DEV 3 12K 510 lockf 32 12K 63 vt 11 11K 11 acpisem 28 11K 28 CAM XPT 22 11K 543 Unitno 27 11K 47 uidinfo 3 9K 12 pf_ifnet 10 9K 19 pf_rule 2 8K 2 memdesc 1 8K 1 ipsec-saq 2 8K 2 evdev 4 8K 4 acpidev 20 8K 20 hhook 15 8K 17 select 19 8K 58 mount 16 7K 90 fpukern_ctx 3 6K 3 terminal 11 6K 11 ip6ndp 12 6K 13 sctp_ifa 13 5K 14 ipsecpolicy 2 5K 2 encap_export_host 12 5K 12 clone 9 5K 9 pf_table 1 4K 1 local_apic 1 4K 1 io_apic 1 4K 1 sahead 1 4K 1 secasvar 1 4K 1 nhops 6 4K 8 pci_link 10 4K 10 msi 9 4K 9 DEVFS 9 4K 10 tun 7 4K 7 ipsec 3 3K 3 mkdir 8 3K 30854 in_multi 6 3K 8 nexusdev 7 3K 7 feeder 7 3K 7 procdesc 5 3K 14 freework 5 3K 15436 sctp_ifn 6 3K 14 mld 6 3K 6 igmp 6 3K 6 toponodes 6 3K 6 prison 6 3K 6 isadev 6 3K 6 softdep 1 2K 1 freeblks 4 2K 15435 vnodemarker 2 2K 96 NFSD session 1 2K 1 linux 5 2K 6 CAM periph 4 2K 271 soname 5 2K 40042 filecaps 5 2K 98 crypto 4 2K 163 newdirblk 4 2K 15427 diradd 4 2K 15482 pfil 4 2K 4 CAM path 4 2K 1034 tcpfunc 3 2K 3 loginclass 3 2K 6 chacha20random 1 1K 1 vnodes 1 1K 1 CAM SIM 2 1K 2 ktls 1 1K 1 cdev 2 1K 2 aesni_data 2 1K 2 tcp_fsb 2 1K 1240 cpus 2 1K 2 atkbddev 2 1K 2 CAM dev queue 2 1K 2 xform 2 1K 642 entropy 2 1K 46 iov 2 1K 29953 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 pmchooks 1 1K 1 sctp_vrf 1 1K 1 sctp_aadr 1 1K 2 apmdev 1 1K 1 CAM I/O Scheduler 1 1K 1 vnet_data_free 1 1K 1 vnet 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 acpiintr 1 1K 1 pmc 1 1K 1 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_temp 0 0K 0 sctp_mcore 0 0K 0 sctp_socko 0 0K 8324 sctp_iter 0 0K 10 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 0 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 10 sctp_stri 0 0K 244 mqdata 0 0K 0 cubic data 0 0K 46 cdg data 0 0K 0 tcp_do 0 0K 0 vegas data 0 0K 0 chd data 0 0K 0 dctcp data 0 0K 42 htcp data 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 iavf 0 0K 0 ixl 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 axgbe 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xen_intr 0 0K 0 xenstore 0 0K 0 ciss_data 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 qpidrv 0 0K 0 BACKLIGHT 0 0K 0 xnb 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 sysmouse 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 vtfont 0 0K 0 vm_fictitious 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 amr 0 0K 0 pvscsi 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 UMAHash 0 0K 0 scsi_cd 0 0K 0 AHCI driver 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 15095 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 46 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 6 allocindir 0 0K 0 indirdep 0 0K 10 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 agp 0 0K 0 nvme_da 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 ktls_ocf 0 0K 0 twsbuf 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpifw 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 31 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 twe_commands 0 0K 0 LRO 0 0K 0 newreno data 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 7 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 2 VN POLL 0 0K 0 twa_commands 0 0K 0 statfs 0 0K 15646 namei_tracker 0 0K 0 export_host 0 0K 0 cl_savebuf 0 0K 4 tcp_log_dev 0 0K 994 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 2787 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 biobuf 0 0K 0 aios 0 0K 5 lio 0 0K 0 acl 0 0K 0 tempbuff 0 0K 0 mbuf_tag 0 0K 138 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 accf 0 0K 0 pts 0 0K 0 ioctlops 0 0K 143 eventfd 0 0K 0 Witness 0 0K 0 stack 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 sbuf 0 0K 288 md_sectors 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 md_disk 0 0K 0 SWAP 0 0K 0 malodev 0 0K 0 LED 0 0K 0 sysctltmp 0 0K 845 sysctl 0 0K 3 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 ix_sriov 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 cache 0 0K 0 iirbuf 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 1 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8361 946 1263934 0 254 38121472 0 malloc-8192 8192 592 1326 11769 0 1 15712256 0 malloc-384 384 34708 62 98552 0 30 13351680 0 malloc-1024 1024 1789 3983 27757 0 16 5910528 0 malloc-384 384 4929 6641 147161 0 30 4442880 0 sctp_asoc 2288 589 1326 3823 0 254 4381520 0 malloc-1024 1024 4126 10 4636 0 16 4235264 0 malloc-2048 2048 591 1319 3833 0 8 3911680 0 tcp_log 416 0 8289 112511 0 254 3448224 0 malloc-8192 8192 392 7 16852 0 1 3268608 0 mbuf 256 9272 3028 9372110 0 254 3148800 0 malloc-1024 1024 0 2508 15096 0 16 2568192 0 malloc-1024 1024 164 2336 15601 0 16 2560000 0 pbuf 2624 0 973 0 0 2 2553152 0 sctp_ep 1280 589 1370 11753 0 254 2507520 0 RADIX NODE 144 15702 423 488921 0 63 2322000 0 UMA Slabs 0 112 17873 16 17873 0 126 2003568 0 BUF TRIE 144 191 13277 1229 0 62 1939392 0 malloc-384 384 1532 2648 14462 0 30 1605120 0 malloc-384 384 1825 2345 35456 0 30 1601280 0 sctp_raddr 736 589 1325 3823 0 254 1408704 0 mbuf_cluster 2048 660 2 660 0 254 1355776 0 malloc-512 512 163 2341 15543 0 30 1282048 0 FFS inode 1160 517 29 15956 0 8 633360 0 tcp_bbr_map 128 0 4557 2033415 0 127 583296 0 ertt_txseginfo 40 0 14544 7393904 0 254 581760 0 malloc-32768 32768 8 7 15438 0 1 491520 0 256 Bucket 2048 226 2 48653 0 8 466944 0 malloc-512 512 234 590 35467 0 30 421888 0 malloc-65536 65536 5 1 181 0 1 393216 0 sctp_chunk 152 269 2227 3014 0 254 379392 0 malloc-16384 16384 23 0 23 0 1 376832 0 lkpicurr 160 2 2348 2 0 62 376000 0 lkpimm 160 1 2324 1 0 62 372000 0 VM OBJECT 264 1297 98 235927 0 30 368280 0 THREAD 1808 165 22 20456 0 8 338096 0 socket 944 90 250 31453 0 254 320960 0 malloc-1024 1024 271 33 21403 0 16 311296 0 VNODE 448 555 93 15996 0 30 290304 0 vmem btag 56 5109 63 5109 0 254 289632 0 malloc-2048 2048 124 12 450 0 8 278528 0 tcp_inpcb 488 50 518 14608 0 254 277184 0 malloc-384 384 605 65 70192 0 30 257280 0 DEVCTL 1024 0 216 132 0 0 221184 0 tcpcb 1080 50 153 14608 0 254 219240 0 malloc-384 384 499 71 33241 0 30 218880 0 MAP ENTRY 96 1639 419 894323 0 126 197568 0 malloc-65536 65536 3 0 3 0 1 196608 0 malloc-4096 4096 13 35 3614 0 3 196608 0 malloc-384 384 433 57 230987 0 30 188160 0 UMA Zones 768 242 2 242 0 16 187392 0 malloc-8192 8192 18 3 652 0 1 172032 0 mbuf_packet 256 130 530 38268 0 254 168960 0 malloc-16384 16384 6 4 268 0 1 163840 0 malloc-16384 16384 10 0 10 0 1 163840 0 32 Bucket 256 562 53 24107 0 62 157440 0 S VFS Cache 104 1024 380 17499 0 126 146016 0 FFS2 dinode 256 517 38 15956 0 62 142080 0 malloc-512 512 250 22 3476 0 30 139264 0 malloc-16384 16384 7 1 12 0 1 131072 0 VMSPACE 2544 39 12 16340 0 4 129744 0 ksiginfo 112 69 975 1642 0 126 116928 0 PROC 1328 65 19 16365 0 8 111552 0 malloc-1024 1024 98 2 104 0 16 102400 0 malloc-32768 32768 1 2 122 0 1 98304 0 malloc-32768 32768 2 1 17 0 1 98304 0 128 Bucket 1024 36 59 9449 0 16 97280 0 16 Bucket 144 613 59 6364 0 62 96768 0 malloc-1024 1024 86 6 87 0 16 94208 0 malloc-512 512 165 19 32067 0 30 94208 0 filedesc0 1072 66 18 16366 0 8 90048 0 UMA Kegs 384 227 6 227 0 30 89472 0 clpbuf 2624 0 32 21 0 16 83968 0 tcp_bbr_pcb 832 0 99 7992 0 16 82368 0 g_bio 408 0 180 8595 0 30 73440 0 malloc-512 512 132 4 484 0 30 69632 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-65536 65536 0 1 1 0 1 65536 0 malloc-2048 2048 29 3 33 0 8 65536 0 malloc-1024 1024 57 3 875 0 16 61440 0 malloc-8192 8192 2 5 15652 0 1 57344 0 malloc-8192 8192 7 0 8 0 1 57344 0 sctp_laddr 48 1068 108 1321 0 254 56448 0 64 Bucket 512 46 58 12116 0 30 53248 0 sctp_stream_msg_out 112 325 143 3076 0 254 52416 0 malloc-4096 4096 9 3 220 0 2 49152 0 malloc-2048 2048 7 17 515 0 8 49152 0 malloc-384 384 64 56 1403 0 30 46080 0 DIRHASH 1024 34 6 34 0 16 40960 0 malloc-4096 4096 3 7 27 0 2 40960 0 malloc-512 512 50 30 161 0 30 40960 0 pcpu-8 8 4731 389 5188 0 254 40960 0 Files 80 238 262 58798 0 126 40000 0 NAMEI 1024 0 36 78240 0 16 36864 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-16384 16384 0 2 4 0 1 32768 0 malloc-16384 16384 1 1 5 0 1 32768 0 malloc-16384 16384 2 0 2 0 1 32768 0 malloc-8192 8192 2 2 70 0 1 32768 0 malloc-4096 4096 6 2 22 0 2 32768 0 pcpu-64 64 487 25 487 0 254 32768 0 pipe 744 23 17 605 0 16 29760 0 malloc-4096 4096 6 1 8 0 2 28672 0 malloc-512 512 21 35 15989 0 30 28672 0 TURNSTILE 136 188 22 188 0 62 28560 0 2 Bucket 32 733 149 5819 0 254 28224 0 KNOTE 160 28 147 324061 0 62 28000 0 8 Bucket 80 207 143 6333 0 126 28000 0 ripcb 488 5 51 2447 0 254 27328 0 malloc-8192 8192 2 1 6 0 1 24576 0 PWD 32 20 736 15526 0 254 24192 0 ttyinq 160 135 15 300 0 62 24000 0 udplite_inpcb 488 4 44 1077 0 254 23424 0 ttyoutq 256 72 18 160 0 62 23040 0 tcp_rack_pcb 832 2 25 1240 0 16 22464 0 malloc-4096 4096 2 3 44 0 2 20480 0 4 Bucket 48 16 404 763 0 254 20160 0 SLEEPQUEUE 88 188 36 188 0 126 19712 0 Mountpoints 2752 2 5 2 0 4 19264 0 malloc-16384 16384 1 0 1 0 1 16384 0 ertt 72 50 174 14608 0 126 16128 0 vtnet_tx_hdr 24 0 668 7928983 0 254 16032 0 malloc-4096 4096 3 0 3 0 2 12288 0 malloc-2048 2048 0 6 3085 0 8 12288 0 udp_inpcb 488 6 18 314 0 254 11712 0 kenv 258 15 30 1038 0 30 11610 0 routing nhops 256 27 18 34 0 62 11520 0 unpcb 256 20 25 1206 0 254 11520 0 malloc-8192 8192 1 0 1 0 1 8192 0 malloc-4096 4096 0 2 15 0 2 8192 0 malloc-2048 2048 0 4 9 0 8 8192 0 malloc-512 512 4 12 94 0 30 8192 0 rtentry 176 30 16 34 0 62 8096 0 PGRP 88 35 57 52 0 126 8096 0 rl_entry 40 64 138 64 0 254 8080 0 tcp_rack_map 112 0 72 1639 0 126 8064 0 udpcb 32 10 242 1391 0 254 8064 0 AIOP 32 4 248 4 0 254 8064 0 sctp_readq 152 0 52 55 0 254 7904 0 AIO 208 0 38 10 0 62 7904 0 cryptop 280 0 28 24 0 30 7840 0 AIOCB 552 0 14 21 0 16 7728 0 FPU_save_area 832 1 8 1 0 16 7488 0 cpuset 104 7 55 7 0 126 6448 0 epoch_record pcpu 256 4 12 4 0 62 4096 0 malloc-2048 2048 1 1 1 0 8 4096 0 malloc-2048 2048 1 1 1 0 8 4096 0 pcpu-16 16 7 249 7 0 254 4096 0 hostcache 64 1 62 1 0 254 4032 0 syncache 168 0 24 4 0 254 4032 0 itimer 352 0 11 2 0 30 3872 0 UMA Slabs 1 176 9 13 9 0 62 3872 0 mqnode 416 3 6 3 0 30 3744 0 KMAP ENTRY 96 12 27 12 0 0 3744 0 vmem 1856 1 1 1 0 8 3712 0 SMR CPU 32 3 60 3 0 254 2016 0 SMR SHARED 24 3 60 3 0 254 1512 0 FFS1 dinode 128 0 0 0 0 126 0 0 da_ccb 544 0 0 0 0 16 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 62 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 304 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 48 0 0 0 0 254 0 0 tcp_log_node 120 0 0 0 0 126 0 0 tcp_log_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 88 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 AIOLIO 272 0 0 0 0 30 0 0 NCLNODE 584 0 0 0 0 16 0 0 TMPFS node 224 0 0 0 0 62 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 linux_dma_object 24 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 ktls_session 128 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 domainset 40 0 0 0 0 254 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0