panic: sx lock still held cpuid = 1 time = 1622664961 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x47/frame 0xfffffe0090009890 vpanic() at vpanic+0x1c7/frame 0xfffffe00900098f0 panic() at panic+0x43/frame 0xfffffe0090009950 sx_destroy() at sx_destroy+0x63/frame 0xfffffe0090009970 solisten_proto() at solisten_proto+0xd2/frame 0xfffffe00900099d0 tcp_usr_listen() at tcp_usr_listen+0x188/frame 0xfffffe0090009a30 solisten() at solisten+0x7a/frame 0xfffffe0090009a70 kern_listen() at kern_listen+0x113/frame 0xfffffe0090009ab0 amd64_syscall() at amd64_syscall+0x247/frame 0xfffffe0090009bf0 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0090009bf0 --- syscall (198, FreeBSD ELF64, nosys), rip = 0x286eda, rsp = 0x7fffdffdcf08, rbp = 0x7fffdffdcf70 --- KDB: enter: panic [ thread pid 25911 tid 130422 ] Stopped at kdb_enter+0x67: movq $0,0x163656e(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0x28 rax 0x12 rcx 0xfffffe0089a00000 rdx 0x3ffff rbx 0 rsp 0xfffffe0090009870 rbp 0xfffffe0090009890 rsi 0x40001 rdi 0xffffffff8113bc26 vprintf+0x176 r8 0 r9 0x8080808080808080 r10 0xfffffe0090009760 r11 0x1ff6ffff59c r12 0xffffffff82267b90 ddb_dbbe r13 0 r14 0xffffffff81a7c877 r15 0xffffffff81a7c877 rip 0xffffffff81132c17 kdb_enter+0x67 rflags 0x82 kdb_enter+0x67: movq $0,0x163656e(%rip) db> show proc Process 25911 (syz-executor.1) at 0xfffff80059e9f530: state: NORMAL uid: 0 gids: 0, 0, 5 parent: pid 784 at 0xfffff800273ab000 ABI: FreeBSD ELF64 flag: 0x10000080 flag2: 0 arguments: /root/syz-executor.1 reaper: 0xfffff80004bd7530 reapsubtree: 1 sigparent: 20 vmspace: 0xfffffe0088b1b000 (map 0xfffffe0088b1b000) (map.pmap 0xfffffe0088b1b0c0) (pmap 0xfffffe0088b1b120) threads: 3 130353 RunQ syz-executor.1 130421 Run CPU 0 syz-executor.1 130422 Run CPU 1 syz-executor.1 db> ps pid ppid pgrp uid state wmesg wchan cmd 25911 784 784 0 R (threaded) syz-executor.1 130353 RunQ syz-executor.1 130421 Run CPU 0 syz-executor.1 130422 Run CPU 1 syz-executor.1 25908 795 795 0 S (threaded) syz-executor.2 129823 S nanslp 0xffffffff8273c8e1 syz-executor.2 130416 S connec 0xfffff80059d31840 syz-executor.2 130420 S uwait 0xfffff80027cb4d00 syz-executor.2 25907 783 783 0 S (threaded) syz-executor.0 128741 S nanslp 0xffffffff8273c8e0 syz-executor.0 130411 S connec 0xfffff80059f82bf0 syz-executor.0 130413 S uwait 0xfffff800594af500 syz-executor.0 25906 870 870 0 S (threaded) syz-executor.3 126950 S nanslp 0xffffffff8273c8e0 syz-executor.3 130410 S connec 0xfffff80059bcf490 syz-executor.3 130412 S uwait 0xfffff80027de0700 syz-executor.3 25041 1 25039 0 S uwait 0xfffff80027e4fb00 syz-executor.2 25040 1 25039 0 S uwait 0xfffff80059302d00 syz-executor.2 19354 1 795 0 S uwait 0xfffff80015d63300 syz-executor.2 19352 1 795 0 S uwait 0xfffff80059302100 syz-executor.2 9832 1 795 0 S uwait 0xfffff80027cb4900 syz-executor.2 9831 1 795 0 S umtxn 0xfffff80027e4f700 syz-executor.2 9827 1 795 0 S uwait 0xfffff80027f02d00 syz-executor.2 9825 1 795 0 S uwait 0xfffff80004ca4900 syz-executor.2 8174 1 8174 65 Ss select 0xfffff80027f18a40 dhclient 7106 1 7106 0 Ss select 0xfffff800594b85c0 dhclient 7103 1 7103 0 Ss select 0xfffff8005995d140 dhclient 7084 1 7084 65 Ss select 0xfffff800591592c0 dhclient 5266 1 5266 0 Ss select 0xfffff80059351bc0 dhclient 5263 1 5263 0 Ss select 0xfffff80015dc15c0 dhclient 5244 1 5244 65 Ss select 0xfffff800594b1440 dhclient 3867 1 3867 0 Ss select 0xfffff800593518c0 dhclient 3864 1 3864 0 Ss select 0xfffff800594b18c0 dhclient 3850 1 3850 65 Ss select 0xfffff800594ac740 dhclient 1731 1 1731 0 Ss select 0xfffff80027fefa40 dhclient 1725 1 1725 0 Ss select 0xfffff800591bad40 dhclient 870 780 870 0 Ss nanslp 0xffffffff8273c8e0 syz-executor.3 795 780 795 0 Ss nanslp 0xffffffff8273c8e1 syz-executor.2 784 780 784 0 Ss nanslp 0xffffffff8273c8e0 syz-executor.1 783 780 783 0 Ss nanslp 0xffffffff8273c8e0 syz-executor.0 780 778 778 0 S (threaded) syz-fuzzer 100091 S uwait 0xfffff80015cf6500 syz-fuzzer 100123 S uwait 0xfffff80015cf6b00 syz-fuzzer 100124 S uwait 0xfffff80015cf6d00 syz-fuzzer 100125 S uwait 0xfffff80015cf6f00 syz-fuzzer 100126 S uwait 0xfffff800273ad500 syz-fuzzer 100127 S uwait 0xfffff80004ca4100 syz-fuzzer 100128 S uwait 0xfffff80004ca4300 syz-fuzzer 100129 S uwait 0xfffff80004ca4500 syz-fuzzer 100394 S kqread 0xfffff800271cf500 syz-fuzzer 778 776 778 0 Ss pause 0xfffff80015e71b10 csh 776 694 776 0 Ss select 0xfffff80027528a40 sshd 760 1 760 0 Ss+ ttyin 0xfffff8001541bcb0 getty 759 1 759 0 Ss+ ttyin 0xfffff80015988cb0 getty 758 1 758 0 Ss+ ttyin 0xfffff80015a41cb0 getty 757 1 757 0 Ss+ ttyin 0xfffff80015a454b0 getty 756 1 756 0 Ss+ ttyin 0xfffff80015a45cb0 getty 755 1 755 0 Ss+ ttyin 0xfffff80015a494b0 getty 754 1 754 0 Ss+ ttyin 0xfffff80015a49cb0 getty 753 1 753 0 Ss+ ttyin 0xfffff80015a4d4b0 getty 752 1 752 0 Ss+ ttyin 0xfffff80015a4dcb0 getty 698 1 698 0 Ss nanslp 0xffffffff8273c8e1 cron 694 1 694 0 Ss select 0xfffff80027250ec0 sshd 507 1 507 0 Ss select 0xfffff80027154bc0 syslogd 436 1 436 0 Ss select 0xfffff80015e502c0 devd 435 1 435 65 Ss select 0xfffff80015e50740 dhclient 350 1 350 0 Ss select 0xfffff80015e505c0 dhclient 347 1 347 0 Ss select 0xfffff80027285140 dhclient 23 0 0 0 DL vlruwt 0xfffff80015ae5000 [vnlru] 22 0 0 0 DL syncer 0xffffffff8282bcd0 [syncer] 21 0 0 0 DL (threaded) [bufdaemon] 100081 D qsleep 0xffffffff8282ada0 [bufdaemon] 100084 D - 0xffffffff8220ae80 [bufspacedaemon-0] 100098 D sdflush 0xfffff80004d594e8 [/ worker] 20 0 0 0 DL psleep 0xffffffff82852c08 [vmdaemon] 19 0 0 0 DL (threaded) [pagedaemon] 100079 D psleep 0xffffffff82847078 [dom0] 100082 D launds 0xffffffff82847084 [laundry: dom0] 100083 D umarcl 0xffffffff815d04a0 [uma] 18 0 0 0 DL - 0xffffffff82570c88 [rand_harvestq] 17 0 0 0 DL waiting 0xffffffff82f96828 [sctp_iterator] 16 0 0 0 DL pftm 0xffffffff8309c3c0 [pf purge] 15 0 0 0 DL - 0xffffffff8282845c [soaiod4] 9 0 0 0 DL - 0xffffffff8282845c [soaiod3] 8 0 0 0 DL - 0xffffffff8282845c [soaiod2] 7 0 0 0 DL - 0xffffffff8282845c [soaiod1] 6 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff82448140 [doneq0] 100045 D - 0xffffffff824480c0 [async] 100078 D - 0xffffffff82447f90 [scanner] 14 0 0 0 DL seqstat 0xfffff80004debc88 [sequencer 00] 5 0 0 0 DL crypto_ 0xfffff80004d3dd80 [crypto returns 1] 4 0 0 0 DL crypto_ 0xfffff80004d3dd30 [crypto returns 0] 3 0 0 0 DL crypto_ 0xffffffff82844560 [crypto] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff8271c120 [g_event] 100036 D - 0xffffffff8271c128 [g_up] 100037 D - 0xffffffff8271c130 [g_down] 2 0 0 0 DL (threaded) [KTLS] 100028 D - 0xfffff80004c4ad00 [thr_0] 100029 D - 0xfffff80004c4ad80 [thr_1] 12 0 0 0 WL (threaded) [intr] 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 I [swi3: vm] 100031 I [swi4: clock (0)] 100032 I [swi4: clock (1)] 100033 I [swi1: netisr 0] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100087 I [swi1: hpts] 100088 I [swi1: hpts] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffff80004bd7530 [init] 10 0 0 0 DL audit_w 0xffffffff82844a70 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff8271c6b0 [swapper] 100005 D - 0xfffff8000443a500 [if_io_tqg_0] 100006 D - 0xfffff8000443a100 [if_io_tqg_1] 100007 D - 0xfffff80004c73d00 [if_config_tqg_0] 100008 D - 0xfffff80004c73900 [softirq_0] 100009 D - 0xfffff80004c73500 [softirq_1] 100010 D - 0xfffff80004c73100 [aiod_kick taskq] 100012 D - 0xfffff80004c70900 [kqueue_ctx taskq] 100013 D - 0xfffff80004c70500 [pci_hp taskq] 100015 D - 0xfffff80004c66d00 [inm_free taskq] 100017 D - 0xfffff80004c66500 [linuxkpi_irq_wq] 100018 D - 0xfffff80004c66100 [thread taskq] 100019 D - 0xfffff80004c63d00 [in6m_free taskq] 100020 D - 0xfffff80004c63900 [linuxkpi_short_wq_0] 100021 D - 0xfffff80004c63900 [linuxkpi_short_wq_1] 100022 D - 0xfffff80004c63900 [linuxkpi_short_wq_2] 100023 D - 0xfffff80004c63900 [linuxkpi_short_wq_3] 100024 D - 0xfffff80004c63500 [linuxkpi_long_wq_0] 100025 D - 0xfffff80004c63500 [linuxkpi_long_wq_1] 100026 D - 0xfffff80004c63500 [linuxkpi_long_wq_2] 100027 D - 0xfffff80004c63500 [linuxkpi_long_wq_3] 100034 D - 0xfffff80004c63100 [firmware taskq] 100038 D - 0xfffff80004db8d00 [crypto_0] 100039 D - 0xfffff80004db8d00 [crypto_1] 100055 D - 0xfffff80004db8100 [vtnet0 rxq 0] 100056 D - 0xfffff80015361d00 [vtnet0 txq 0] 100057 D - 0xfffff80015361900 [vtnet0 rxq 1] 100058 D - 0xfffff80015361500 [vtnet0 txq 1] 100060 D vtbslp 0xfffff80015372300 [virtio_balloon] 100064 D - 0xfffff80015361100 [mca taskq] 100066 D - 0xffffffff81e29510 [deadlkres] 100074 D - 0xfffff80015a5e900 [acpi_task_0] 100075 D - 0xfffff80015a5e900 [acpi_task_1] 100076 D - 0xfffff80015a5e900 [acpi_task_2] 100077 D - 0xfffff80004db8500 [CAM taskq] db> show all locks Process 25911 (syz-executor.1) thread 0xfffffe0088ff3900 (130421) exclusive sx so_snd_sx (so_snd_sx) r = 0 (0xfffff80059b3dd90) locked @ /syzkaller/managers/main/kernel/sys/kern/uipc_sockbuf.c:467 Process 25911 (syz-executor.1) thread 0xfffffe0088c34720 (130422) exclusive sleep mutex socket (socket) r = 0 (0xfffff80059b3db10) locked @ /syzkaller/managers/main/kernel/sys/netinet/tcp_usrreq.c:457 exclusive rw tcpinp (tcpinp) r = 0 (0xfffff80059d4f3f0) locked @ /syzkaller/managers/main/kernel/sys/netinet/tcp_usrreq.c:450 db> show malloc Type InUse MemUse Requests sysctloid 34288 12858K 34355 pf_hash 5 11560K 5 devbuf 4216 6982K 4244 tcp_hpts 5 3219K 5 kobj 328 2624K 488 vtbuf 24 2064K 46 newblk 9 1036K 25521 vfscache 3 1035K 3 inodedep 186 705K 25034 subproc 159 656K 26002 acpica 1674 649K 54692 pcb 37 634K 41861 filedesc 34 534K 49797 callout 2 528K 2 ufs_quota 1 520K 1 vfs_hash 1 520K 1 intr 4 480K 4 bus 990 378K 3485 BPF 46 337K 56 linker 348 270K 409 DEVFS1 107 214K 124 kdtrace 241 197K 56335 module 512 192K 512 vnet_data 1 176K 1 tidhash 3 164K 3 umtx 374 164K 374 pagedep 9 140K 24906 tfo_ccache 1 136K 1 pf_osfp 347 131K 347 sem 4 120K 4 UMA 270 102K 270 dirrem 177 89K 24940 audit_evclass 236 89K 294 mtx_pool 2 80K 2 temp 35 77K 4751 syncache 1 76K 1 sctp_timw 150 75K 150 routetbl 137 73K 2874 vmem 3 72K 6 msg 4 68K 4 freefile 177 67K 24932 ifaddr 71 67K 73 acpitask 1 64K 1 ddb_capture 1 64K 1 DEVFS3 126 63K 136 ether_multi 157 59K 167 gtaskqueue 18 57K 18 DEVFS_RULE 56 54K 56 kenv 95 52K 95 eventhandler 133 50K 133 ithread 99 43K 99 rman 84 42K 425 KTRACE 100 38K 100 taskqueue 60 36K 60 kqueue 73 36K 25958 lltable 41 35K 142 proc 3 34K 3 ufs_mount 5 34K 6 bus-sc 33 34K 1696 devstat 4 33K 4 hostcache 1 32K 1 tty 16 32K 16 shm 1 32K 9 in6_multi 71 31K 71 DEVFSP 80 30K 2477 GEOM 60 29K 489 kbdmux 6 28K 6 pwddesc 71 27K 25912 cred 26 26K 302 ifnet 7 25K 7 plimit 24 24K 501 kcovinfo 64 24K 68 ksem 4 21K 3199 CAM queue 5 21K 1528 selfd 55 21K 361118 pfs_nodes 20 20K 20 proc-args 52 20K 722 session 36 18K 59 UART 12 18K 12 bmsafemap 2 17K 25020 rpc 2 16K 2 shmfd 1 16K 35 pfs_vncache 1 16K 1 ufs_dirhash 24 14K 24 sglist 5 13K 5 lockf 33 13K 75 MCA 32 12K 32 CAM DEV 3 12K 510 vt 11 11K 11 acpisem 28 11K 28 CAM XPT 22 11K 543 Unitno 27 11K 47 uidinfo 3 9K 13 pf_ifnet 10 9K 700 inpcbpolicy 22 9K 18541 memdesc 1 8K 1 ipsec-saq 2 8K 2 evdev 4 8K 4 acpidev 20 8K 20 hhook 15 8K 17 select 19 8K 105 mount 16 7K 158 fpukern_ctx 3 6K 3 terminal 11 6K 11 sctp_ifa 14 6K 15 ip6ndp 12 6K 15 osd 13 5K 10958 ipsecpolicy 2 5K 2 encap_export_host 12 5K 12 clone 9 5K 9 local_apic 1 4K 1 io_apic 1 4K 1 sahead 1 4K 1 secasvar 1 4K 1 nhops 6 4K 8 pci_link 10 4K 10 msi 9 4K 9 DEVFS 9 4K 10 tun 7 4K 7 ipsec 3 3K 3 mkdir 8 3K 49766 in_multi 6 3K 8 soname 8 3K 41333 nexusdev 7 3K 7 feeder 7 3K 7 procdesc 5 3K 18 freework 5 3K 24890 sctp_ifn 6 3K 15 mld 6 3K 6 igmp 6 3K 6 toponodes 6 3K 6 prison 6 3K 6 isadev 6 3K 6 sctp_atcl 2 2K 26673 softdep 1 2K 1 freeblks 4 2K 24889 vnodemarker 2 2K 94 NFSD session 1 2K 1 linux 5 2K 6 CAM periph 4 2K 271 filecaps 5 2K 127 crypto 4 2K 46 newdirblk 4 2K 24883 diradd 4 2K 24982 pfil 4 2K 4 CAM path 4 2K 1034 tcpfunc 3 2K 3 loginclass 3 2K 6 chacha20random 1 1K 1 vnodes 1 1K 1 CAM SIM 2 1K 2 ktls 1 1K 1 cdev 2 1K 2 aesni_data 2 1K 2 pf_rule 2 1K 1046 sctp_athm 2 1K 27237 sctp_atky 2 1K 35932 cpus 2 1K 2 atkbddev 2 1K 2 CAM dev queue 2 1K 2 xform 2 1K 2454 entropy 2 1K 49 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 pmchooks 1 1K 1 sctp_vrf 1 1K 1 apmdev 1 1K 1 CAM I/O Scheduler 1 1K 1 vnet_data_free 1 1K 1 vnet 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 acpiintr 1 1K 1 pmc 1 1K 1 cdg data 0 0K 17 cubic data 0 0K 20 pf_table 0 0K 609 pf_altq 0 0K 0 pf_temp 0 0K 0 sctp_mcore 0 0K 0 sctp_socko 0 0K 8707 sctp_iter 0 0K 55 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 45 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 10 sctp_aadr 0 0K 8 sctp_stro 0 0K 8641 sctp_stri 0 0K 1510 sctp_map 0 0K 17280 chd data 0 0K 16 dctcp data 0 0K 23 htcp data 0 0K 29 vegas data 0 0K 25 mqdata 0 0K 0 tcp_do 0 0K 0 tcp_fsb 0 0K 1072 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 iavf 0 0K 0 ixl 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 axgbe 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xen_intr 0 0K 0 xenstore 0 0K 0 ciss_data 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 qpidrv 0 0K 0 BACKLIGHT 0 0K 0 xnb 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 sysmouse 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 vtfont 0 0K 0 vm_fictitious 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 amr 0 0K 0 pvscsi 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 UMAHash 0 0K 0 scsi_cd 0 0K 0 AHCI driver 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 23227 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 44 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 6 allocindir 0 0K 0 indirdep 0 0K 10 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 agp 0 0K 0 nvme_da 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 ktls_ocf 0 0K 0 twsbuf 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpifw 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 210 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 twe_commands 0 0K 0 LRO 0 0K 0 newreno data 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 182 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 25 VN POLL 0 0K 0 twa_commands 0 0K 0 statfs 0 0K 25106 namei_tracker 0 0K 4 export_host 0 0K 0 cl_savebuf 0 0K 4 tcp_log_dev 0 0K 182 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 2910 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 biobuf 0 0K 0 aios 0 0K 0 lio 0 0K 0 acl 0 0K 0 tempbuff 0 0K 0 mbuf_tag 0 0K 147 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 accf 0 0K 0 pts 0 0K 0 iov 0 0K 25586 ioctlops 0 0K 10269 eventfd 0 0K 0 Witness 0 0K 0 stack 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 sbuf 0 0K 296 md_sectors 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 md_disk 0 0K 0 SWAP 0 0K 0 malodev 0 0K 0 LED 0 0K 0 sysctltmp 0 0K 895 sysctl 0 0K 3 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 ix_sriov 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 cache 0 0K 0 iirbuf 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 3 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8320 984 1455939 0 254 38109184 0 malloc-384 384 35348 1152 88925 0 30 14016000 0 malloc-8192 8192 402 554 43484 0 1 7831552 0 malloc-1024 1024 4173 2359 27424 0 16 6688768 0 tcp_log 416 0 12618 17515 0 254 5249088 0 RADIX NODE 144 18562 8875 815939 0 63 3950928 0 malloc-1024 1024 207 2457 25063 0 16 2727936 0 mbuf 256 8805 1380 5392938 0 254 2607360 0 pbuf 2624 0 778 0 0 2 2041472 0 BUF TRIE 144 183 13285 1296 0 62 1939392 0 UMA Slabs 0 112 14956 17 14956 0 126 1676976 0 mbuf_cluster 2048 705 1 705 0 254 1445888 0 malloc-1024 1024 220 1120 57625 0 16 1372160 0 malloc-512 512 201 2463 24964 0 30 1363968 0 sctp_asoc 2288 0 585 8640 0 254 1338480 0 malloc-384 384 539 2831 139992 0 30 1294080 0 malloc-384 384 785 2445 81546 0 30 1240320 0 malloc-2048 2048 4 570 9108 0 8 1175552 0 sctp_ep 1280 2 697 17083 0 254 894720 0 malloc-1024 1024 119 565 17303 0 16 700416 0 malloc-384 384 1696 44 53644 0 30 668160 0 malloc-32768 32768 16 4 24896 0 1 655360 0 FFS inode 1160 523 23 25456 0 8 633360 0 malloc-384 384 1371 19 9975 0 30 533760 0 malloc-16384 16384 27 2 31 0 1 475136 0 sctp_raddr 736 0 627 10402 0 254 461472 0 socket 944 49 403 37122 0 254 426688 0 256 Bucket 2048 179 21 29979 0 8 409600 0 VM OBJECT 264 1414 56 369460 0 30 388080 0 lkpimm 160 1 2324 1 0 62 372000 0 lkpicurr 160 2 2323 2 0 62 372000 0 ertt_txseginfo 40 636 8454 2946657 0 254 363600 0 malloc-512 512 62 602 26184 0 30 339968 0 THREAD 1808 169 18 30422 0 8 338096 0 tcp_bbr_map 128 59 2328 250227 0 126 305536 0 VNODE 448 561 87 25496 0 30 290304 0 malloc-4096 4096 8 58 532 0 2 270336 0 malloc-384 384 620 70 431632 0 30 264960 0 malloc-65536 65536 4 0 4 0 1 262144 0 malloc-2048 2048 120 4 137 0 8 253952 0 tcp_inpcb 488 12 508 10946 0 254 253760 0 MAP ENTRY 96 2120 274 1401200 0 126 229824 0 DEVCTL 1024 0 216 132 0 0 221184 0 malloc-512 512 347 53 25497 0 30 204800 0 malloc-65536 65536 1 2 177 0 1 196608 0 malloc-8192 8192 18 5 117 0 1 188416 0 UMA Zones 768 242 2 242 0 16 187392 0 mbuf_packet 256 17 688 44850 0 254 180480 0 malloc-16384 16384 8 3 14 0 1 180224 0 vmem btag 56 3107 49 3107 0 254 176736 0 malloc-384 384 312 128 78812 0 30 168960 0 malloc-32768 32768 4 1 19 0 1 163840 0 malloc-16384 16384 6 4 42 0 1 163840 0 FFS2 dinode 256 523 77 25455 0 62 153600 0 malloc-4096 4096 2 35 9478 0 2 151552 0 S VFS Cache 104 1072 371 28229 0 126 150072 0 VMSPACE 2544 48 9 25890 0 4 145008 0 tcpcb 1064 9 124 10946 0 254 141512 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-65536 65536 2 0 2 0 1 131072 0 128 Bucket 1024 90 33 8144 0 16 125952 0 sctp_chunk 152 0 780 3572 0 254 118560 0 ksiginfo 112 72 972 2524 0 126 116928 0 PROC 1328 70 17 25911 0 8 115536 0 32 Bucket 256 386 49 56404 0 62 111360 0 malloc-1024 1024 65 39 1737 0 16 106496 0 filedesc0 1072 71 20 25912 0 8 97552 0 malloc-512 512 166 18 1033 0 30 94208 0 malloc-512 512 106 70 77740 0 30 90112 0 UMA Kegs 384 227 6 227 0 30 89472 0 clpbuf 2624 0 32 21 0 16 83968 0 malloc-8192 8192 9 1 11 0 1 81920 0 16 Bucket 144 461 71 8912 0 62 76608 0 malloc-4096 4096 11 6 4026 0 2 69632 0 malloc-1024 1024 62 6 929 0 16 69632 0 malloc-512 512 130 6 636 0 30 69632 0 malloc-65536 65536 0 1 1 0 1 65536 0 malloc-32768 32768 0 2 130 0 1 65536 0 malloc-32768 32768 2 0 2 0 1 65536 0 malloc-8192 8192 3 5 25109 0 1 65536 0 g_bio 408 0 160 9087 0 30 65280 0 malloc-2048 2048 11 19 1822 0 8 61440 0 malloc-384 384 114 26 746 0 30 53760 0 pcpu-8 8 4705 1951 24121 0 254 53248 0 64 Bucket 512 42 62 23665 0 30 53248 0 malloc-16384 16384 3 0 3 0 1 49152 0 malloc-16384 16384 1 2 232 0 1 49152 0 udp_inpcb 488 8 80 5017 0 254 42944 0 DIRHASH 1024 34 6 34 0 16 40960 0 malloc-8192 8192 2 3 13 0 1 40960 0 malloc-4096 4096 2 8 620 0 2 40960 0 malloc-512 512 68 12 3557 0 30 40960 0 Files 80 241 259 86781 0 126 40000 0 pipe 744 23 27 871 0 16 37200 0 NAMEI 1024 0 36 128227 0 16 36864 0 malloc-4096 4096 6 3 49 0 2 36864 0 malloc-2048 2048 3 15 152 0 8 36864 0 malloc-2048 2048 16 2 16 0 8 36864 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-16384 16384 2 0 2 0 1 32768 0 malloc-16384 16384 2 0 2 0 1 32768 0 malloc-8192 8192 2 2 2116 0 1 32768 0 malloc-4096 4096 3 5 450 0 2 32768 0 malloc-1024 1024 3 29 2881 0 16 32768 0 pcpu-64 64 487 25 487 0 254 32768 0 tcp_rack_pcb 832 0 36 1072 0 16 29952 0 tcp_bbr_pcb 832 3 33 6613 0 16 29952 0 TURNSTILE 136 188 22 188 0 62 28560 0 KNOTE 160 28 147 623334 0 62 28000 0 ttyinq 160 135 40 300 0 62 28000 0 8 Bucket 80 108 242 5745 0 126 28000 0 malloc-4096 4096 6 0 6 0 2 24576 0 malloc-512 512 12 36 1412 0 30 24576 0 PWD 32 25 731 24981 0 254 24192 0 ripcb 488 5 43 791 0 254 23424 0 udplite_inpcb 488 0 48 1787 0 254 23424 0 ttyoutq 256 72 18 160 0 62 23040 0 sctp_laddr 48 0 420 4321 0 254 20160 0 2 Bucket 32 394 236 7554 0 254 20160 0 SLEEPQUEUE 88 188 36 188 0 126 19712 0 Mountpoints 2752 2 5 3 0 4 19264 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-8192 8192 0 2 12 0 1 16384 0 malloc-2048 2048 8 0 8 0 8 16384 0 sctp_stream_msg_out 112 0 144 1191 0 254 16128 0 vtnet_tx_hdr 24 2 666 3568256 0 254 16032 0 unpcb 256 20 40 1430 0 254 15360 0 malloc-1024 1024 9 3 14 0 16 12288 0 tcp_rack_map 112 0 108 2061 0 126 12096 0 ertt 72 9 159 10946 0 126 12096 0 kenv 258 15 30 1048 0 30 11610 0 malloc-4096 4096 2 0 2 0 2 8192 0 rtentry 176 31 15 35 0 62 8096 0 PGRP 88 36 56 59 0 126 8096 0 rl_entry 40 75 127 75 0 254 8080 0 udpcb 32 8 244 6804 0 254 8064 0 4 Bucket 48 6 162 517 0 254 8064 0 sctp_readq 152 0 52 107 0 254 7904 0 itimer 352 0 22 4 0 30 7744 0 routing nhops 256 27 3 35 0 62 7680 0 FPU_save_area 832 1 8 1 0 16 7488 0 cpuset 104 7 55 7 0 126 6448 0 epoch_record pcpu 256 4 12 4 0 62 4096 0 malloc-2048 2048 2 0 2 0 8 4096 0 pcpu-16 16 7 249 7 0 254 4096 0 syncache 168 0 24 4 0 254 4032 0 ipq 56 0 72 1 0 254 4032 0 UMA Slabs 1 176 12 10 12 0 62 3872 0 mqnode 416 3 6 3 0 30 3744 0 KMAP ENTRY 96 12 27 12 0 0 3744 0 vmem 1856 1 1 1 0 8 3712 0 SMR CPU 32 3 60 3 0 254 2016 0 SMR SHARED 24 3 60 3 0 254 1512 0 FFS1 dinode 128 0 0 0 0 126 0 0 da_ccb 544 0 0 0 0 16 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 62 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 304 0 0 0 0 0 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 48 0 0 0 0 254 0 0 tcp_log_node 120 0 0 0 0 126 0 0 tcp_log_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 hostcache 64 0 0 0 0 254 0 0 tcptw 88 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 AIOLIO 272 0 0 0 0 30 0 0 AIOCB 552 0 0 0 0 16 0 0 AIOP 32 0 0 0 0 254 0 0 AIO 208 0 0 0 0 62 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 TMPFS node 224 0 0 0 0 62 0 0 NCLNODE 584 0 0 0 0 16 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 cryptop 280 0 0 0 0 30 0 0 linux_dma_object 24 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 ktls_session 128 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 domainset 40 0 0 0 0 254 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-2048 2048 0 0 0 0 8 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0