panic: mtx_lock() of spin mutex (null) @ /syzkaller/managers/main/kernel/sys/netinet/tcp_output.c:355 cpuid = 1 time = 1625101747 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x47/frame 0xfffffe0051734630 vpanic() at vpanic+0x1cc/frame 0xfffffe0051734690 panic() at panic+0x43/frame 0xfffffe00517346f0 __mtx_lock_flags() at __mtx_lock_flags+0x1d7/frame 0xfffffe0051734750 tcp_output() at tcp_output+0x6dc/frame 0xfffffe0051734940 tcp_usr_connect() at tcp_usr_connect+0x259/frame 0xfffffe00517349b0 soconnectat() at soconnectat+0x183/frame 0xfffffe0051734a10 kern_connectat() at kern_connectat+0x21b/frame 0xfffffe0051734a70 sys_connect() at sys_connect+0xd9/frame 0xfffffe0051734ab0 amd64_syscall() at amd64_syscall+0x247/frame 0xfffffe0051734bf0 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0051734bf0 --- syscall (198, FreeBSD ELF64, nosys), rip = 0x287eda, rsp = 0x7fffdffdcf08, rbp = 0x7fffdffdcf70 --- KDB: enter: panic [ thread pid 42844 tid 154755 ] Stopped at kdb_enter+0x67: movq $0,0x16400ee(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0x28 rax 0x12 rcx 0xfffffe009a800000 rdx 0x3ffff rbx 0 rsp 0xfffffe0051734610 rbp 0xfffffe0051734630 rsi 0x40001 rdi 0xffffffff811327b6 vprintf+0x176 r8 0 r9 0x8080808080808080 r10 0xfffffe0051734500 r11 0x1ffaefff59c r12 0xffffffff82267d30 ddb_dbbe r13 0xffffffff81a6e500 r14 0xffffffff81a74f8a r15 0xffffffff81a74f8a rip 0xffffffff81129817 kdb_enter+0x67 rflags 0x82 kdb_enter+0x67: movq $0,0x16400ee(%rip) db> show proc Process 42844 (syz-executor.2) at 0xfffff80029f1da70: state: NORMAL uid: 0 gids: 0, 0, 5 parent: pid 794 at 0xfffff800273ce000 ABI: FreeBSD ELF64 flag: 0x10000080 flag2: 0 arguments: /root/syz-executor.2 reaper: 0xfffff80004bda538 reapsubtree: 1 sigparent: 20 vmspace: 0xfffffe009d9be3e0 (map 0xfffffe009d9be3e0) (map.pmap 0xfffffe009d9be4a0) (pmap 0xfffffe009d9be500) threads: 4 152900 RunQ syz-executor.2 154749 S connec 0xfffff8002e453840 syz-executor.2 154755 Run CPU 1 syz-executor.2 154759 Run CPU 0 syz-executor.2 db> ps pid ppid pgrp uid state wmesg wchan cmd 42844 794 794 0 R (threaded) syz-executor.2 152900 RunQ syz-executor.2 154749 S connec 0xfffff8002e453840 syz-executor.2 154755 Run CPU 1 syz-executor.2 154759 Run CPU 0 syz-executor.2 37546 1 37546 0 S uwait 0xfffff80015e7c700 syz-executor.2 37126 1 794 0 SV uwait 0xfffff80029533b00 syz-executor.2 37120 1 794 0 SV uwait 0xfffff8002e86cf00 syz-executor.2 36971 1 36971 0 S uwait 0xfffff80027382700 syz-executor.1 36969 1 36969 0 S uwait 0xfffff80027f81100 syz-executor.1 36734 1 794 0 S uwait 0xfffff80029533d00 syz-executor.2 36730 1 794 0 S uwait 0xfffff80027ec2d00 syz-executor.2 35469 1 789 0 S uwait 0xfffff80015cc2300 syz-executor.3 35468 1 789 0 S uwait 0xfffff8002e86c700 syz-executor.3 35467 1 789 0 S uwait 0xfffff80027374f00 syz-executor.3 35466 1 789 0 S uwait 0xfffff80015abab00 syz-executor.3 33363 1 788 0 S uwait 0xfffff800273c6700 syz-executor.1 33362 1 788 0 S uwait 0xfffff8002e86c500 syz-executor.1 31157 1 794 0 S uwait 0xfffff800273c6500 syz-executor.2 31153 1 794 0 S uwait 0xfffff80027a20d00 syz-executor.2 31057 1 787 0 S uwait 0xfffff80027f88b00 syz-executor.0 31056 1 787 0 S uwait 0xfffff80027ec2f00 syz-executor.0 29891 29886 789 0 SV uwait 0xfffff80027f88700 syz-executor.3 29886 1 789 0 DV ppwait 0xfffff80027541f58 syz-executor.3 29625 1 789 0 S uwait 0xfffff800273c6300 syz-executor.3 29623 1 789 0 S uwait 0xfffff80027d45100 syz-executor.3 29325 1 29318 0 S uwait 0xfffff80027f88d00 syz-executor.0 29322 1 787 0 S uwait 0xfffff80027fd4900 syz-executor.0 29122 1 787 0 S uwait 0xfffff80027ec2500 syz-executor.0 29121 1 787 0 S uwait 0xfffff80027fd4b00 syz-executor.0 26771 26770 789 0 SV uwait 0xfffff80027da0f00 syz-executor.3 26770 1 789 0 DV ppwait 0xfffff8002e700f58 syz-executor.3 23195 1 794 0 S uwait 0xfffff80015cc2500 syz-executor.2 23194 1 794 0 S uwait 0xfffff80027f81b00 syz-executor.2 7103 0 0 0 DL aiordy 0xfffff80015f75000 [aiod4] 7102 0 0 0 DL aiordy 0xfffff80027e6a000 [aiod3] 7101 0 0 0 DL aiordy 0xfffff8002e095000 [aiod2] 7100 0 0 0 DL aiordy 0xfffff800159d5538 [aiod1] 2939 1 2939 65 Ss select 0xfffff80015ef2ec0 dhclient 2329 1 2329 0 Ss select 0xfffff8002e029a40 dhclient 2326 1 2326 0 Ss select 0xfffff8002e171a40 dhclient 2299 1 2299 65 Ss select 0xfffff8002e029440 dhclient 1916 1 1916 0 Ss select 0xfffff80027eea5c0 dhclient 1913 1 1913 0 Ss select 0xfffff80027b67bc0 dhclient 1891 1 1891 65 Ss select 0xfffff8002e029740 dhclient 1555 1 1555 0 Ss select 0xfffff80027b67740 dhclient 1552 1 1552 0 Ss select 0xfffff8002e029ec0 dhclient 1533 1 1533 65 Ss select 0xfffff800274888c0 dhclient 1085 1 1085 0 Ss select 0xfffff80027488140 dhclient 1082 1 1082 0 Ss select 0xfffff80015d0f440 dhclient 794 780 794 0 Ss nanslp 0xffffffff8273d0a1 syz-executor.2 789 780 789 0 Ss piperd 0xfffff80015e28ba0 syz-executor.3 788 780 788 0 Ss piperd 0xfffff80015e2fba0 syz-executor.1 787 780 787 0 Ss piperd 0xfffff80027cd08b8 syz-executor.0 780 778 778 0 S (threaded) syz-fuzzer 100120 S uwait 0xfffff80027a20900 syz-fuzzer 100121 S uwait 0xfffff80015e7cb00 syz-fuzzer 100122 S kqread 0xfffff80027a20100 syz-fuzzer 100123 S uwait 0xfffff80027a20300 syz-fuzzer 100124 S uwait 0xfffff80027a20b00 syz-fuzzer 100125 S uwait 0xfffff80027a20500 syz-fuzzer 100126 S uwait 0xfffff80027a20700 syz-fuzzer 100127 S uwait 0xfffff80027534b00 syz-fuzzer 100131 S uwait 0xfffff80027534d00 syz-fuzzer 778 776 778 0 Ss pause 0xfffff800273b7b20 csh 776 694 776 0 Ss select 0xfffff8002759cd40 sshd 760 1 760 0 Ss+ ttyin 0xfffff800154a5cb0 getty 759 1 759 0 Ss+ ttyin 0xfffff80015b244b0 getty 758 1 758 0 Ss+ ttyin 0xfffff80015b24cb0 getty 757 1 757 0 Ss+ ttyin 0xfffff80015b284b0 getty 756 1 756 0 Ss+ ttyin 0xfffff80015b28cb0 getty 755 1 755 0 Ss+ ttyin 0xfffff80015b2b4b0 getty 754 1 754 0 Ss+ ttyin 0xfffff80015b2bcb0 getty 753 1 753 0 Ss+ ttyin 0xfffff80015b2f4b0 getty 752 1 752 0 Ss+ ttyin 0xfffff80015b2fcb0 getty 698 1 698 0 Ss nanslp 0xffffffff8273d0a0 cron 694 1 694 0 Ss select 0xfffff80015ef28c0 sshd 507 1 507 0 Ss select 0xfffff800272dc140 syslogd 436 1 436 0 Ss select 0xfffff80015ef05c0 devd 435 1 435 65 Ss select 0xfffff80015ef2bc0 dhclient 350 1 350 0 Ss select 0xfffff80015d0f2c0 dhclient 347 1 347 0 Ss select 0xfffff80015ef0440 dhclient 23 0 0 0 DL vlruwt 0xfffff80015cc5a70 [vnlru] 22 0 0 0 DL syncer 0xffffffff8282c450 [syncer] 21 0 0 0 DL (threaded) [bufdaemon] 100081 D qsleep 0xffffffff8282b520 [bufdaemon] 100086 D - 0xffffffff8220aec0 [bufspacedaemon-0] 100095 D sdflush 0xfffff80004ddc4e8 [/ worker] 20 0 0 0 DL psleep 0xffffffff82853388 [vmdaemon] 19 0 0 0 DL (threaded) [pagedaemon] 100079 D psleep 0xffffffff828477f8 [dom0] 100087 D launds 0xffffffff82847804 [laundry: dom0] 100088 D umarcl 0xffffffff815c5b10 [uma] 18 0 0 0 DL - 0xffffffff82571458 [rand_harvestq] 17 0 0 0 DL waiting 0xffffffff830df8b8 [sctp_iterator] 16 0 0 0 DL pftm 0xffffffff82f03410 [pf purge] 15 0 0 0 DL - 0xffffffff82828bdc [soaiod4] 9 0 0 0 DL - 0xffffffff82828bdc [soaiod3] 8 0 0 0 DL - 0xffffffff82828bdc [soaiod2] 7 0 0 0 DL - 0xffffffff82828bdc [soaiod1] 6 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff82448b40 [doneq0] 100045 D - 0xffffffff82448ac0 [async] 100078 D - 0xffffffff82448990 [scanner] 14 0 0 0 DL seqstat 0xfffff80004d8a488 [sequencer 00] 5 0 0 0 DL crypto_ 0xfffff80004d53d80 [crypto returns 1] 4 0 0 0 DL crypto_ 0xfffff80004d53d30 [crypto returns 0] 3 0 0 0 DL crypto_ 0xffffffff82844ce0 [crypto] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff8271c8e0 [g_event] 100036 D - 0xffffffff8271c8e8 [g_up] 100037 D - 0xffffffff8271c8f0 [g_down] 2 0 0 0 DL (threaded) [KTLS] 100028 D - 0xfffff80004c69100 [thr_0] 100029 D - 0xfffff80004c69180 [thr_1] 12 0 0 0 WL (threaded) [intr] 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 I [swi3: vm] 100031 I [swi4: clock (0)] 100032 I [swi4: clock (1)] 100033 I [swi1: netisr 0] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100084 I [swi1: hpts] 100085 I [swi1: hpts] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffff80004bda538 [init] 10 0 0 0 DL audit_w 0xffffffff828451f0 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff8271ce70 [swapper] 100005 D - 0xfffff8000443b500 [if_io_tqg_0] 100006 D - 0xfffff8000443b100 [if_io_tqg_1] 100007 D - 0xfffff80004c78d00 [if_config_tqg_0] 100008 D - 0xfffff80004c78900 [softirq_0] 100009 D - 0xfffff80004c78500 [softirq_1] 100010 D - 0xfffff80004c78100 [aiod_kick taskq] 100012 D - 0xfffff80004c75900 [kqueue_ctx taskq] 100013 D - 0xfffff80004c75500 [pci_hp taskq] 100015 D - 0xfffff80004c6ed00 [inm_free taskq] 100017 D - 0xfffff80004c6e500 [linuxkpi_irq_wq] 100018 D - 0xfffff80004c6e100 [thread taskq] 100019 D - 0xfffff80004c69d00 [in6m_free taskq] 100020 D - 0xfffff80004c69900 [linuxkpi_short_wq_0] 100021 D - 0xfffff80004c69900 [linuxkpi_short_wq_1] 100022 D - 0xfffff80004c69900 [linuxkpi_short_wq_2] 100023 D - 0xfffff80004c69900 [linuxkpi_short_wq_3] 100024 D - 0xfffff80004c69500 [linuxkpi_long_wq_0] 100025 D - 0xfffff80004c69500 [linuxkpi_long_wq_1] 100026 D - 0xfffff80004c69500 [linuxkpi_long_wq_2] 100027 D - 0xfffff80004c69500 [linuxkpi_long_wq_3] 100034 D - 0xfffff80004cd3d00 [firmware taskq] 100038 D - 0xfffff80004cd3900 [crypto_0] 100039 D - 0xfffff80004cd3900 [crypto_1] 100055 D - 0xfffff800153ebd00 [vtnet0 rxq 0] 100056 D - 0xfffff800153eb900 [vtnet0 txq 0] 100057 D - 0xfffff800153eb500 [vtnet0 rxq 1] 100058 D - 0xfffff800153eb100 [vtnet0 txq 1] 100060 D vtbslp 0xfffff80015463700 [virtio_balloon] 100064 D - 0xfffff80015468900 [mca taskq] 100066 D - 0xffffffff81e21841 [deadlkres] 100074 D - 0xfffff80015a41900 [acpi_task_0] 100075 D - 0xfffff80015a41900 [acpi_task_1] 100076 D - 0xfffff80015a41900 [acpi_task_2] 100077 D - 0xfffff80004cd3500 [CAM taskq] 18902 1 788 0 Z syz-executor.1 18903 1 788 0 Z syz-executor.1 db> show all locks Process 42844 (syz-executor.2) thread 0xfffffe00950bf740 (154755) exclusive rw tcpinp (tcpinp) r = 0 (0xfffff8002e803020) locked @ /syzkaller/managers/main/kernel/sys/netinet/tcp_usrreq.c:574 db> show malloc Type InUse MemUse Requests sysctloid 34288 12858K 34355 pf_hash 5 11560K 5 devbuf 4216 6982K 4244 tcp_hpts 5 3219K 5 pcb 489 2676K 81749 kobj 326 2608K 486 vtbuf 24 2064K 46 filedesc 68 1085K 83553 vfscache 3 1035K 3 newblk 5 1034K 43410 subproc 209 867K 42978 acpica 1674 649K 54806 inodedep 70 589K 41902 callout 2 528K 2 ufs_quota 1 520K 1 vfs_hash 1 520K 1 intr 4 480K 4 sctp_atcl 427 427K 49629 sctp_stro 198 396K 14797 bus 986 377K 3469 BPF 46 337K 74 linker 348 270K 457 sctp_atky 625 235K 65357 kdtrace 287 228K 97612 DEVFS1 107 214K 124 umtx 440 193K 440 module 510 192K 510 vnet_data 1 176K 1 tidhash 3 164K 3 sctp_athm 427 161K 50205 sctp_map 396 149K 29594 pagedep 9 140K 41776 tfo_ccache 1 136K 1 sem 4 120K 4 sctp_timw 224 112K 224 vmem 3 104K 6 UMA 270 102K 270 pf_osfp 251 95K 251 audit_evclass 236 89K 294 temp 36 81K 7887 mtx_pool 2 80K 2 routetbl 141 77K 7293 syncache 1 76K 1 msg 4 68K 4 ifaddr 71 67K 73 acpitask 1 64K 1 ddb_capture 1 64K 1 DEVFS3 126 63K 136 ether_multi 157 59K 167 gtaskqueue 18 57K 18 DEVFS_RULE 56 54K 56 kenv 95 52K 95 eventhandler 133 50K 133 sctp_aadr 123 47K 4306 kqueue 97 45K 43242 ithread 99 43K 99 rman 84 42K 425 lltable 47 41K 236 KTRACE 100 38K 100 taskqueue 60 36K 60 pwddesc 92 35K 43157 DEVFSP 91 35K 13105 proc 3 34K 3 ufs_mount 5 34K 6 bus-sc 33 34K 1684 devstat 4 33K 4 hostcache 1 32K 1 tty 16 32K 16 shm 1 32K 12 dirrem 61 31K 41785 in6_multi 71 31K 71 GEOM 60 29K 489 kbdmux 6 28K 6 cred 26 26K 368 ifnet 7 25K 7 plimit 24 24K 595 kcovinfo 64 24K 68 freefile 59 23K 41757 ksem 4 21K 4321 CAM queue 5 21K 1528 selfd 55 21K 574806 pfs_nodes 20 20K 20 proc-args 52 20K 782 session 36 18K 74 UART 12 18K 12 bmsafemap 2 17K 41893 rpc 2 16K 2 shmfd 1 16K 46 pfs_vncache 1 16K 1 ufs_dirhash 24 14K 24 sglist 5 13K 5 lockf 33 13K 120 MCA 32 12K 32 CAM DEV 3 12K 510 Unitno 30 12K 47 vt 11 11K 11 acpisem 28 11K 28 CAM XPT 22 11K 543 pf_ifnet 13 10K 1435 inpcbpolicy 25 10K 25114 uidinfo 3 9K 24 pf_table 2 8K 1719 memdesc 1 8K 1 ipsec-saq 2 8K 2 evdev 4 8K 4 acpidev 20 8K 20 hhook 15 8K 17 select 19 8K 300 mount 16 7K 90 fpukern_ctx 3 6K 3 terminal 11 6K 11 sctp_ifa 14 6K 15 ip6ndp 12 6K 15 osd 13 5K 17930 xform 13 5K 14330 ipsecpolicy 2 5K 2 encap_export_host 12 5K 12 clone 9 5K 9 local_apic 1 4K 1 io_apic 1 4K 1 sahead 1 4K 1 secasvar 1 4K 1 nhops 6 4K 8 pci_link 10 4K 10 ip6opt 8 4K 110 msi 9 4K 9 DEVFS 9 4K 10 tun 7 4K 7 ipsec 3 3K 3 mkdir 8 3K 83496 in_multi 6 3K 8 nexusdev 7 3K 7 feeder 7 3K 7 procdesc 5 3K 18 freework 5 3K 41744 sctp_ifn 6 3K 15 mld 6 3K 6 igmp 6 3K 6 soname 6 3K 71831 toponodes 6 3K 6 prison 6 3K 6 isadev 6 3K 6 softdep 1 2K 1 freeblks 4 2K 41743 vnodemarker 2 2K 188 NFSD session 1 2K 1 lkpikmalloc 5 2K 6 CAM periph 4 2K 271 filecaps 5 2K 144 crypto 4 2K 88 newdirblk 4 2K 41748 diradd 4 2K 41829 pfil 4 2K 4 CAM path 4 2K 1034 tcpfunc 3 2K 3 loginclass 3 2K 7 chacha20random 1 1K 1 vnodes 1 1K 8 ktls 1 1K 1 cdev 2 1K 2 aesni_data 2 1K 2 cpus 2 1K 2 atkbddev 2 1K 2 CAM dev queue 2 1K 2 entropy 2 1K 51 CAM SIM 2 1K 2 filedesc_to_leader 2 1K 4 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 pmchooks 1 1K 1 sctp_vrf 1 1K 1 pf_rule 1 1K 1582 apmdev 1 1K 1 CAM I/O Scheduler 1 1K 1 vnet_data_free 1 1K 1 vnet 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 acpiintr 1 1K 1 pmc 1 1K 1 sctp_mcore 0 0K 0 sctp_socko 0 0K 26004 sctp_iter 0 0K 17 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 9 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 9 sctp_stri 0 0K 1136 cdg data 0 0K 0 chd data 0 0K 23 cubic data 0 0K 0 dctcp data 0 0K 0 htcp data 0 0K 0 pf_altq 0 0K 0 pf_temp 0 0K 0 vegas data 0 0K 9 tcp_do 0 0K 0 tcp_fsb 0 0K 3522 mqdata 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 iavf 0 0K 0 ixl 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 axgbe 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xen_intr 0 0K 0 xenstore 0 0K 0 ciss_data 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 qpidrv 0 0K 0 BACKLIGHT 0 0K 0 xnb 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 sysmouse 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 vtfont 0 0K 0 vm_fictitious 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 amr 0 0K 0 pvscsi 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 UMAHash 0 0K 0 scsi_cd 0 0K 0 AHCI driver 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 39060 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 93 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 7 allocindir 0 0K 0 indirdep 0 0K 12 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 agp 0 0K 0 nvme_da 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 ktls_ocf 0 0K 0 twsbuf 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpifw 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 twe_commands 0 0K 0 LRO 0 0K 0 newreno data 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 3 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 2 VN POLL 0 0K 0 twa_commands 0 0K 0 statfs 0 0K 42149 namei_tracker 0 0K 2 export_host 0 0K 0 cl_savebuf 0 0K 8 tcp_log_dev 0 0K 1740 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 4268 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 biobuf 0 0K 0 aios 0 0K 728 lio 0 0K 21 acl 0 0K 0 tempbuff 0 0K 0 mbuf_tag 0 0K 197 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 accf 0 0K 0 pts 0 0K 0 iov 0 0K 51697 ioctlops 0 0K 11339 eventfd 0 0K 71 Witness 0 0K 0 stack 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 sbuf 0 0K 288 md_sectors 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 md_disk 0 0K 0 SWAP 0 0K 0 malodev 0 0K 0 LED 0 0K 0 sysctltmp 0 0K 1137 sysctl 0 0K 3 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 ix_sriov 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 cache 0 0K 0 iirbuf 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 1 pwd 0 0K 0 tty console 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8324 969 1245333 0 254 38064128 0 malloc-384 384 35895 1925 68691 0 30 14522880 0 malloc-8192 8192 231 947 34270 0 1 9650176 0 malloc-1024 1024 4174 10 4211 0 16 4284416 0 tcp_log 416 0 8964 156322 0 254 3729024 0 malloc-1024 1024 133 2811 41050 0 16 3014656 0 malloc-384 384 2289 5431 757923 0 30 2964480 0 malloc-1024 1024 152 2732 43270 0 16 2953216 0 RADIX NODE 144 16676 3593 1142687 0 63 2918736 0 malloc-8192 8192 328 2 1099 0 1 2703360 0 mbuf 256 9099 1401 6908546 0 254 2688000 0 sctp_asoc 2288 198 962 14797 0 254 2654080 0 pbuf 2624 0 957 0 0 2 2511168 0 malloc-1024 1024 432 1912 50790 0 16 2400256 0 malloc-2048 2048 200 956 16202 0 8 2367488 0 BUF TRIE 144 276 13192 2427 0 62 1939392 0 UMA Slabs 0 112 16696 5 16696 0 126 1870512 0 mbuf_cluster 2048 840 2 840 0 254 1724416 0 sctp_ep 1280 229 962 34264 0 254 1524480 0 malloc-512 512 203 2741 45806 0 30 1507328 0 malloc-1024 1024 434 966 92724 0 16 1433600 0 malloc-32768 32768 35 7 41777 0 1 1376256 0 malloc-384 384 644 2706 215474 0 30 1286400 0 sctp_raddr 736 198 1100 15288 0 254 955328 0 malloc-8192 8192 96 7 42852 0 1 843776 0 malloc-384 384 1726 44 54452 0 30 679680 0 FFS inode 1160 538 22 42303 0 8 649600 0 tcp_bbr_map 128 1 4370 1140238 0 126 559488 0 VM OBJECT 264 1804 116 613740 0 30 506880 0 malloc-4096 4096 9 111 6948 0 2 491520 0 ertt_txseginfo 40 2054 9157 4624080 0 254 448440 0 malloc-16384 16384 24 2 33 0 1 425984 0 256 Bucket 2048 180 22 34703 0 8 413696 0 THREAD 1808 191 29 54759 0 8 397760 0 lkpicurr 160 2 2348 2 0 62 376000 0 lkpimm 160 1 2324 1 0 62 372000 0 socket 944 83 309 61180 0 254 370048 0 malloc-384 384 630 310 26601 0 30 360960 0 tcpcb 1080 9 299 17918 0 254 332640 0 malloc-65536 65536 5 0 5 0 1 327680 0 malloc-512 512 27 613 54234 0 30 327680 0 MAP ENTRY 96 2939 421 2329908 0 126 322560 0 tcp_inpcb 488 20 604 17918 0 254 304512 0 VNODE 448 577 80 42344 0 30 294336 0 malloc-384 384 395 305 182691 0 30 268800 0 malloc-2048 2048 125 3 146 0 8 262144 0 vmem btag 56 4320 60 4320 0 254 245280 0 DEVCTL 1024 0 220 154 0 0 225280 0 mbuf_packet 256 72 768 88268 0 254 215040 0 malloc-8192 8192 23 3 88 0 1 212992 0 g_bio 408 0 490 13865 0 30 199920 0 malloc-65536 65536 3 0 3 0 1 196608 0 malloc-16384 16384 11 1 247 0 1 196608 0 VMSPACE 2544 63 12 42818 0 4 190800 0 UMA Zones 768 242 2 242 0 16 187392 0 tcp_bbr_pcb 832 1 215 9699 0 16 179712 0 32 Bucket 256 558 87 38164 0 62 165120 0 sctp_chunk 152 190 876 3398 0 254 162032 0 malloc-384 384 357 63 58500 0 30 161280 0 malloc-384 384 343 67 61778 0 30 157440 0 PROC 1336 94 20 42851 0 8 152304 0 S VFS Cache 104 1093 350 45972 0 126 150072 0 FFS2 dinode 256 538 32 42302 0 62 145920 0 malloc-512 512 201 63 87991 0 30 135168 0 malloc-512 512 238 26 42364 0 30 135168 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-1024 1024 127 1 145 0 16 131072 0 malloc-512 512 238 10 485 0 30 126976 0 clpbuf 2624 0 48 77 0 16 125952 0 filedesc0 1072 92 20 43157 0 8 120064 0 ksiginfo 112 99 945 3651 0 126 116928 0 malloc-16384 16384 2 5 36 0 1 114688 0 128 Bucket 1024 76 35 18206 0 16 113664 0 16 Bucket 144 595 105 9597 0 62 100800 0 malloc-512 512 121 71 45801 0 30 98304 0 malloc-512 512 25 167 2287 0 30 98304 0 UMA Kegs 384 227 6 227 0 30 89472 0 malloc-4096 4096 13 5 4668 0 2 73728 0 malloc-65536 65536 0 1 1 0 1 65536 0 malloc-65536 65536 0 1 176 0 1 65536 0 malloc-32768 32768 1 1 16 0 1 65536 0 malloc-32768 32768 1 1 121 0 1 65536 0 malloc-16384 16384 4 0 4 0 1 65536 0 malloc-16384 16384 2 2 9 0 1 65536 0 malloc-8192 8192 2 6 43964 0 1 65536 0 malloc-2048 2048 10 20 574 0 8 61440 0 pcpu-8 8 4765 2915 47091 0 254 61440 0 malloc-8192 8192 3 4 77 0 1 57344 0 udplite_inpcb 488 3 109 6437 0 254 54656 0 malloc-4096 4096 7 6 699 0 2 53248 0 malloc-16384 16384 3 0 3 0 1 49152 0 malloc-16384 16384 3 0 3 0 1 49152 0 64 Bucket 512 45 51 17857 0 30 49152 0 pipe 744 23 37 1459 0 16 44640 0 Files 80 286 264 143850 0 126 44000 0 malloc-2048 2048 16 4 25 0 8 40960 0 sctp_stream_msg_out 112 13 347 3169 0 254 40320 0 sctp_laddr 48 8 832 8256 0 254 40320 0 DIRHASH 1024 34 2 34 0 16 36864 0 NAMEI 1024 0 36 210846 0 16 36864 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-4096 4096 6 2 3070 0 2 32768 0 pcpu-64 64 487 25 487 0 254 32768 0 TURNSTILE 136 221 10 221 0 62 31416 0 tcp_rack_pcb 832 0 36 3522 0 16 29952 0 malloc-512 512 34 22 1331 0 30 28672 0 ertt 72 9 383 17918 0 126 28224 0 2 Bucket 32 606 276 11915 0 254 28224 0 KNOTE 160 28 147 519070 0 62 28000 0 malloc-8192 8192 0 3 13 0 1 24576 0 malloc-8192 8192 1 2 116 0 1 24576 0 malloc-4096 4096 1 5 10 0 2 24576 0 malloc-4096 4096 4 2 779 0 2 24576 0 malloc-2048 2048 2 10 153 0 8 24576 0 PWD 32 33 723 41852 0 254 24192 0 ttyinq 160 135 15 300 0 62 24000 0 8 Bucket 80 164 136 8813 0 126 24000 0 ttyoutq 256 72 18 160 0 62 23040 0 SLEEPQUEUE 88 221 35 221 0 126 22528 0 Mountpoints 2752 2 5 2 0 4 19264 0 unpcb 256 20 55 1729 0 254 19200 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-4096 4096 2 2 7839 0 2 16384 0 tcp_rack_map 112 0 144 5308 0 126 16128 0 vtnet_tx_hdr 24 0 668 5076065 0 254 16032 0 ripcb 488 7 25 310 0 254 15616 0 udp_inpcb 488 6 26 449 0 254 15616 0 malloc-2048 2048 2 4 6 0 8 12288 0 kenv 258 15 30 1042 0 30 11610 0 routing nhops 256 27 18 35 0 62 11520 0 malloc-2048 2048 3 1 163 0 8 8192 0 malloc-2048 2048 4 0 4 0 8 8192 0 malloc-1024 1024 1 7 15 0 16 8192 0 malloc-1024 1024 0 8 55 0 16 8192 0 rtentry 176 31 15 35 0 62 8096 0 PGRP 88 39 53 77 0 126 8096 0 rl_entry 40 113 89 113 0 254 8080 0 syncache 168 0 48 4 0 254 8064 0 udpcb 32 9 243 6886 0 254 8064 0 4 Bucket 48 7 161 876 0 254 8064 0 sctp_readq 152 0 52 395 0 254 7904 0 AIO 208 0 38 111 0 62 7904 0 itimer 352 0 22 21 0 30 7744 0 AIOCB 552 0 14 169 0 16 7728 0 AIOLIO 272 0 28 21 0 30 7616 0 FPU_save_area 832 1 8 1 0 16 7488 0 cpuset 104 7 55 7 0 126 6448 0 epoch_record pcpu 256 4 12 4 0 62 4096 0 malloc-4096 4096 1 0 1 0 2 4096 0 pcpu-16 16 7 249 7 0 254 4096 0 hostcache 64 1 62 1 0 254 4032 0 ipq 56 0 72 1 0 254 4032 0 AIOP 32 4 122 4 0 254 4032 0 UMA Slabs 1 176 14 8 14 0 62 3872 0 mqnode 416 3 6 3 0 30 3744 0 KMAP ENTRY 96 12 27 12 0 0 3744 0 vmem 1856 1 1 1 0 8 3712 0 SMR CPU 32 3 60 3 0 254 2016 0 SMR SHARED 24 3 60 3 0 254 1512 0 FFS1 dinode 128 0 0 0 0 126 0 0 da_ccb 544 0 0 0 0 16 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 62 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 304 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 48 0 0 0 0 254 0 0 tcp_log_node 120 0 0 0 0 126 0 0 tcp_log_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 88 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 NCLNODE 584 0 0 0 0 16 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 TMPFS node 224 0 0 0 0 62 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 cryptop 280 0 0 0 0 30 0 0 linux_dma_object 24 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 ktls_session 128 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 domainset 40 0 0 0 0 254 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0