panic: mtx_lock() of spin mutex (null) @ /syzkaller/managers/i386/kernel/sys/modules/tcp/rack/../../../netinet/tcp_stacks/rack.c:16480 cpuid = 1 time = 1625424292 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x47/frame 0xfffffe0088779550 vpanic() at vpanic+0x1cc/frame 0xfffffe00887795b0 panic() at panic+0x43/frame 0xfffffe0088779610 __mtx_lock_flags() at __mtx_lock_flags+0x1d7/frame 0xfffffe0088779670 rack_output() at rack_output+0x4b83/frame 0xfffffe00887799c0 tcp_hpts_thread() at tcp_hpts_thread+0xe50/frame 0xfffffe0088779b10 ithread_loop() at ithread_loop+0x33f/frame 0xfffffe0088779bb0 fork_exit() at fork_exit+0xb3/frame 0xfffffe0088779bf0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0088779bf0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- KDB: enter: panic [ thread pid 12 tid 100085 ] Stopped at kdb_enter+0x67: movq $0,0x164233e(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0 rax 0x12 rcx 0xffffffff81132530 vprintf+0x140 rdx 0x1 rbx 0 rsp 0xfffffe0088779530 rbp 0xfffffe0088779550 rsi 0 rdi 0xffffffff81132566 vprintf+0x176 r8 0 r9 0x8080808080808080 r10 0xfffffe0088779420 r11 0x1ff77fff59c r12 0xffffffff82267d50 ddb_dbbe r13 0xffffffff81a6f000 r14 0xffffffff81a759cb r15 0xffffffff81a759cb rip 0xffffffff811295c7 kdb_enter+0x67 rflags 0x86 kdb_enter+0x67: movq $0,0x164233e(%rip) db> show proc Process 12 (intr) at 0xfffff80004c74a70: state: NORMAL uid: 0 gids: 0 parent: pid 0 at 0xffffffff8271ee70 ABI: null flag: 0x10000284 flag2: 0 reaper: 0xffffffff8271ee70 reapsubtree: 12 sigparent: 20 vmspace: 0xffffffff8271faf0 (map 0xffffffff8271faf0) (map.pmap 0xffffffff8271fbb0) (pmap 0xffffffff8271fc10) threads: 23 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 I [swi3: vm] 100031 I [swi4: clock (0)] 100032 I [swi4: clock (1)] 100033 I [swi1: netisr 0] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100084 I [swi1: hpts] 100085 Run CPU 1 [swi1: hpts] db> ps pid ppid pgrp uid state wmesg wchan cmd 27134 784 784 0 R (threaded) syz-executor.1 125641 RunQ syz-executor.1 132447 S uwait 0xfffff80015e5fd00 syz-executor.1 27133 792 792 0 R syz-executor.2 27132 783 783 0 R (threaded) syz-executor.0 128291 RunQ syz-executor.0 132445 RunQ syz-executor.0 27130 863 27130 0 Rs (threaded) syz-executor.3 130628 Run CPU 0 syz-executor.3 132444 RunQ syz-executor.3 26579 26575 783 0 SV uwait 0xfffff8003fdeaf00 syz-executor.0 26575 1 783 0 DV ppwait 0xfffff8004024bf58 syz-executor.0 23735 1 783 0 S uwait 0xfffff8003f0b7500 syz-executor.0 23734 1 783 0 S uwait 0xfffff8003c53ed00 syz-executor.0 23695 1 783 0 S uwait 0xfffff8003f1e4500 syz-executor.0 23693 1 783 0 S uwait 0xfffff8003cb1fd00 syz-executor.0 14578 1 14578 65 Ss select 0xfffff8003f030a40 dhclient 13942 1 13942 0 Ss select 0xfffff8003fc96440 dhclient 13939 1 13939 0 Ss select 0xfffff800402148c0 dhclient 13916 1 13916 65 Ss select 0xfffff8003c223ec0 dhclient 13186 1 792 0 S uwait 0xfffff8003ce80900 syz-executor.2 13185 1 792 0 S uwait 0xfffff8003cac3d00 syz-executor.2 11620 1 11620 0 Ss select 0xfffff8003f4b4a40 dhclient 11617 1 11617 0 Ss select 0xfffff8003f9e28c0 dhclient 11598 1 11598 65 Ss select 0xfffff8003f4b48c0 dhclient 7473 1 7473 0 Ss select 0xfffff8003f926bc0 dhclient 7467 1 7467 0 Ss select 0xfffff8003c5598c0 dhclient 7437 1 7437 65 Ss select 0xfffff80015dfa440 dhclient 5459 1 5457 0 S uwait 0xfffff80015e0d300 syz-executor.2 5455 1 5453 0 S uwait 0xfffff80015e0d700 syz-executor.0 5452 1 5445 0 S uwait 0xfffff8003ce80d00 syz-executor.2 5441 1 5436 0 S uwait 0xfffff80015e0df00 syz-executor.2 5432 1 5428 0 S uwait 0xfffff8003cd44100 syz-executor.2 5422 1 5419 0 S uwait 0xfffff8003c53ef00 syz-executor.2 5417 1 5411 0 S uwait 0xfffff8003c53bf00 syz-executor.2 3722 0 0 0 DL aiordy 0xfffff8003f6d4538 [aiod4] 3721 0 0 0 DL aiordy 0xfffff8003f6d4a70 [aiod3] 3720 0 0 0 DL aiordy 0xfffff8003cdcb538 [aiod2] 3719 0 0 0 DL aiordy 0xfffff80015ef9a70 [aiod1] 3559 1 3553 0 S uwait 0xfffff8003c53eb00 syz-executor.1 3557 1 3553 0 S uwait 0xfffff8003cac3f00 syz-executor.1 1076 1 1076 0 Ss select 0xfffff8003c43aa40 dhclient 1070 1 1070 0 Ss select 0xfffff8003c2238c0 dhclient 863 780 863 0 Rs syz-executor.3 792 780 792 0 Rs syz-executor.2 784 780 784 0 Rs syz-executor.1 783 780 783 0 Rs syz-executor.0 780 778 778 0 S (threaded) syz-fuzzer 100105 S uwait 0xfffff8003c323b00 syz-fuzzer 100118 S uwait 0xfffff80015d09900 syz-fuzzer 100119 S uwait 0xfffff80015d09b00 syz-fuzzer 100120 S uwait 0xfffff80015d09d00 syz-fuzzer 100121 S uwait 0xfffff8003c53e700 syz-fuzzer 100122 S uwait 0xfffff8003c53e900 syz-fuzzer 100123 S kqread 0xfffff80015d06100 syz-fuzzer 100124 S uwait 0xfffff80015ac3100 syz-fuzzer 102659 S uwait 0xfffff80015e5fb00 syz-fuzzer 102660 S uwait 0xfffff8003ce1bf00 syz-fuzzer 778 776 778 0 Ss pause 0xfffff80015ef95e8 csh 776 694 776 0 Ss select 0xfffff8003c43a740 sshd 760 1 760 0 Ss+ ttyin 0xfffff800154a5cb0 getty 759 1 759 0 Ss+ ttyin 0xfffff80015b2e4b0 getty 758 1 758 0 Ss+ ttyin 0xfffff80015b2ecb0 getty 757 1 757 0 Ss+ ttyin 0xfffff80015ab64b0 getty 756 1 756 0 Ss+ ttyin 0xfffff80015ab6cb0 getty 755 1 755 0 Ss+ ttyin 0xfffff80015ab84b0 getty 754 1 754 0 Ss+ ttyin 0xfffff80015ab8cb0 getty 753 1 753 0 Ss+ ttyin 0xfffff80015abf4b0 getty 752 1 752 0 Ss+ ttyin 0xfffff80015abfcb0 getty 698 1 698 0 Ss nanslp 0xffffffff8273f0a0 cron 694 1 694 0 Ss select 0xfffff8003c2232c0 sshd 507 1 507 0 Ss select 0xfffff8003c423d40 syslogd 436 1 436 0 Ss select 0xfffff8003c223d40 devd 435 1 435 65 Ss select 0xfffff8003c423440 dhclient 350 1 350 0 Ss select 0xfffff80015e0fd40 dhclient 347 1 347 0 Ss select 0xfffff80015ef8ec0 dhclient 23 0 0 0 DL vlruwt 0xfffff80015cd1538 [vnlru] 22 0 0 0 DL syncer 0xffffffff8282e450 [syncer] 21 0 0 0 DL (threaded) [bufdaemon] 100081 D qsleep 0xffffffff8282d520 [bufdaemon] 100086 D - 0xffffffff8220aec0 [bufspacedaemon-0] 100099 D sdflush 0xfffff80004ddc4e8 [/ worker] 20 0 0 0 DL psleep 0xffffffff82855388 [vmdaemon] 19 0 0 0 DL (threaded) [pagedaemon] 100079 D psleep 0xffffffff828497f8 [dom0] 100087 D launds 0xffffffff82849804 [laundry: dom0] 100088 D umarcl 0xffffffff815c58a0 [uma] 18 0 0 0 DL - 0xffffffff82573458 [rand_harvestq] 17 0 0 0 DL waiting 0xffffffff82ff08b8 [sctp_iterator] 16 0 0 0 DL pftm 0xffffffff830ea410 [pf purge] 15 0 0 0 DL - 0xffffffff8282abdc [soaiod4] 9 0 0 0 DL - 0xffffffff8282abdc [soaiod3] 8 0 0 0 DL - 0xffffffff8282abdc [soaiod2] 7 0 0 0 DL - 0xffffffff8282abdc [soaiod1] 6 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff8244ab40 [doneq0] 100045 D - 0xffffffff8244aac0 [async] 100078 D - 0xffffffff8244a990 [scanner] 14 0 0 0 DL seqstat 0xfffff80004d8a488 [sequencer 00] 5 0 0 0 DL crypto_ 0xfffff80004d53d80 [crypto returns 1] 4 0 0 0 DL crypto_ 0xfffff80004d53d30 [crypto returns 0] 3 0 0 0 DL crypto_ 0xffffffff82846ce0 [crypto] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff8271e8e0 [g_event] 100036 D - 0xffffffff8271e8e8 [g_up] 100037 D - 0xffffffff8271e8f0 [g_down] 2 0 0 0 DL (threaded) [KTLS] 100028 D - 0xfffff80004c69100 [thr_0] 100029 D - 0xfffff80004c69180 [thr_1] 12 0 0 0 RL (threaded) [intr] 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 I [swi3: vm] 100031 I [swi4: clock (0)] 100032 I [swi4: clock (1)] 100033 I [swi1: netisr 0] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100084 I [swi1: hpts] 100085 Run CPU 1 [swi1: hpts] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffff80004bda538 [init] 10 0 0 0 DL audit_w 0xffffffff828471f0 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff8271ee70 [swapper] 100005 D - 0xfffff8000443b500 [if_io_tqg_0] 100006 D - 0xfffff8000443b100 [if_io_tqg_1] 100007 D - 0xfffff80004c78d00 [if_config_tqg_0] 100008 D - 0xfffff80004c78900 [softirq_0] 100009 D - 0xfffff80004c78500 [softirq_1] 100010 D - 0xfffff80004c78100 [aiod_kick taskq] 100012 D - 0xfffff80004c75900 [kqueue_ctx taskq] 100013 D - 0xfffff80004c75500 [pci_hp taskq] 100015 D - 0xfffff80004c6ed00 [inm_free taskq] 100017 D - 0xfffff80004c6e500 [linuxkpi_irq_wq] 100018 D - 0xfffff80004c6e100 [thread taskq] 100019 D - 0xfffff80004c69d00 [in6m_free taskq] 100020 D - 0xfffff80004c69900 [linuxkpi_short_wq_0] 100021 D - 0xfffff80004c69900 [linuxkpi_short_wq_1] 100022 D - 0xfffff80004c69900 [linuxkpi_short_wq_2] 100023 D - 0xfffff80004c69900 [linuxkpi_short_wq_3] 100024 D - 0xfffff80004c69500 [linuxkpi_long_wq_0] 100025 D - 0xfffff80004c69500 [linuxkpi_long_wq_1] 100026 D - 0xfffff80004c69500 [linuxkpi_long_wq_2] 100027 D - 0xfffff80004c69500 [linuxkpi_long_wq_3] 100034 D - 0xfffff80004cd3d00 [firmware taskq] 100038 D - 0xfffff80004cd3900 [crypto_0] 100039 D - 0xfffff80004cd3900 [crypto_1] 100055 D - 0xfffff800153ebd00 [vtnet0 rxq 0] 100056 D - 0xfffff800153eb900 [vtnet0 txq 0] 100057 D - 0xfffff800153eb500 [vtnet0 rxq 1] 100058 D - 0xfffff800153eb100 [vtnet0 txq 1] 100060 D vtbslp 0xfffff80015463700 [virtio_balloon] 100064 D - 0xfffff80015468900 [mca taskq] 100066 D - 0xffffffff81e225e1 [deadlkres] 100074 D - 0xfffff80015ad5100 [acpi_task_0] 100075 D - 0xfffff80015ad5100 [acpi_task_1] 100076 D - 0xfffff80015ad5100 [acpi_task_2] 100077 D - 0xfffff80004cd3500 [CAM taskq] db> show all locks Process 12 (intr) thread 0xfffffe0055378740 (100085) exclusive rw tcpinp (tcpinp) r = 0 (0xfffff8003f2c33f0) locked @ /syzkaller/managers/i386/kernel/sys/netinet/tcp_hpts.c:1536 db> show malloc Type InUse MemUse Requests sysctloid 34288 12858K 34355 pf_hash 5 11560K 5 devbuf 4216 6982K 4244 pcb 1070 5295K 35558 tcp_hpts 5 3219K 5 kobj 326 2608K 486 vtbuf 24 2064K 46 vfscache 3 1035K 3 newblk 6 1035K 26671 sctp_atcl 1021 1021K 18654 sctp_stro 501 1002K 5378 filedesc 52 826K 52209 inodedep 298 817K 26230 subproc 184 772K 27236 acpica 1674 649K 54806 sctp_atky 1523 572K 24756 callout 2 528K 2 ufs_quota 1 520K 1 vfs_hash 1 520K 1 intr 4 480K 4 sctp_athm 1021 383K 19340 bus 986 377K 3469 sctp_map 1002 376K 10668 pf_rule 85 340K 344 BPF 46 337K 65 linker 348 270K 433 DEVFS1 107 214K 124 kdtrace 262 210K 59583 module 510 192K 510 vnet_data 1 176K 1 umtx 396 174K 396 tidhash 3 164K 3 dirrem 289 145K 26132 pagedep 10 141K 26100 tfo_ccache 1 136K 1 sem 4 120K 4 freefile 288 108K 26115 UMA 270 102K 270 audit_evclass 236 89K 294 mtx_pool 2 80K 2 temp 35 77K 9917 syncache 1 76K 1 routetbl 141 76K 494 vmem 3 72K 5 msg 4 68K 4 ifaddr 71 67K 73 sctp_timw 132 66K 132 acpitask 1 64K 1 ddb_capture 1 64K 1 DEVFS3 126 63K 136 ether_multi 157 59K 167 gtaskqueue 18 57K 18 DEVFS_RULE 56 54K 56 kenv 95 52K 95 eventhandler 133 50K 133 ithread 99 43K 99 rman 84 42K 425 kqueue 86 40K 29912 lltable 44 38K 80 KTRACE 100 38K 100 taskqueue 60 36K 60 proc 3 34K 3 DEVFSP 90 34K 12341 ufs_mount 5 34K 6 bus-sc 33 34K 1684 devstat 4 33K 4 hostcache 1 32K 1 tty 16 32K 16 shm 1 32K 28 pwddesc 84 32K 27135 in6_multi 71 31K 71 GEOM 60 29K 489 kbdmux 6 28K 6 cred 26 26K 311 ifnet 7 25K 7 plimit 24 24K 499 kcovinfo 64 24K 68 session 44 22K 182 ksem 4 21K 37 CAM queue 5 21K 1528 selfd 55 21K 369020 pfs_nodes 20 20K 20 proc-args 52 20K 735 UART 12 18K 12 bmsafemap 2 17K 26216 rpc 2 16K 2 shmfd 1 16K 115 pfs_vncache 1 16K 1 ufs_dirhash 24 14K 24 sglist 5 13K 5 lockf 33 13K 133 MCA 32 12K 32 CAM DEV 3 12K 510 vt 11 11K 11 Unitno 29 11K 47 crypto 22 11K 1701 acpisem 28 11K 28 CAM XPT 22 11K 543 uidinfo 3 9K 16 pf_ifnet 10 9K 19 inpcbpolicy 22 9K 12802 memdesc 1 8K 1 ipsec-saq 2 8K 2 evdev 4 8K 4 xform 21 8K 14981 acpidev 20 8K 20 hhook 15 8K 17 select 19 8K 135 mount 16 7K 117 fpukern_ctx 3 6K 3 terminal 11 6K 11 sctp_ifa 14 6K 15 ip6ndp 12 6K 15 osd 13 5K 11599 ipsecpolicy 2 5K 2 encap_export_host 12 5K 12 clone 9 5K 9 pf_table 1 4K 14 local_apic 1 4K 1 io_apic 1 4K 1 sahead 1 4K 1 secasvar 1 4K 1 nhops 6 4K 8 pci_link 10 4K 10 msi 9 4K 9 DEVFS 9 4K 10 tun 7 4K 7 ipsec 3 3K 3 mkdir 8 3K 52156 in_multi 6 3K 8 nexusdev 7 3K 7 feeder 7 3K 7 procdesc 5 3K 18 sctp_ifn 6 3K 15 mld 6 3K 6 igmp 6 3K 6 toponodes 6 3K 6 prison 6 3K 6 isadev 6 3K 6 softdep 1 2K 1 vnodemarker 2 2K 108 NFSD session 1 2K 1 newdirblk 5 2K 26078 lkpikmalloc 5 2K 6 CAM periph 4 2K 271 filecaps 5 2K 124 diradd 4 2K 26174 pfil 4 2K 4 CAM path 4 2K 1034 soname 4 2K 45154 tcpfunc 3 2K 3 loginclass 3 2K 6 chacha20random 1 1K 1 vnodes 1 1K 1 ktls 1 1K 1 cdev 2 1K 2 aesni_data 2 1K 2 tcp_fsb 2 1K 617 cpus 2 1K 2 atkbddev 2 1K 2 CAM dev queue 2 1K 2 entropy 2 1K 49 CAM SIM 2 1K 2 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 pmchooks 1 1K 1 sctp_vrf 1 1K 1 apmdev 1 1K 1 freework 1 1K 26076 CAM I/O Scheduler 1 1K 1 vnet_data_free 1 1K 1 vnet 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 acpiintr 1 1K 1 pmc 1 1K 1 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_temp 0 0K 0 sctp_mcore 0 0K 0 sctp_socko 0 0K 10922 sctp_iter 0 0K 23 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 13 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 10 sctp_aadr 0 0K 1 sctp_stri 0 0K 1362 mqdata 0 0K 0 cdg data 0 0K 17 chd data 0 0K 0 cubic data 0 0K 30 tcp_do 0 0K 0 dctcp data 0 0K 15 htcp data 0 0K 0 vegas data 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 iavf 0 0K 0 ixl 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 axgbe 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xen_intr 0 0K 0 xenstore 0 0K 0 ciss_data 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 qpidrv 0 0K 0 BACKLIGHT 0 0K 0 xnb 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 sysmouse 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 vtfont 0 0K 0 vm_fictitious 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 amr 0 0K 0 pvscsi 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 UMAHash 0 0K 0 scsi_cd 0 0K 0 AHCI driver 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 20846 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 51 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freeblks 0 0K 26075 freefrag 0 0K 6 allocindir 0 0K 0 indirdep 0 0K 10 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 agp 0 0K 0 nvme_da 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 ktls_ocf 0 0K 0 twsbuf 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpifw 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 101 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 twe_commands 0 0K 0 LRO 0 0K 0 newreno data 0 0K 2 ip_msource 0 0K 0 ip_moptions 0 0K 15 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 0 VN POLL 0 0K 0 twa_commands 0 0K 0 statfs 0 0K 26334 namei_tracker 0 0K 2 export_host 0 0K 0 cl_savebuf 0 0K 4 tcp_log_dev 0 0K 3714 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 3004 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 biobuf 0 0K 0 aios 0 0K 28 lio 0 0K 34 acl 0 0K 0 tempbuff 0 0K 0 mbuf_tag 0 0K 131 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 accf 0 0K 0 pts 0 0K 0 iov 0 0K 38801 ioctlops 0 0K 579 eventfd 0 0K 44 Witness 0 0K 0 stack 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 sbuf 0 0K 288 md_sectors 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 md_disk 0 0K 0 SWAP 0 0K 0 malodev 0 0K 0 LED 0 0K 0 sysctltmp 0 0K 958 sysctl 0 0K 3 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 ix_sriov 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 cache 0 0K 0 iirbuf 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 1 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8322 969 1033744 0 254 38055936 0 malloc-384 384 36501 79 49792 0 30 14046720 0 tcp_log 416 0 13302 656215 0 254 5533632 0 malloc-1024 1024 377 4599 26882 0 16 5095424 0 malloc-8192 8192 522 14 12644 0 1 4390912 0 malloc-1024 1024 4174 6 4196 0 16 4280320 0 malloc-512 512 429 4979 26791 0 30 2768896 0 malloc-8192 8192 328 2 1099 0 1 2703360 0 pbuf 2624 0 989 0 0 2 2595136 0 mbuf 256 8844 711 6122484 0 254 2446080 0 malloc-384 384 858 5052 144648 0 30 2269440 0 malloc-1024 1024 133 1899 22600 0 16 2080768 0 BUF TRIE 144 188 13280 1363 0 62 1939392 0 RADIX NODE 144 12523 354 635212 0 63 1854288 0 UMA Slabs 0 112 15717 12 15903 0 126 1761648 0 malloc-384 384 3648 82 455993 0 30 1432320 0 mbuf_cluster 2048 690 0 690 0 254 1413120 0 sctp_asoc 2288 501 64 5334 0 254 1292720 0 malloc-1024 1024 1026 46 20932 0 16 1097728 0 malloc-2048 2048 503 23 6698 0 8 1077248 0 malloc-32768 32768 28 4 26105 0 1 1048576 0 sctp_ep 1280 520 86 12638 0 254 775680 0 malloc-1024 1024 709 35 45301 0 16 761856 0 malloc-8192 8192 86 4 27137 0 1 737280 0 malloc-384 384 1725 35 53852 0 30 675840 0 FFS inode 1160 531 22 26647 0 8 641480 0 sctp_raddr 736 513 323 8764 0 254 615296 0 tcp_bbr_map 128 6 4551 1172018 0 126 583296 0 socket 944 88 392 26859 0 254 453120 0 ertt_txseginfo 40 1 10806 4444771 0 254 432280 0 VM OBJECT 264 1500 60 387842 0 30 411840 0 malloc-16384 16384 24 1 31 0 1 409600 0 256 Bucket 2048 182 16 35985 0 8 405504 0 malloc-4096 4096 92 2 367 0 2 385024 0 lkpimm 160 1 2324 1 0 62 372000 0 lkpicurr 160 2 2323 2 0 62 372000 0 THREAD 1808 177 21 32447 0 8 357984 0 tcpcb 1080 9 278 11587 0 254 309960 0 malloc-512 512 28 556 28956 0 30 299008 0 malloc-384 384 626 144 13531 0 30 295680 0 VNODE 448 570 87 26688 0 30 294336 0 MAP ENTRY 96 2532 282 1467572 0 126 270144 0 malloc-65536 65536 4 0 4 0 1 262144 0 malloc-2048 2048 125 3 146 0 8 262144 0 tcp_inpcb 488 22 506 11587 0 254 257664 0 malloc-4096 4096 8 54 3979 0 2 253952 0 DEVCTL 1024 0 220 154 0 0 225280 0 malloc-8192 8192 23 2 53 0 1 204800 0 malloc-384 384 386 144 116101 0 30 203520 0 vmem btag 56 3477 111 3477 0 254 200928 0 malloc-65536 65536 3 0 3 0 1 196608 0 malloc-16384 16384 11 1 247 0 1 196608 0 UMA Zones 768 242 2 242 0 16 187392 0 mbuf_packet 256 158 532 41954 0 254 176640 0 VMSPACE 2544 56 10 27112 0 4 167904 0 32 Bucket 256 391 224 36919 0 62 157440 0 malloc-384 384 332 68 43415 0 30 153600 0 S VFS Cache 104 1037 367 27471 0 126 146016 0 FFS2 dinode 256 531 39 26646 0 62 145920 0 malloc-512 512 189 75 57512 0 30 135168 0 PROC 1336 83 16 27134 0 8 132264 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-65536 65536 0 2 176 0 1 131072 0 malloc-512 512 217 31 29135 0 30 126976 0 ksiginfo 112 84 960 1954 0 126 116928 0 malloc-16384 16384 2 5 36 0 1 114688 0 malloc-16384 16384 4 3 10 0 1 114688 0 malloc-1024 1024 112 0 118 0 16 114688 0 filedesc0 1072 84 21 27135 0 8 112560 0 128 Bucket 1024 79 28 13098 0 16 109568 0 tcp_bbr_pcb 832 2 124 9171 0 16 104832 0 malloc-32768 32768 1 2 121 0 1 98304 0 16 Bucket 144 431 241 6879 0 62 96768 0 g_bio 408 0 230 9481 0 30 93840 0 UMA Kegs 384 227 6 227 0 30 89472 0 malloc-512 512 129 31 26655 0 30 81920 0 sctp_chunk 152 491 29 2417 0 254 79040 0 malloc-512 512 146 6 4401 0 30 77824 0 malloc-65536 65536 0 1 1 0 1 65536 0 malloc-32768 32768 1 1 16 0 1 65536 0 malloc-16384 16384 2 2 8 0 1 65536 0 malloc-8192 8192 3 5 32 0 1 65536 0 malloc-4096 4096 12 3 175 0 2 61440 0 malloc-2048 2048 10 20 518 0 8 61440 0 malloc-384 384 95 65 36354 0 30 61440 0 malloc-8192 8192 2 5 26709 0 1 57344 0 64 Bucket 512 47 57 12755 0 30 53248 0 malloc-16384 16384 3 0 3 0 1 49152 0 malloc-16384 16384 3 0 3 0 1 49152 0 malloc-512 512 52 36 1905 0 30 45056 0 pcpu-8 8 5399 233 7842 0 254 45056 0 Files 80 299 251 80186 0 126 44000 0 clpbuf 2624 0 16 20 0 16 41984 0 malloc-2048 2048 2 18 169 0 8 40960 0 malloc-2048 2048 16 4 25 0 8 40960 0 malloc-512 512 23 57 1444 0 30 40960 0 DIRHASH 1024 34 2 34 0 16 36864 0 NAMEI 1024 0 36 136950 0 16 36864 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-4096 4096 6 2 3334 0 2 32768 0 pcpu-64 64 487 25 487 0 254 32768 0 pipe 744 23 17 826 0 16 29760 0 TURNSTILE 136 199 11 199 0 62 28560 0 PWD 32 33 849 26178 0 254 28224 0 KNOTE 160 28 147 663816 0 62 28000 0 8 Bucket 80 44 306 6139 0 126 28000 0 malloc-8192 8192 1 2 58 0 1 24576 0 malloc-4096 4096 1 5 10 0 2 24576 0 malloc-4096 4096 4 2 72 0 2 24576 0 ertt 72 9 327 11587 0 126 24192 0 ttyinq 160 135 15 300 0 62 24000 0 ttyoutq 256 72 18 160 0 62 23040 0 tcp_rack_pcb 832 2 25 617 0 16 22464 0 2 Bucket 32 111 519 6686 0 254 20160 0 SLEEPQUEUE 88 199 25 199 0 126 19712 0 ripcb 488 5 35 751 0 254 19520 0 Mountpoints 2752 2 5 2 0 4 19264 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-8192 8192 0 2 13 0 1 16384 0 malloc-4096 4096 2 2 66 0 2 16384 0 vtnet_tx_hdr 24 0 668 4894508 0 254 16032 0 udp_inpcb 488 8 24 335 0 254 15616 0 unpcb 256 38 22 1369 0 254 15360 0 malloc-2048 2048 3 3 163 0 8 12288 0 malloc-2048 2048 2 4 6 0 8 12288 0 sctp_stream_msg_out 112 37 71 2228 0 254 12096 0 sctp_readq 152 0 78 816 0 254 11856 0 udplite_inpcb 488 0 24 129 0 254 11712 0 kenv 258 15 30 1042 0 30 11610 0 routing nhops 256 27 18 35 0 62 11520 0 malloc-2048 2048 4 0 4 0 8 8192 0 malloc-1024 1024 1 7 13 0 16 8192 0 malloc-1024 1024 0 8 21 0 16 8192 0 pf tags 104 5 73 66 0 126 8112 0 rtentry 176 31 15 35 0 62 8096 0 PGRP 88 44 48 284 0 126 8096 0 rl_entry 40 80 122 80 0 254 8080 0 sctp_laddr 48 8 160 417 0 254 8064 0 tcp_rack_map 112 2 70 751 0 126 8064 0 udpcb 32 8 244 464 0 254 8064 0 4 Bucket 48 6 162 571 0 254 8064 0 AIO 208 0 38 71 0 62 7904 0 cryptop 280 0 28 39 0 30 7840 0 itimer 352 0 22 2 0 30 7744 0 AIOCB 552 0 14 128 0 16 7728 0 AIOLIO 272 0 28 17 0 30 7616 0 FPU_save_area 832 1 8 1 0 16 7488 0 cpuset 104 7 55 7 0 126 6448 0 epoch_record pcpu 256 4 12 4 0 62 4096 0 malloc-4096 4096 1 0 1 0 2 4096 0 pcpu-16 16 7 249 7 0 254 4096 0 hostcache 64 1 62 1 0 254 4032 0 syncache 168 0 24 4 0 254 4032 0 AIOP 32 4 122 4 0 254 4032 0 UMA Slabs 1 176 10 12 10 0 62 3872 0 mqnode 416 3 6 3 0 30 3744 0 KMAP ENTRY 96 12 27 12 0 0 3744 0 vmem 1856 1 1 1 0 8 3712 0 SMR CPU 32 3 60 3 0 254 2016 0 SMR SHARED 24 3 60 3 0 254 1512 0 FFS1 dinode 128 0 0 0 0 126 0 0 da_ccb 544 0 0 0 0 16 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 62 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 304 0 0 0 0 254 0 0 pf mtags 48 0 0 0 0 254 0 0 tcp_log_node 120 0 0 0 0 126 0 0 tcp_log_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 88 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 NCLNODE 584 0 0 0 0 16 0 0 TMPFS node 224 0 0 0 0 62 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 linux_dma_object 24 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 ktls_session 128 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 domainset 40 0 0 0 0 254 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0