panic: mtx_lock() of spin mutex (null) @ /syzkaller/managers/i386/kernel/sys/modules/tcp/bbr/../../../netinet/tcp_stacks/bbr.c:12379 cpuid = 0 time = 1623038255 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x47/frame 0xfffffe00516e05d0 vpanic() at vpanic+0x1c7/frame 0xfffffe00516e0630 panic() at panic+0x43/frame 0xfffffe00516e0690 __mtx_lock_flags() at __mtx_lock_flags+0x202/frame 0xfffffe00516e06f0 bbr_output_wtime() at bbr_output_wtime+0x6b02/frame 0xfffffe00516e0990 bbr_output() at bbr_output+0x67/frame 0xfffffe00516e09c0 tcp_hpts_thread() at tcp_hpts_thread+0xe34/frame 0xfffffe00516e0b10 ithread_loop() at ithread_loop+0x33f/frame 0xfffffe00516e0bb0 fork_exit() at fork_exit+0xb3/frame 0xfffffe00516e0bf0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00516e0bf0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- KDB: enter: panic [ thread pid 12 tid 100084 ] Stopped at kdb_enter+0x67: movq $0,0x1635ede(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0 rax 0x12 rcx 0xffffffff8113c280 vprintf+0x140 rdx 0x1 rbx 0 rsp 0xfffffe00516e05b0 rbp 0xfffffe00516e05d0 rsi 0 rdi 0xffffffff8113c2b6 vprintf+0x176 r8 0 r9 0x8080808080808080 r10 0xfffffe00516e04a0 r11 0x1ffaefff59c r12 0xffffffff82267ba0 ddb_dbbe r13 0 r14 0xffffffff81a7ca3a r15 0xffffffff81a7ca3a rip 0xffffffff811332a7 kdb_enter+0x67 rflags 0x82 kdb_enter+0x67: movq $0,0x1635ede(%rip) db> show proc Process 12 (intr) at 0xfffff80004c6fa60: state: NORMAL uid: 0 gids: 0 parent: pid 0 at 0xffffffff8271c6b0 ABI: null flag: 0x10000284 flag2: 0 reaper: 0xffffffff8271c6b0 reapsubtree: 12 sigparent: 20 vmspace: 0xffffffff8271d310 (map 0xffffffff8271d310) (map.pmap 0xffffffff8271d3d0) (pmap 0xffffffff8271d430) threads: 23 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 I [swi3: vm] 100031 I [swi4: clock (0)] 100032 I [swi4: clock (1)] 100033 I [swi1: netisr 0] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100084 Run CPU 0 [swi1: hpts] 100085 I [swi1: hpts] db> ps pid ppid pgrp uid state wmesg wchan cmd 26202 784 784 0 S (threaded) syz-executor.2 129751 S nanslp 0xffffffff8273c8e0 syz-executor.2 131316 S connec 0xfffff80069d50840 syz-executor.2 131318 S uwait 0xfffff800698e3500 syz-executor.2 26201 839 839 0 S (threaded) syz-executor.3 131309 S nanslp 0xffffffff8273c8e1 syz-executor.3 131314 S connec 0xfffff80069f9e490 syz-executor.3 131317 S uwait 0xfffff8004fea6900 syz-executor.3 26200 783 783 0 S (threaded) syz-executor.0 130995 S nanslp 0xffffffff8273c8e1 syz-executor.0 131311 S connec 0xfffff80069df2490 syz-executor.0 131313 S uwait 0xfffff80078ce5f00 syz-executor.0 26199 793 26199 0 Ss (threaded) syz-executor.1 131258 S nanslp 0xffffffff8273c8e1 syz-executor.1 131310 S connec 0xfffff80069789bf0 syz-executor.1 131312 S uwait 0xfffff80069dc8500 syz-executor.1 26096 1 784 0 S uwait 0xfffff80069c8c300 syz-executor.2 26088 1 784 0 S uwait 0xfffff8004ffc7500 syz-executor.2 25002 1 24994 0 S uwait 0xfffff800697f2100 syz-executor.2 24997 1 784 0 S uwait 0xfffff8004ff42700 syz-executor.2 24032 24031 839 0 SV uwait 0xfffff80069c8cf00 syz-executor.3 24031 1 839 0 DV ppwait 0xfffff800785dc4e0 syz-executor.3 19414 1 783 0 SV sigwait 0xfffffe0095532000 syz-executor.0 19123 1 784 0 S uwait 0xfffff800697f2500 syz-executor.2 19120 1 784 0 S uwait 0xfffff8004fa33700 syz-executor.2 19099 1 19095 0 S uwait 0xfffff80069922300 syz-executor.2 19097 1 784 0 S uwait 0xfffff800697f2900 syz-executor.2 19070 19067 783 0 SV uwait 0xfffff8004fa34f00 syz-executor.0 19067 1 783 0 DV ppwait 0xfffff800785be4e0 syz-executor.0 19033 1 839 0 S uwait 0xfffff8004ff13b00 syz-executor.3 19028 1 839 0 S uwait 0xfffff8004fc8f900 syz-executor.3 18774 1 784 0 SV uwait 0xfffff8004ffc7100 syz-executor.2 18765 1 784 0 SV sigwait 0xfffffe009555d000 syz-executor.2 18262 1 793 0 S uwait 0xfffff8004ffc3f00 syz-executor.1 18261 1 793 0 S uwait 0xfffff8006914f500 syz-executor.1 17953 1 784 0 SV uwait 0xfffff80015ec7900 syz-executor.2 17946 1 784 0 SV uwait 0xfffff8004fa34500 syz-executor.2 16255 1 16251 0 S uwait 0xfffff8004f552100 syz-executor.0 16253 1 783 0 S uwait 0xfffff8004ffc3900 syz-executor.0 16037 1 783 0 S uwait 0xfffff8004fd7bb00 syz-executor.0 16036 1 783 0 S uwait 0xfffff800697f2f00 syz-executor.0 15884 1 783 0 S uwait 0xfffff8004fa34d00 syz-executor.0 15880 1 783 0 S uwait 0xfffff80015ec7700 syz-executor.0 15804 15803 783 0 SV uwait 0xfffff8006900d500 syz-executor.0 15803 1 783 0 DV ppwait 0xfffff8004f4d8f40 syz-executor.0 7194 1 784 0 S uwait 0xfffff8004ffc3b00 syz-executor.2 7190 1 784 0 S uwait 0xfffff8004fa33b00 syz-executor.2 6105 1 839 0 S uwait 0xfffff8004ffc7300 syz-executor.3 6104 1 839 0 S uwait 0xfffff8004fa34700 syz-executor.3 6092 1 839 0 S uwait 0xfffff8004f552900 syz-executor.3 6086 1 839 0 S uwait 0xfffff8006914ff00 syz-executor.3 5235 1 5235 65 Ss select 0xfffff800691c3a40 dhclient 4485 1 4485 0 Ss select 0xfffff8006908d140 dhclient 4482 1 4482 0 Ss select 0xfffff800691c38c0 dhclient 4463 1 4463 65 Ss select 0xfffff800691c3bc0 dhclient 4075 1 4075 0 Ss select 0xfffff8006932c5c0 dhclient 4072 1 4072 0 Ss select 0xfffff8004ff3dd40 dhclient 4053 1 4053 65 Ss select 0xfffff80069146ec0 dhclient 3204 3203 3203 0 SV uwait 0xfffff80015f0fd00 syz-executor.1 3203 1 3203 0 DV ppwait 0xfffff800690a1f40 syz-executor.1 2602 1 2602 0 Ss select 0xfffff8004ffddd40 dhclient 2599 1 2599 0 Ss select 0xfffff8004fea4bc0 dhclient 2580 1 2580 65 Ss select 0xfffff8006932cbc0 dhclient 1282 0 0 0 DL aiordy 0xfffff800690f5a60 [aiod4] 1281 0 0 0 DL aiordy 0xfffff80015e90000 [aiod3] 1280 0 0 0 DL aiordy 0xfffff800690b9a60 [aiod2] 1279 0 0 0 DL aiordy 0xfffff800690b9000 [aiod1] 1119 1 1119 0 Ss select 0xfffff8004fe69440 dhclient 1116 1 1116 0 Ss select 0xfffff8004f9a3140 dhclient 839 780 839 0 Ss nanslp 0xffffffff8273c8e0 syz-executor.3 793 780 793 0 Ss nanslp 0xffffffff8273c8e1 syz-executor.1 784 780 784 0 Ss nanslp 0xffffffff8273c8e1 syz-executor.2 783 780 783 0 Ss nanslp 0xffffffff8273c8e0 syz-executor.0 780 778 778 0 S (threaded) syz-fuzzer 100114 S uwait 0xfffff80015ce0d00 syz-fuzzer 100115 S uwait 0xfffff80015ec7b00 syz-fuzzer 100116 S uwait 0xfffff80015ec7d00 syz-fuzzer 100117 S uwait 0xfffff80015f0f900 syz-fuzzer 100118 S uwait 0xfffff80015f0fb00 syz-fuzzer 100119 S uwait 0xfffff80015ce0f00 syz-fuzzer 100120 S kqread 0xfffff8004f194f00 syz-fuzzer 100122 S uwait 0xfffff8004fa34300 syz-fuzzer 101888 S uwait 0xfffff8004ffc3500 syz-fuzzer 778 776 778 0 Ss pause 0xfffff80015e725e0 csh 776 694 776 0 Ss select 0xfffff8004f9a3ec0 sshd 760 1 760 0 Ss+ ttyin 0xfffff8001541bcb0 getty 759 1 759 0 Ss+ ttyin 0xfffff80015aa54b0 getty 758 1 758 0 Ss+ ttyin 0xfffff80015aa5cb0 getty 757 1 757 0 Ss+ ttyin 0xfffff80015aa94b0 getty 756 1 756 0 Ss+ ttyin 0xfffff80015aa9cb0 getty 755 1 755 0 Ss+ ttyin 0xfffff80015aad4b0 getty 754 1 754 0 Ss+ ttyin 0xfffff80015aadcb0 getty 753 1 753 0 Ss+ ttyin 0xfffff80015a364b0 getty 752 1 752 0 Ss+ ttyin 0xfffff80015a36cb0 getty 698 1 698 0 Ss nanslp 0xffffffff8273c8e1 cron 694 1 694 0 Ss select 0xfffff8004f18f140 sshd 507 1 507 0 Ss select 0xfffff80015eaf2c0 syslogd 436 1 436 0 Ss select 0xfffff8004f2e0ec0 devd 435 1 435 65 Ss select 0xfffff8004f28d5c0 dhclient 350 1 350 0 Ss select 0xfffff80015ee65c0 dhclient 347 1 347 0 Ss select 0xfffff80015eaf140 dhclient 23 0 0 0 DL vlruwt 0xfffff80015a8d000 [vnlru] 22 0 0 0 DL syncer 0xffffffff8282bcd0 [syncer] 21 0 0 0 DL (threaded) [bufdaemon] 100081 D qsleep 0xffffffff8282ada0 [bufdaemon] 100088 D - 0xffffffff8220aec0 [bufspacedaemon-0] 100098 D sdflush 0xfffff80004d594e8 [/ worker] 20 0 0 0 DL psleep 0xffffffff82852c08 [vmdaemon] 19 0 0 0 DL (threaded) [pagedaemon] 100079 D psleep 0xffffffff82847078 [dom0] 100086 D launds 0xffffffff82847084 [laundry: dom0] 100087 D umarcl 0xffffffff815d0d50 [uma] 18 0 0 0 DL - 0xffffffff82570c88 [rand_harvestq] 17 0 0 0 DL waiting 0xffffffff830e4828 [sctp_iterator] 16 0 0 0 DL pftm 0xffffffff82e1d3c0 [pf purge] 15 0 0 0 DL - 0xffffffff8282845c [soaiod4] 9 0 0 0 DL - 0xffffffff8282845c [soaiod3] 8 0 0 0 DL - 0xffffffff8282845c [soaiod2] 7 0 0 0 DL - 0xffffffff8282845c [soaiod1] 6 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff82448140 [doneq0] 100045 D - 0xffffffff824480c0 [async] 100078 D - 0xffffffff82447f90 [scanner] 14 0 0 0 DL seqstat 0xfffff80004debc88 [sequencer 00] 5 0 0 0 DL crypto_ 0xfffff80004d3dd80 [crypto returns 1] 4 0 0 0 DL crypto_ 0xfffff80004d3dd30 [crypto returns 0] 3 0 0 0 DL crypto_ 0xffffffff82844560 [crypto] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff8271c120 [g_event] 100036 D - 0xffffffff8271c128 [g_up] 100037 D - 0xffffffff8271c130 [g_down] 2 0 0 0 DL (threaded) [KTLS] 100028 D - 0xfffff80004c4ad00 [thr_0] 100029 D - 0xfffff80004c4ad80 [thr_1] 12 0 0 0 RL (threaded) [intr] 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 I [swi3: vm] 100031 I [swi4: clock (0)] 100032 I [swi4: clock (1)] 100033 I [swi1: netisr 0] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100084 Run CPU 0 [swi1: hpts] 100085 I [swi1: hpts] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 Run CPU 1 [idle: cpu1] 1 0 1 0 SLs wait 0xfffff80004bd7530 [init] 10 0 0 0 DL audit_w 0xffffffff82844a70 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff8271c6b0 [swapper] 100005 D - 0xfffff8000443a500 [if_io_tqg_0] 100006 D - 0xfffff8000443a100 [if_io_tqg_1] 100007 D - 0xfffff80004c73d00 [if_config_tqg_0] 100008 D - 0xfffff80004c73900 [softirq_0] 100009 D - 0xfffff80004c73500 [softirq_1] 100010 D - 0xfffff80004c73100 [aiod_kick taskq] 100012 D - 0xfffff80004c70900 [kqueue_ctx taskq] 100013 D - 0xfffff80004c70500 [pci_hp taskq] 100015 D - 0xfffff80004c66d00 [inm_free taskq] 100017 D - 0xfffff80004c66500 [linuxkpi_irq_wq] 100018 D - 0xfffff80004c66100 [thread taskq] 100019 D - 0xfffff80004c63d00 [in6m_free taskq] 100020 D - 0xfffff80004c63900 [linuxkpi_short_wq_0] 100021 D - 0xfffff80004c63900 [linuxkpi_short_wq_1] 100022 D - 0xfffff80004c63900 [linuxkpi_short_wq_2] 100023 D - 0xfffff80004c63900 [linuxkpi_short_wq_3] 100024 D - 0xfffff80004c63500 [linuxkpi_long_wq_0] 100025 D - 0xfffff80004c63500 [linuxkpi_long_wq_1] 100026 D - 0xfffff80004c63500 [linuxkpi_long_wq_2] 100027 D - 0xfffff80004c63500 [linuxkpi_long_wq_3] 100034 D - 0xfffff80004c63100 [firmware taskq] 100038 D - 0xfffff80004db8d00 [crypto_0] 100039 D - 0xfffff80004db8d00 [crypto_1] 100055 D - 0xfffff80004db8100 [vtnet0 rxq 0] 100056 D - 0xfffff80015361d00 [vtnet0 txq 0] 100057 D - 0xfffff80015361900 [vtnet0 rxq 1] 100058 D - 0xfffff80015361500 [vtnet0 txq 1] 100060 D vtbslp 0xfffff80015372300 [virtio_balloon] 100064 D - 0xfffff80015361100 [mca taskq] 100066 D - 0xffffffff81e29a00 [deadlkres] 100074 D - 0xfffff80015a32100 [acpi_task_0] 100075 D - 0xfffff80015a32100 [acpi_task_1] 100076 D - 0xfffff80015a32100 [acpi_task_2] 100077 D - 0xfffff80004db8500 [CAM taskq] db> show all locks Process 12 (intr) thread 0xfffffe0055378740 (100084) exclusive rw tcpinp (tcpinp) r = 0 (0xfffff800691dbb90) locked @ /syzkaller/managers/i386/kernel/sys/netinet/tcp_hpts.c:1536 db> show malloc Type InUse MemUse Requests sysctloid 34288 12858K 34355 pf_hash 5 11560K 5 devbuf 4216 6982K 4244 pf_rule 1732 6928K 4865 tcp_hpts 5 3219K 5 kobj 328 2624K 488 vtbuf 24 2064K 46 filedesc 92 1473K 50369 newblk 10 1037K 25770 vfscache 3 1035K 3 subproc 225 953K 26330 pcb 72 779K 40542 acpica 1674 649K 54692 inodedep 49 568K 26252 callout 2 528K 2 ufs_quota 1 520K 1 vfs_hash 1 520K 1 intr 4 480K 4 bus 990 378K 3485 BPF 46 337K 103 linker 348 270K 473 kdtrace 306 241K 57522 DEVFS1 107 214K 124 umtx 440 193K 440 module 512 192K 512 vnet_data 1 176K 1 temp 59 173K 11980 tidhash 3 164K 3 routetbl 249 157K 2626 pagedep 10 141K 25402 tfo_ccache 1 136K 1 sem 4 120K 4 pf_table 28 112K 547 vmem 3 104K 6 UMA 270 102K 270 audit_evclass 236 89K 294 mtx_pool 2 80K 2 syncache 1 76K 1 msg 4 68K 4 ifaddr 71 67K 73 acpitask 1 64K 1 ddb_capture 1 64K 1 DEVFS3 126 63K 136 ether_multi 157 59K 182 gtaskqueue 18 57K 18 DEVFS_RULE 56 54K 56 kenv 95 52K 95 sctp_timw 102 51K 102 eventhandler 133 50K 133 kqueue 106 48K 26389 ithread 99 43K 99 rman 84 42K 425 lltable 47 41K 414 pwddesc 104 39K 26204 KTRACE 100 38K 100 taskqueue 60 36K 60 proc 3 34K 3 ufs_mount 5 34K 6 bus-sc 33 34K 1696 devstat 4 33K 4 hostcache 1 32K 1 tty 16 32K 16 shm 1 32K 30 in6_multi 71 31K 71 DEVFSP 80 30K 3348 sctp_atcl 29 29K 20350 GEOM 60 29K 489 kbdmux 6 28K 6 cred 26 26K 343 ifnet 7 25K 7 plimit 24 24K 889 kcovinfo 64 24K 68 sctp_stro 11 22K 5975 CAM queue 5 21K 1528 selfd 55 21K 358866 dirrem 40 20K 25229 pfs_nodes 20 20K 20 proc-args 52 20K 722 session 36 18K 86 UART 12 18K 12 inpcbpolicy 45 17K 21381 ksem 2 17K 27 bmsafemap 2 17K 25661 rpc 2 16K 2 shmfd 1 16K 98 pfs_vncache 1 16K 1 sctp_atky 40 15K 26839 freefile 39 15K 25207 osd 38 15K 18282 ufs_dirhash 24 14K 24 sglist 5 13K 5 MCA 32 12K 32 CAM DEV 3 12K 510 lockf 32 12K 63 vt 11 11K 11 sctp_athm 29 11K 21077 Unitno 29 11K 47 pf_ifnet 14 11K 1049 acpisem 28 11K 28 CAM XPT 22 11K 543 uidinfo 3 9K 20 sctp_map 22 9K 11432 memdesc 1 8K 1 ipsec-saq 2 8K 2 evdev 4 8K 4 acpidev 20 8K 20 hhook 15 8K 17 select 19 8K 97 mount 16 7K 108 fpukern_ctx 3 6K 3 terminal 11 6K 11 sctp_ifa 14 6K 15 ip6ndp 12 6K 15 ipsecpolicy 2 5K 2 encap_export_host 12 5K 12 clone 9 5K 9 local_apic 1 4K 1 io_apic 1 4K 1 sahead 1 4K 1 secasvar 1 4K 1 nhops 6 4K 8 pci_link 10 4K 10 msi 9 4K 9 DEVFS 9 4K 10 tun 7 4K 7 ipsec 3 3K 3 mkdir 8 3K 50288 in_multi 6 3K 11 soname 8 3K 49666 nexusdev 7 3K 7 feeder 7 3K 7 procdesc 5 3K 18 freework 5 3K 25150 sctp_ifn 6 3K 15 mld 6 3K 6 igmp 6 3K 6 toponodes 6 3K 6 prison 6 3K 6 isadev 6 3K 6 softdep 1 2K 1 freeblks 4 2K 25137 vnodemarker 2 2K 1672 NFSD session 1 2K 1 newdirblk 5 2K 25144 linux 5 2K 6 CAM periph 4 2K 271 filecaps 5 2K 122 crypto 4 2K 2277 diradd 4 2K 25270 pfil 4 2K 4 CAM path 4 2K 1034 tcpfunc 3 2K 3 loginclass 3 2K 6 chacha20random 1 1K 1 vnodes 1 1K 1 CAM SIM 2 1K 2 ktls 1 1K 1 cdev 2 1K 2 aesni_data 2 1K 2 cpus 2 1K 2 atkbddev 2 1K 2 CAM dev queue 2 1K 2 xform 2 1K 6320 entropy 2 1K 48 sigio 2 1K 23 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 pmchooks 1 1K 1 sctp_vrf 1 1K 1 apmdev 1 1K 1 CAM I/O Scheduler 1 1K 1 vnet_data_free 1 1K 1 vnet 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 acpiintr 1 1K 1 pmc 1 1K 1 sctp_mcore 0 0K 0 sctp_socko 0 0K 14066 sctp_iter 0 0K 36 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 27 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 9 sctp_aadr 0 0K 0 sctp_stri 0 0K 1460 cdg data 0 0K 160 chd data 0 0K 8 vegas data 0 0K 61 dctcp data 0 0K 15 mqdata 0 0K 0 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_temp 0 0K 0 htcp data 0 0K 22 cubic data 0 0K 18 tcp_do 0 0K 0 tcp_fsb 0 0K 1980 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 iavf 0 0K 0 ixl 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 axgbe 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xen_intr 0 0K 0 xenstore 0 0K 0 ciss_data 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 qpidrv 0 0K 0 BACKLIGHT 0 0K 0 xnb 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 sysmouse 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 vtfont 0 0K 0 vm_fictitious 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 amr 0 0K 0 pvscsi 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 UMAHash 0 0K 0 scsi_cd 0 0K 0 AHCI driver 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 19103 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 833 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 6 allocindir 0 0K 0 indirdep 0 0K 31 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 agp 0 0K 0 nvme_da 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 ktls_ocf 0 0K 0 twsbuf 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpifw 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 1341 ip6_msource 0 0K 0 ip6_moptions 0 0K 2 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 twe_commands 0 0K 0 LRO 0 0K 0 newreno data 0 0K 4 ip_msource 0 0K 1 ip_moptions 0 0K 46 in_mfilter 0 0K 7 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 0 VN POLL 0 0K 0 twa_commands 0 0K 0 statfs 0 0K 25433 namei_tracker 0 0K 4 export_host 0 0K 0 cl_savebuf 0 0K 4 tcp_log_dev 0 0K 943 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 11876 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 biobuf 0 0K 0 aios 0 0K 32 lio 0 0K 24 acl 0 0K 0 tempbuff 0 0K 0 mbuf_tag 0 0K 411 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 accf 0 0K 0 pts 0 0K 0 iov 0 0K 33476 ioctlops 0 0K 11145 eventfd 0 0K 0 Witness 0 0K 0 stack 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 sbuf 0 0K 288 md_sectors 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 md_disk 0 0K 0 SWAP 0 0K 0 malodev 0 0K 0 LED 0 0K 0 sysctltmp 0 0K 875 sysctl 0 0K 3 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 ix_sriov 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 cache 0 0K 0 iirbuf 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8328 965 1109280 0 254 38064128 0 malloc-384 384 35457 1093 90725 0 30 14035200 0 malloc-8192 8192 451 555 40589 0 1 8241152 0 malloc-4096 4096 1766 8 8224 0 2 7266304 0 malloc-1024 1024 4173 2083 23688 0 16 6406144 0 ertt_txseginfo 40 141464 1148 1746616 0 254 5704480 0 mbuf 256 8876 1354 3638014 0 254 2618880 0 pbuf 2624 0 973 0 0 2 2553152 0 tcp_log 416 0 5256 35635 0 254 2186496 0 malloc-1024 1024 70 2030 26281 0 16 2150400 0 RADIX NODE 144 14463 346 618300 0 63 2132496 0 mbuf_cluster 2048 975 1 975 0 254 1998848 0 BUF TRIE 144 187 13281 6914 0 62 1939392 0 UMA Slabs 0 112 16671 30 16671 0 126 1870512 0 malloc-32768 32768 45 4 25182 0 1 1605632 0 malloc-1024 1024 279 1105 53113 0 16 1417216 0 malloc-384 384 641 2779 125995 0 30 1313280 0 sctp_asoc 2288 11 554 5716 0 254 1292720 0 malloc-2048 2048 15 551 6608 0 8 1159168 0 malloc-512 512 64 2040 25261 0 30 1077248 0 malloc-384 384 710 1990 85021 0 30 1036800 0 sctp_raddr 736 11 1045 6375 0 254 777216 0 sctp_ep 1280 18 585 13897 0 254 771840 0 malloc-1024 1024 174 570 14429 0 16 761856 0 tcp_bbr_map 128 959 4466 308831 0 127 694400 0 malloc-384 384 1696 44 53644 0 30 668160 0 FFS inode 1160 537 23 25745 0 8 649600 0 VM OBJECT 264 1789 71 374667 0 30 491040 0 malloc-16384 16384 27 2 45 0 1 475136 0 malloc-65536 65536 2 5 22 0 1 458752 0 socket 944 77 387 36983 0 254 438016 0 256 Bucket 2048 150 52 24811 0 8 413696 0 tcpcb 1064 34 344 18270 0 254 402192 0 malloc-384 384 1023 17 15283 0 30 399360 0 THREAD 1808 201 19 31318 0 8 397760 0 lkpimm 160 1 2324 1 0 62 372000 0 lkpicurr 160 2 2323 2 0 62 372000 0 malloc-384 384 339 601 91073 0 30 360960 0 MAP ENTRY 96 3359 253 1417290 0 126 346752 0 malloc-384 384 620 250 447476 0 30 334080 0 tcp_inpcb 488 36 612 18270 0 254 316224 0 malloc-512 512 67 541 26881 0 30 311296 0 VNODE 448 575 82 25785 0 30 294336 0 malloc-65536 65536 4 0 4 0 1 262144 0 malloc-2048 2048 120 2 174 0 8 249856 0 mbuf_packet 256 59 916 103363 0 254 249600 0 DEVCTL 1024 0 216 132 0 0 221184 0 vmem btag 56 3649 83 3649 0 254 208992 0 VMSPACE 2544 71 10 26169 0 4 206064 0 tcp_rack_pcb 832 0 243 1980 0 16 202176 0 malloc-65536 65536 1 2 177 0 1 196608 0 malloc-8192 8192 18 6 182 0 1 196608 0 UMA Zones 768 242 2 242 0 16 187392 0 malloc-16384 16384 6 5 42 0 1 180224 0 malloc-16384 16384 8 3 18 0 1 180224 0 malloc-512 512 333 19 26740 0 30 180224 0 pcpu-8 8 19017 2487 56706 0 254 172032 0 malloc-32768 32768 3 2 19 0 1 163840 0 malloc-4096 4096 2 38 15979 0 2 163840 0 malloc-1024 1024 121 39 3559 0 16 163840 0 PROC 1328 103 17 26202 0 8 159360 0 FFS2 dinode 256 537 63 25744 0 62 153600 0 S VFS Cache 104 1046 397 27403 0 126 150072 0 malloc-4096 4096 30 6 558 0 2 147456 0 malloc-65536 65536 2 0 2 0 1 131072 0 filedesc0 1072 104 15 26204 0 8 127568 0 128 Bucket 1024 57 66 6796 0 16 125952 0 ksiginfo 112 104 940 3952 0 126 116928 0 32 Bucket 256 374 76 25505 0 62 115200 0 malloc-512 512 162 62 80429 0 30 114688 0 sctp_chunk 152 13 741 2196 0 254 114608 0 tcp_rack_map 112 0 936 3890 0 126 104832 0 malloc-512 512 166 26 1682 0 30 98304 0 UMA Kegs 384 227 6 227 0 30 89472 0 clpbuf 2624 0 32 20 0 16 83968 0 malloc-16384 16384 1 4 5 0 1 81920 0 malloc-8192 8192 9 1 11 0 1 81920 0 malloc-512 512 130 30 1380 0 30 81920 0 16 Bucket 144 433 99 5904 0 62 76608 0 malloc-1024 1024 62 6 909 0 16 69632 0 g_bio 408 0 170 44969 0 30 69360 0 malloc-65536 65536 1 0 2 0 1 65536 0 malloc-65536 65536 0 1 3 0 1 65536 0 malloc-32768 32768 1 1 3 0 1 65536 0 malloc-32768 32768 0 2 130 0 1 65536 0 malloc-32768 32768 2 0 2 0 1 65536 0 malloc-8192 8192 3 5 25436 0 1 65536 0 malloc-2048 2048 11 21 2001 0 8 65536 0 malloc-4096 4096 8 7 532 0 2 61440 0 malloc-2048 2048 16 14 523 0 8 61440 0 sctp_stream_msg_out 112 6 498 1083 0 254 56448 0 malloc-384 384 114 26 716 0 30 53760 0 64 Bucket 512 49 55 10585 0 30 53248 0 tcp_bbr_pcb 832 25 38 10779 0 16 52416 0 malloc-16384 16384 3 0 3 0 1 49152 0 malloc-16384 16384 1 2 237 0 1 49152 0 malloc-8192 8192 2 4 6488 0 1 49152 0 pipe 744 26 34 1168 0 16 44640 0 Files 80 296 254 91843 0 126 44000 0 DIRHASH 1024 34 6 34 0 16 40960 0 malloc-8192 8192 2 3 6 0 1 40960 0 malloc-512 512 68 12 1895 0 30 40960 0 malloc-512 512 12 68 1412 0 30 40960 0 NAMEI 1024 0 36 135267 0 16 36864 0 malloc-4096 4096 6 3 125 0 2 36864 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-16384 16384 2 0 2 0 1 32768 0 malloc-16384 16384 2 0 2 0 1 32768 0 malloc-4096 4096 3 5 450 0 2 32768 0 pcpu-64 64 487 25 487 0 254 32768 0 sctp_laddr 48 8 664 1038 0 254 32256 0 TURNSTILE 136 221 10 221 0 62 31416 0 ripcb 488 5 59 2253 0 254 31232 0 ertt 72 34 358 18270 0 126 28224 0 KNOTE 160 28 147 492676 0 62 28000 0 8 Bucket 80 112 238 5408 0 126 28000 0 malloc-4096 4096 6 0 6 0 2 24576 0 malloc-2048 2048 3 9 152 0 8 24576 0 malloc-2048 2048 8 4 56 0 8 24576 0 malloc-1024 1024 3 21 2702 0 16 24576 0 PWD 32 41 715 25239 0 254 24192 0 2 Bucket 32 379 377 13377 0 254 24192 0 ttyinq 160 135 15 300 0 62 24000 0 ttyoutq 256 72 18 160 0 62 23040 0 SLEEPQUEUE 88 221 35 221 0 126 22528 0 Mountpoints 2752 2 5 2 0 4 19264 0 unpcb 256 20 55 1652 0 254 19200 0 malloc-8192 8192 0 2 12 0 1 16384 0 vtnet_tx_hdr 24 2 666 2155061 0 254 16032 0 udp_inpcb 488 6 26 573 0 254 15616 0 malloc-1024 1024 9 3 14 0 16 12288 0 4 Bucket 48 7 245 543 0 254 12096 0 udplite_inpcb 488 0 24 285 0 254 11712 0 kenv 258 15 30 1048 0 30 11610 0 routing nhops 256 27 18 35 0 62 11520 0 malloc-4096 4096 2 0 2 0 2 8192 0 pf tags 104 39 39 219 0 126 8112 0 rtentry 176 31 15 35 0 62 8096 0 PGRP 88 40 52 241 0 126 8096 0 rl_entry 40 87 115 87 0 254 8080 0 udpcb 32 6 246 858 0 254 8064 0 ipq 56 0 144 2 0 254 8064 0 AIOP 32 4 248 4 0 254 8064 0 UMA Slabs 1 176 41 4 41 0 62 7920 0 sctp_readq 152 0 52 38 0 254 7904 0 AIO 208 0 38 86 0 62 7904 0 cryptop 280 0 28 238 0 30 7840 0 itimer 352 0 22 14 0 30 7744 0 AIOCB 552 0 14 149 0 16 7728 0 AIOLIO 272 0 28 12 0 30 7616 0 FPU_save_area 832 1 8 1 0 16 7488 0 cpuset 104 7 55 7 0 126 6448 0 epoch_record pcpu 256 4 12 4 0 62 4096 0 malloc-2048 2048 2 0 2 0 8 4096 0 pcpu-16 16 7 249 7 0 254 4096 0 tcp_log_bucket 176 0 23 3 0 62 4048 0 hostcache 64 1 62 1 0 254 4032 0 syncache 168 0 24 4 0 254 4032 0 tcp_log_node 120 0 33 5 0 126 3960 0 mqnode 416 3 6 3 0 30 3744 0 KMAP ENTRY 96 12 27 12 0 0 3744 0 vmem 1856 1 1 1 0 8 3712 0 SMR CPU 32 3 60 3 0 254 2016 0 SMR SHARED 24 3 60 3 0 254 1512 0 FFS1 dinode 128 0 0 0 0 126 0 0 da_ccb 544 0 0 0 0 16 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 0 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 62 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 1 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 304 0 0 0 0 254 0 0 pf mtags 48 0 0 0 0 254 0 0 tcpreass 48 0 0 0 0 254 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 88 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 NCLNODE 584 0 0 0 0 16 0 0 TMPFS node 224 0 0 0 0 62 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 linux_dma_object 24 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 ktls_session 128 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 domainset 40 0 0 0 0 254 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-2048 2048 0 0 0 0 8 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0