panic: seq_out not found rack:0xfffffe0099922e00 tp:0xfffffe00928f49b0 cpuid = 0 time = 1642527783 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0xc7/frame 0xfffffe0053b35670 kdb_backtrace() at kdb_backtrace+0xd3/frame 0xfffffe0053b357d0 vpanic() at vpanic+0x2b8/frame 0xfffffe0053b358b0 panic() at panic+0xb5/frame 0xfffffe0053b35970 rack_log_output() at rack_log_output+0x1392/frame 0xfffffe0053b35b40 rack_output() at rack_output+0x9b7e/frame 0xfffffe0053b36170 rack_do_segment_nounlock() at rack_do_segment_nounlock+0x5912/frame 0xfffffe0053b36830 rack_do_segment() at rack_do_segment+0x1a8/frame 0xfffffe0053b36920 tcp_input_with_port() at tcp_input_with_port+0x1c90/frame 0xfffffe0053b36bb0 tcp_input() at tcp_input+0x1f/frame 0xfffffe0053b36bd0 ip_input() at ip_input+0x80e/frame 0xfffffe0053b36cf0 swi_net() at swi_net+0x2e5/frame 0xfffffe0053b36d90 ithread_loop() at ithread_loop+0x4f1/frame 0xfffffe0053b36ef0 fork_exit() at fork_exit+0xd0/frame 0xfffffe0053b36f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0053b36f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- KDB: enter: panic [ thread pid 12 tid 100029 ] Stopped at kdb_enter+0x6b: movq $0,0x27105ba(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0x28 rax 0x12 rcx 0xd5724bc08a300a71 rdx 0x1 rbx 0 rsp 0xfffffe0053b357b0 rbp 0xfffffe0053b357d0 rsi 0 rdi 0xffffffff817627da vprintf+0x35a r8 0 r9 0xffffffff r10 0 r11 0x78f9ff9f r12 0xfffffe0053e933a0 r13 0xfffffe0053b35801 r14 0xffffffff82b5d160 .str.18 r15 0xffffffff82b5d160 .str.18 rip 0xffffffff81755d5b kdb_enter+0x6b rflags 0x46 kdb_enter+0x6b: movq $0,0x27105ba(%rip) db> show proc Process 12 (intr) at 0xfffffe0053de1548: state: NORMAL uid: 0 gids: 0 parent: pid 0 at 0xffffffff83e09b40 ABI: null flag: 0x10000284 flag2: 0 reaper: 0xffffffff83e09b40 reapsubtree: 12 sigparent: 20 vmspace: 0xffffffff83e0aae0 (map 0xffffffff83e0aae0) (map.pmap 0xffffffff83e0aba0) (pmap 0xffffffff83e0ac08) threads: 21 100015 I [swi5: fast taskq] 100018 I [swi6: task queue] 100019 I [swi6: Giant taskq] 100029 Run CPU 0 [swi1: netisr 0] 100032 Run CPU 1 [swi1: hpts] 100033 I [swi1: hpts] 100034 I [swi3: busdma] 100047 I [irq24: virtio_pci0] 100048 I [irq25: virtio_pci0] 100049 I [irq26: virtio_pci0] 100050 I [irq27: virtio_pci0] 100051 I [irq28: virtio_pci1] 100052 I [irq29: virtio_pci1] 100053 I [irq30: virtio_pci1] 100054 I [irq31: virtio_pci1] 100055 I [irq32: virtio_pci1] 100060 I [irq10: virtio_pci2] 100062 I [irq1: atkbd0] 100063 I [irq12: psm0] 100064 I [swi0: uart uart++] 100068 I [swi1: pf send] db> ps pid ppid pgrp uid state wmesg wchan cmd 59372 782 782 0 R (threaded) syz-executor.1 169369 RunQ syz-executor.1 169429 RunQ syz-executor.1 59371 790 790 0 R (threaded) syz-executor.2 169423 RunQ syz-executor.2 169428 S connec 0xfffffe009cc0849a syz-executor.2 59370 791 791 0 R (threaded) syz-executor.3 169407 RunQ syz-executor.3 169427 S sbwait 0xfffffe009a4962dc syz-executor.3 55500 1 55499 0 S uwait 0xfffffe005731d580 syz-executor.3 55498 1 55497 0 S uwait 0xfffffe0053dec180 syz-executor.3 55496 1 55495 0 S uwait 0xfffffe005731aa00 syz-executor.3 53167 1 791 0 SV uwait 0xfffffe0097c9c480 syz-executor.3 53158 1 791 0 SV uwait 0xfffffe0097c9d100 syz-executor.3 51242 1 782 0 S uwait 0xfffffe0097c9c180 syz-executor.1 51238 1 782 0 S uwait 0xfffffe005731ab00 syz-executor.1 51236 1 782 0 S uwait 0xfffffe0097c9bf00 syz-executor.1 51230 1 782 0 S uwait 0xfffffe0097c9ce80 syz-executor.1 51228 1 781 0 SV uwait 0xfffffe0097c9c280 syz-executor.0 51219 1 781 0 SV uwait 0xfffffe0057873200 syz-executor.0 51210 1 781 0 SV uwait 0xfffffe0097c9c380 syz-executor.0 47214 1 791 0 S uwait 0xfffffe005731db80 syz-executor.3 47208 1 791 0 S uwait 0xfffffe0057873d00 syz-executor.3 47203 1 791 0 S uwait 0xfffffe005731ad00 syz-executor.3 36487 1 782 0 S uwait 0xfffffe0097c9e180 syz-executor.1 36473 1 791 0 S uwait 0xfffffe0057873300 syz-executor.3 36471 1 791 0 S uwait 0xfffffe0057873c00 syz-executor.3 36467 1 791 0 S uwait 0xfffffe0057873800 syz-executor.3 36457 1 791 0 S uwait 0xfffffe0057873500 syz-executor.3 36450 1 791 0 S uwait 0xfffffe005731dd80 syz-executor.3 36439 1 791 0 S uwait 0xfffffe0053dec580 syz-executor.3 36432 1 791 0 S uwait 0xfffffe0053dec980 syz-executor.3 36420 1 791 0 S umtxn 0xfffffe0097c9d500 syz-executor.3 36410 1 791 0 S uwait 0xfffffe0053deca80 syz-executor.3 36027 1 782 0 S uwait 0xfffffe005731c680 syz-executor.1 36021 1 782 0 S uwait 0xfffffe0057873a00 syz-executor.1 36007 1 782 0 S uwait 0xfffffe005731ba80 syz-executor.1 35839 1 782 0 S uwait 0xfffffe0097c9db00 syz-executor.1 35838 1 782 0 S uwait 0xfffffe005731ac00 syz-executor.1 35831 1 782 0 S uwait 0xfffffe0097c9d900 syz-executor.1 35825 1 782 0 S uwait 0xfffffe0097c9dc00 syz-executor.1 35431 1 790 0 SV uwait 0xfffffe0057874680 syz-executor.2 35420 1 790 0 SV uwait 0xfffffe0097c9da00 syz-executor.2 35408 1 790 0 SV uwait 0xfffffe0057873000 syz-executor.2 12545 1 12545 65 Ss select 0xfffffe009a027bc0 dhclient 10171 1 10171 0 Ss select 0xfffffe0099ff8140 dhclient 10168 1 10168 0 Ss select 0xfffffe009a0384c0 dhclient 10149 1 10149 65 Ss select 0xfffffe0099ecdac0 dhclient 7814 1 7814 0 Ss select 0xfffffe009a06bec0 dhclient 7806 1 7806 0 Ss select 0xfffffe0097d2dec0 dhclient 7748 1 7748 65 Ss select 0xfffffe009a06c240 dhclient 3433 1 3433 0 Ss select 0xfffffe0097c92e40 dhclient 3424 1 3424 0 Ss select 0xfffffe0099ff9bc0 dhclient 3371 1 3371 65 Ss select 0xfffffe0097d6cac0 dhclient 1775 1 1775 0 Ss select 0xfffffe009988ebc0 dhclient 1772 1 1772 0 Ss select 0xfffffe009986d840 dhclient 834 0 0 0 DL aiordy 0xfffffe0097c31a90 [aiod4] 833 0 0 0 DL aiordy 0xfffffe0097c33000 [aiod3] 832 0 0 0 DL aiordy 0xfffffe0092d89a90 [aiod2] 831 0 0 0 DL aiordy 0xfffffe0092677548 [aiod1] 791 778 791 0 Rs syz-executor.3 790 778 790 0 Rs syz-executor.2 782 778 782 0 Rs syz-executor.1 781 778 781 0 Rs syz-executor.0 778 776 776 0 S (threaded) syz-fuzzer 100111 S uwait 0xfffffe0057874f00 syz-fuzzer 100113 S uwait 0xfffffe0057875180 syz-fuzzer 100114 S uwait 0xfffffe0057875280 syz-fuzzer 100115 S uwait 0xfffffe0057874780 syz-fuzzer 100116 S kqread 0xfffffe0090019100 syz-fuzzer 100117 S uwait 0xfffffe0057874880 syz-fuzzer 100118 S uwait 0xfffffe0057874980 syz-fuzzer 100120 S uwait 0xfffffe005731a900 syz-fuzzer 100591 S uwait 0xfffffe0057873b00 syz-fuzzer 126548 S uwait 0xfffffe0053dece80 syz-fuzzer 776 774 776 0 Ss pause 0xfffffe0092d8a0b0 csh 774 688 774 0 Ss select 0xfffffe0056f27040 sshd 754 1 754 0 Ss+ ttyin 0xfffffe0056f544b0 getty 753 1 753 0 Ss+ ttyin 0xfffffe0058726cb0 getty 752 1 752 0 Ss+ ttyin 0xfffffe00579bacb0 getty 751 1 751 0 Ss+ ttyin 0xfffffe00579bb0b0 getty 750 1 750 0 Ss+ ttyin 0xfffffe00579bb4b0 getty 749 1 749 0 Ss+ ttyin 0xfffffe00579bb8b0 getty 748 1 748 0 Ss+ ttyin 0xfffffe00579bbcb0 getty 747 1 747 0 Ss+ ttyin 0xfffffe00579bc0b0 getty 746 1 746 0 Ss+ ttyin 0xfffffe00579bc4b0 getty 692 1 692 0 Ss nanslp 0xffffffff83e339c0 cron 688 1 688 0 Ss select 0xfffffe0056f275c0 sshd 501 1 501 0 Ss select 0xfffffe0056f271c0 syslogd 430 1 430 0 Ss select 0xfffffe0056f27240 devd 429 1 429 65 Ss select 0xfffffe0056f272c0 dhclient 344 1 344 0 Ss select 0xfffffe0056f27440 dhclient 341 1 341 0 Ss select 0xfffffe0056f27740 dhclient 17 0 0 0 DL syncer 0xffffffff83f591c0 [syncer] 16 0 0 0 DL vlruwt 0xfffffe0056f7ea90 [vnlru] 15 0 0 0 DL (threaded) [bufdaemon] 100078 D psleep 0xffffffff83f577e0 [bufdaemon] 100079 D - 0xffffffff83211f80 [bufspacedaemon-0] 100092 D sdflush 0xfffffe0058ae40e8 [/ worker] 9 0 0 0 DL psleep 0xffffffff83f8acc0 [vmdaemon] 8 0 0 0 DL (threaded) [pagedaemon] 100076 D psleep 0xffffffff83f7eb78 [dom0] 100080 D launds 0xffffffff83f7eb84 [laundry: dom0] 100081 D umarcl 0xffffffff81e5ba90 [uma] 7 0 0 0 DL - 0xffffffff83bef908 [rand_harvestq] 6 0 0 0 DL pftm 0xffffffff846fe530 [pf purge] 5 0 0 0 DL waiting 0xffffffff849e34a0 [sctp_iterator] 4 0 0 0 DL (threaded) [cam] 100045 D - 0xffffffff83a918c0 [doneq0] 100046 D - 0xffffffff83a91840 [async] 100075 D - 0xffffffff83a916c0 [scanner] 14 0 0 0 DL seqstat 0xfffffe0053fc4488 [sequencer 00] 3 0 0 0 DL (threaded) [crypto] 100041 D crypto_ 0xffffffff83f7a3c0 [crypto] 100042 D crypto_ 0xfffffe0053f4a030 [crypto returns 0] 100043 D crypto_ 0xfffffe0053f4a080 [crypto returns 1] 13 0 0 0 DL (threaded) [geom] 100036 D - 0xffffffff83e090a0 [g_event] 100037 D - 0xffffffff83e090c0 [g_up] 100038 D - 0xffffffff83e090e0 [g_down] 2 0 0 0 RL (threaded) [clock] 100030 RunQ [clock (0)] 100031 I [clock (1)] 12 0 0 0 RL (threaded) [intr] 100015 I [swi5: fast taskq] 100018 I [swi6: task queue] 100019 I [swi6: Giant taskq] 100029 Run CPU 0 [swi1: netisr 0] 100032 Run CPU 1 [swi1: hpts] 100033 I [swi1: hpts] 100034 I [swi3: busdma] 100047 I [irq24: virtio_pci0] 100048 I [irq25: virtio_pci0] 100049 I [irq26: virtio_pci0] 100050 I [irq27: virtio_pci0] 100051 I [irq28: virtio_pci1] 100052 I [irq29: virtio_pci1] 100053 I [irq30: virtio_pci1] 100054 I [irq31: virtio_pci1] 100055 I [irq32: virtio_pci1] 100060 I [irq10: virtio_pci2] 100062 I [irq1: atkbd0] 100063 I [irq12: psm0] 100064 I [swi0: uart uart++] 100068 I [swi1: pf send] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffffe0053de2000 [init] 10 0 0 0 DL audit_w 0xffffffff83f7aea0 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff83e09b40 [swapper] 100005 D - 0xfffffe000818d000 [softirq_0] 100006 D - 0xfffffe000818ce00 [softirq_1] 100007 D - 0xfffffe000818cd00 [if_io_tqg_0] 100008 D - 0xfffffe000818cc00 [if_io_tqg_1] 100009 D - 0xfffffe000818cb00 [if_config_tqg_0] 100010 D - 0xfffffe000818ca00 [linuxkpi_irq_wq] 100011 D - 0xfffffe000818c900 [inm_free taskq] 100012 D - 0xfffffe000818c800 [in6m_free taskq] 100013 D - 0xfffffe000818c700 [deferred_unmount ta] 100014 D - 0xfffffe000818c600 [thread taskq] 100016 D - 0xfffffe000818c400 [kqueue_ctx taskq] 100017 D - 0xfffffe000818c300 [pci_hp taskq] 100020 D - 0xfffffe000818c000 [aiod_kick taskq] 100021 D - 0xfffffe000818be00 [linuxkpi_short_wq_0] 100022 D - 0xfffffe000818be00 [linuxkpi_short_wq_1] 100023 D - 0xfffffe000818be00 [linuxkpi_short_wq_2] 100024 D - 0xfffffe000818be00 [linuxkpi_short_wq_3] 100025 D - 0xfffffe000818bd00 [linuxkpi_long_wq_0] 100026 D - 0xfffffe000818bd00 [linuxkpi_long_wq_1] 100027 D - 0xfffffe000818bd00 [linuxkpi_long_wq_2] 100028 D - 0xfffffe000818bd00 [linuxkpi_long_wq_3] 100035 D - 0xfffffe000818bb00 [firmware taskq] 100039 D - 0xfffffe000818b100 [crypto_0] 100040 D - 0xfffffe000818b100 [crypto_1] 100056 D - 0xfffffe0056fcc800 [vtnet0 rxq 0] 100057 D - 0xfffffe0056fcc700 [vtnet0 txq 0] 100058 D - 0xfffffe0056fcc600 [vtnet0 rxq 1] 100059 D - 0xfffffe0056fcc500 [vtnet0 txq 1] 100061 D vtbslp 0xfffffe0056f28280 [virtio_balloon] 100065 D - 0xffffffff82b62c41 [deadlkres] 100069 D - 0xfffffe000818ab00 [acpi_task_0] 100070 D - 0xfffffe000818ab00 [acpi_task_1] 100071 D - 0xfffffe000818ab00 [acpi_task_2] 100072 D - 0xfffffe000818d100 [mca taskq] 100074 D - 0xfffffe000818a900 [CAM taskq] 59369 781 781 0 Z syz-executor.0 db> show all locks Process 59370 (syz-executor.3) thread 0xfffffe0097c991e0 (169427) exclusive sx so_snd_sx (so_snd_sx) r = 0 (0xfffffe009a496140) locked @ /syzkaller/managers/main/kernel/sys/kern/uipc_socket.c:4176 Process 12 (intr) thread 0xfffffe0053e933a0 (100029) exclusive rw tcpinp (tcpinp) r = 0 (0xfffffe009cc2b508) locked @ /syzkaller/managers/main/kernel/sys/netinet/in_pcb.c:1550 Process 12 (intr) thread 0xfffffe0053e91e40 (100032) exclusive sleep mutex tcp_hpts_lck (hpts) r = 0 (0xfffffe0053c91d00) locked @ /syzkaller/managers/main/kernel/sys/netinet/tcp_hpts.c:1431 db> show malloc Type InUse MemUse Requests pf_hash 5 11524K 5 tcp_hpts 6 4801K 6 devbuf 4213 4323K 4241 sysctloid 35454 2089K 35525 vtbuf 24 1968K 46 pcb 815 1428K 222487 kobj 322 1288K 480 newblk 9 1026K 58924 vfscache 3 1025K 3 filedesc 86 682K 116665 inodedep 251 606K 58589 ufs_quota 1 512K 1 vfs_hash 1 512K 1 callout 2 512K 2 intr 4 472K 4 subproc 211 428K 59488 vmem 3 274K 6 sctp_atcl 625 235K 146415 sctp_stro 229 229K 35505 acpica 1674 184K 56282 vnet_data 1 168K 1 tidhash 3 141K 3 linker 354 134K 806 pagedep 9 130K 58305 tfo_ccache 1 128K 1 DEVFS1 109 109K 126 sem 4 106K 4 BPF 46 88K 134 sctp_timw 318 80K 318 bus 973 80K 3426 mtx_pool 2 72K 2 syncache 1 68K 1 acpitask 1 64K 1 ddb_capture 1 64K 1 module 505 64K 505 dirrem 242 61K 58441 umtx 440 55K 440 kdtrace 288 54K 128803 sctp_atky 854 34K 186142 temp 34 33K 3301 DEVFS3 128 32K 138 hostcache 1 32K 1 shm 1 32K 302 freefile 242 31K 58403 msg 4 30K 4 gtaskqueue 18 26K 18 kbdmux 6 22K 6 ifaddr 71 21K 73 DEVFS_RULE 56 20K 56 ufs_mount 4 17K 5 proc 3 17K 3 tty 16 16K 16 routetbl 137 16K 429 ithread 93 15K 93 bus-sc 34 15K 1651 ksem 16 14K 9098 lltable 42 13K 153 ether_multi 157 13K 167 KTRACE 100 13K 100 ifnet 7 13K 7 kenv 95 12K 95 eventhandler 133 12K 133 CAM queue 5 11K 1528 GEOM 61 10K 488 rman 84 10K 425 sctp_athm 625 10K 146782 in6_multi 71 9K 71 bmsafemap 2 9K 58545 kqueue 98 9K 59479 UART 12 9K 12 devstat 4 9K 4 rpc 2 8K 2 shmfd 1 8K 321 pfs_vncache 1 8K 1 pfs_nodes 20 8K 20 audit_evclass 237 8K 296 cred 29 8K 319 sctp_map 458 8K 71010 taskqueue 63 7K 63 sglist 5 7K 5 CAM DEV 3 6K 510 plimit 24 6K 529 pwddesc 95 6K 59373 CC Mem 90 6K 12018 session 38 5K 195 ufs_dirhash 24 5K 24 UMA 272 5K 272 pf_ifnet 10 5K 19 DEVFSP 68 5K 626 vt 11 5K 11 memdesc 1 4K 1 MCA 32 4K 32 evdev 4 4K 4 proc-args 126 4K 60874 acpisem 28 4K 28 selfd 55 4K 825701 lockf 32 4K 84 hhook 15 4K 17 kcovinfo 52 4K 52 inpcbpolicy 102 4K 34867 terminal 11 3K 11 select 19 3K 82 uidinfo 3 3K 16 local_apic 1 2K 1 io_apic 1 2K 1 fpukern_ctx 2 2K 2 ipsec-saq 2 2K 2 ip6ndp 12 2K 15 sctp_ifa 14 2K 15 osd 94 2K 12029 Unitno 30 2K 47 CAM XPT 22 2K 543 in_multi 6 2K 8 ipsecpolicy 2 2K 2 acpidev 20 2K 20 msi 9 2K 9 clone 9 2K 9 tun 7 2K 7 freework 5 2K 58260 softdep 1 1K 1 mkdir 8 1K 116566 freeblks 4 1K 58259 sahead 1 1K 1 secasvar 1 1K 1 nhops 6 1K 8 vnodemarker 2 1K 142 NFSD session 1 1K 1 CAM periph 4 1K 271 tcp_fsb 6 1K 3494 sctp_ifn 6 1K 15 ipsec 3 1K 3 mld 6 1K 6 igmp 6 1K 6 toponodes 6 1K 6 isadev 6 1K 6 mount 16 1K 89 pci_link 10 1K 10 crypto 4 1K 22 encap_export_host 12 1K 12 procdesc 5 1K 18 newdirblk 4 1K 58283 diradd 4 1K 58483 pfil 4 1K 4 cdev 2 1K 2 chacha20random 1 1K 1 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 DEVFS 9 1K 10 vnodes 1 1K 1 CAM SIM 2 1K 2 feeder 7 1K 7 tcpfunc 3 1K 3 loginclass 3 1K 6 prison 6 1K 6 lkpikmalloc 5 1K 6 aesni_data 2 1K 2 nexusdev 8 1K 8 apmdev 1 1K 1 atkbddev 2 1K 2 xform 2 1K 506 CAM dev queue 2 1K 2 CAM I/O Scheduler 1 1K 1 CAM path 4 1K 1034 soname 5 1K 137446 pmchooks 1 1K 1 filecaps 5 1K 240 sctp_vrf 1 1K 1 vnet 1 1K 1 entropy 2 1K 49 acpiintr 1 1K 1 pmc 1 1K 1 cpus 2 1K 2 vnet_data_free 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 mqdata 0 0K 0 tcp_do 0 0K 0 sctp_mcore 0 0K 0 sctp_socko 0 0K 79131 sctp_iter 0 0K 68 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 116 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 12 sctp_aadr 0 0K 17 sctp_stri 0 0K 414 pf_table 0 0K 0 pf_rule 0 0K 0 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_temp 0 0K 0 filemon 0 0K 66 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 ixl 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 iavf 0 0K 0 axgbe 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 xen_intr 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 bounce 0 0K 0 busdma 0 0K 0 qpidrv 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 DEVFS4 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 DEVFS2 0 0K 0 xenbus 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 vm_fictitious 0 0K 0 xenstore 0 0K 0 ciss_data 0 0K 0 BACKLIGHT 0 0K 0 xnb 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 sysmouse 0 0K 0 vtfont 0 0K 0 UMAHash 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 48825 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 68 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 2 allocindir 0 0K 0 indirdep 0 0K 10 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 scsi_da 0 0K 69 pvscsi 0 0K 0 ata_da 0 0K 0 scsi_ch 0 0K 0 scsi_cd 0 0K 0 AHCI driver 0 0K 0 agp 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 ktls_ocf 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 lkpiskb 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpindev 0 0K 0 lkpifw 0 0K 0 lkpi80211 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 1154 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 nvme_da 0 0K 0 LRO 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 60 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 36 VN POLL 0 0K 3 statfs 0 0K 60050 namei_tracker 0 0K 0 export_host 0 0K 0 cl_savebuf 0 0K 3 acpipwr 0 0K 0 acpi_perf 0 0K 0 acpicmbat 0 0K 0 twsbuf 0 0K 0 twe_commands 0 0K 0 tcp_log_dev 0 0K 2357 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 523 PUC 0 0K 0 biobuf 0 0K 0 aios 0 0K 74 lio 0 0K 20 acl 0 0K 0 ppbusdev 0 0K 0 mbuf_tag 0 0K 166 ktls 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 tempbuff 0 0K 0 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 accf 0 0K 0 pts 0 0K 0 iov 0 0K 81112 ioctlops 0 0K 7247 eventfd 0 0K 157 Witness 0 0K 0 stack 0 0K 0 mps_user 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 sbuf 0 0K 288 mfibuf 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 md_sectors 0 0K 0 SWAP 0 0K 0 md_disk 0 0K 0 malodev 0 0K 0 sysctltmp 0 0K 995 sysctl 0 0K 3 LED 0 0K 0 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 ix_sriov 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 ix 0 0K 0 cache 0 0K 0 ipsbuf 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 6 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8379 2797 1851227 0 254 45776896 0 sctp_asoc 2288 229 3076 35505 0 254 7561840 0 malloc-2048 2048 401 2719 117802 0 8 6389760 0 sctp_ep 1208 396 3164 110703 0 254 4300480 0 tcp_log 416 0 10161 304133 0 254 4226976 0 RADIX NODE 144 26332 403 1198901 0 64 3849840 0 mbuf 256 9002 5479 3406889 0 254 3707136 0 mbuf_cluster 2048 1524 0 1524 0 254 3121152 0 malloc-1024 1024 231 2781 35466 0 16 3084288 0 sctp_raddr 736 229 3082 35550 0 254 2436896 0 malloc-384 384 626 5534 146538 0 30 2365440 0 pbuf 2624 0 794 0 0 2 2083456 0 UMA Slabs 0 112 15507 39 15522 0 126 1741152 0 BUF TRIE 144 192 11596 1886 0 62 1697472 0 malloc-384 384 4189 41 4529 0 30 1624320 0 ertt_txseginfo 40 36185 3508 297322 0 254 1587720 0 vmem btag 56 27636 51 27636 0 254 1550472 0 malloc-128 128 11677 320 70168 0 126 1535616 0 malloc-4096 4096 334 2 504 0 2 1376256 0 malloc-384 384 250 2300 58591 0 30 979200 0 socket 960 300 716 147437 0 254 975360 0 malloc-256 256 604 2816 310629 0 62 875520 0 256 Bucket 2048 379 39 8222 0 8 856064 0 VM OBJECT 264 3057 93 1269931 0 30 831600 0 malloc-16384 16384 44 6 58332 0 2 819200 0 tcp_bbr_map 128 4209 1340 97647 0 126 710272 0 malloc-256 256 314 2356 117528 0 62 683520 0 FFS inode 1160 552 29 58956 0 8 673960 0 malloc-256 256 17 2353 50393 0 62 606720 0 tcpcb 1096 90 421 12018 0 254 560056 0 MAP ENTRY 96 4534 380 2972658 0 126 471744 0 malloc-65536 65536 4 3 140 0 1 458752 0 malloc-4096 4096 96 4 59408 0 2 409600 0 THREAD 1808 191 29 69429 0 8 397760 0 lkpimm 168 1 2327 1 0 62 391104 0 lkpicurr 168 2 2326 2 0 62 391104 0 mbuf_packet 256 351 1173 97277 0 254 390144 0 malloc-128 128 662 2376 64994 0 126 388864 0 sctp_chunk 152 54 2234 4222 0 254 347776 0 sctp_stream_msg_out 112 310 2750 6309 0 254 342720 0 VNODE 448 593 100 58999 0 30 310464 0 malloc-64 64 4175 424 63540 0 254 294336 0 malloc-64 64 1031 3064 170230 0 254 262080 0 malloc-256 256 583 347 119078 0 62 238080 0 malloc-16 16 14671 79 14760 0 254 236000 0 DEVCTL 1024 0 220 155 0 0 225280 0 VMSPACE 2552 74 10 59345 0 4 214368 0 malloc-32 32 933 5745 215749 0 254 213696 0 FPU_save_area 832 193 50 77701 0 16 202176 0 malloc-16 16 1109 11391 217963 0 254 200000 0 malloc-32 32 5498 676 40708 0 254 197568 0 malloc-65536 65536 1 2 82 0 1 196608 0 UMA Zones 768 244 0 244 0 16 187392 0 malloc-2048 2048 6 82 575 0 8 180224 0 malloc-128 128 1212 183 30650 0 126 178560 0 128 Bucket 1024 98 65 14503 0 16 166912 0 malloc-1024 1024 129 31 4264 0 16 163840 0 FFS2 dinode 256 552 78 58955 0 62 161280 0 S VFS Cache 104 1041 480 61031 0 126 158184 0 PROC 1352 95 19 59372 0 8 154128 0 64 Bucket 512 195 69 29903 0 30 135168 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-65536 65536 0 2 500 0 1 131072 0 unpcb 256 20 490 1809 0 254 130560 0 32 Bucket 256 344 166 33354 0 62 130560 0 filedesc0 1072 95 17 59373 0 8 120064 0 ksiginfo 112 103 941 9017 0 126 116928 0 malloc-8192 8192 6 7 338 0 1 106496 0 malloc-4096 4096 22 4 33 0 2 106496 0 tcp_inpcb 424 90 153 12018 0 30 103032 0 malloc-128 128 556 219 8613 0 126 99200 0 malloc-8192 8192 7 5 137 0 1 98304 0 udp_inpcb 424 7 209 20626 0 30 91584 0 UMA Kegs 384 230 3 230 0 30 89472 0 pipe 744 23 92 1620 0 16 85560 0 Files 80 489 561 244728 0 126 84000 0 malloc-128 128 464 187 1474 0 126 83328 0 malloc-16384 16384 4 1 164 0 1 81920 0 sctp_readq 152 0 520 317 0 254 79040 0 g_bio 408 0 180 12145 0 30 73440 0 malloc-64 64 647 424 1949 0 254 68544 0 malloc-128 128 139 388 175181 0 126 67456 0 malloc-32768 32768 0 2 120 0 1 65536 0 malloc-32768 32768 2 0 2 0 1 65536 0 ripcb 424 5 148 1776 0 30 64872 0 udplite_inpcb 424 0 153 447 0 30 64872 0 malloc-384 384 115 35 187 0 30 57600 0 malloc-64 64 65 754 825789 0 254 52416 0 malloc-256 256 133 62 3432 0 62 49920 0 DIRHASH 1024 34 14 34 0 16 49152 0 NAMEI 1024 0 48 261204 0 16 49152 0 malloc-2048 2048 8 16 13 0 8 49152 0 malloc-1024 1024 30 18 34 0 16 49152 0 tcp_rack_pcb 896 6 48 3494 0 16 48384 0 16 Bucket 144 242 94 1159 0 62 48384 0 8 Bucket 80 395 205 13553 0 126 48000 0 tcp_bbr_pcb 832 6 48 4603 0 16 44928 0 syncache 168 0 264 4 0 254 44352 0 TURNSTILE 136 221 94 221 0 62 42840 0 clpbuf 2624 0 16 29 0 16 41984 0 malloc-8192 8192 0 5 66 0 1 40960 0 malloc-8192 8192 5 0 5 0 1 40960 0 pcpu-8 8 4738 382 5426 0 254 40960 0 Mountpoints 2752 2 12 2 0 4 38528 0 da_ccb 544 0 70 3151 0 16 38080 0 hostcache 64 2 565 2 0 254 36288 0 PWD 32 54 1080 58390 0 254 36288 0 malloc-64 64 86 481 20394 0 254 36288 0 malloc-64 64 139 428 156 0 254 36288 0 malloc-64 64 9 558 67 0 254 36288 0 malloc-128 128 144 135 803 0 126 35712 0 routing nhops 256 27 108 35 0 62 34560 0 ttyoutq 256 72 63 160 0 62 34560 0 malloc-256 256 15 120 17 0 62 34560 0 malloc-256 256 2 133 22 0 62 34560 0 malloc-256 256 14 121 971 0 62 34560 0 SLEEPQUEUE 88 221 163 221 0 126 33792 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-16384 16384 0 2 22 0 1 32768 0 malloc-16384 16384 1 1 16 0 1 32768 0 malloc-8192 8192 3 1 5 0 1 32768 0 malloc-4096 4096 1 7 60054 0 2 32768 0 malloc-4096 4096 4 4 381 0 2 32768 0 malloc-4096 4096 8 0 8 0 2 32768 0 malloc-2048 2048 3 13 140 0 8 32768 0 malloc-2048 2048 8 8 8 0 8 32768 0 malloc-2048 2048 2 14 26 0 8 32768 0 malloc-2048 2048 1 15 635 0 8 32768 0 malloc-1024 1024 0 32 82 0 16 32768 0 malloc-1024 1024 8 24 18 0 16 32768 0 malloc-1024 1024 6 26 6 0 16 32768 0 malloc-1024 1024 0 32 1398 0 16 32768 0 malloc-512 512 0 64 4 0 30 32768 0 malloc-512 512 3 61 143 0 30 32768 0 malloc-512 512 12 52 178 0 30 32768 0 malloc-512 512 1 63 53 0 30 32768 0 malloc-512 512 0 64 414 0 30 32768 0 malloc-512 512 10 54 101 0 30 32768 0 pcpu-64 64 484 28 484 0 254 32768 0 KNOTE 160 28 172 686365 0 62 32000 0 ttyinq 160 135 65 300 0 62 32000 0 tcp_rack_map 120 7 257 6905 0 126 31680 0 tcp_log_node 120 0 264 3 0 126 31680 0 cpuset 104 7 272 10 0 126 29016 0 udpcb 32 7 875 21073 0 254 28224 0 sctp_laddr 48 0 588 3524 0 254 28224 0 tcp_inpcb ports 32 6 876 7897 0 254 28224 0 ertt 72 90 302 12018 0 126 28224 0 4 Bucket 48 6 582 1241 0 254 28224 0 2 Bucket 32 63 819 5758 0 254 28224 0 malloc-8192 8192 3 0 3 0 1 24576 0 malloc-4096 4096 3 3 626 0 2 24576 0 rtentry 176 31 107 35 0 62 24288 0 PGRP 88 38 238 195 0 126 24288 0 rl_entry 40 80 526 80 0 254 24240 0 malloc-384 384 24 36 26 0 30 23040 0 malloc-384 384 32 28 165 0 30 23040 0 udplite_inpcb ports 32 0 630 6 0 254 20160 0 udp_inpcb ports 32 3 627 69 0 254 20160 0 AIOP 32 4 626 4 0 254 20160 0 malloc-64 64 3 312 4 0 254 20160 0 malloc-32 32 37 593 94857 0 254 20160 0 malloc-32 32 93 537 123 0 254 20160 0 malloc-32 32 137 493 1401 0 254 20160 0 malloc-32 32 144 486 189 0 254 20160 0 malloc-32 32 22 608 1567 0 254 20160 0 AIOCB 552 0 35 722 0 16 19320 0 AIOLIO 272 0 70 20 0 30 19040 0 epoch_record pcpu 256 4 60 4 0 62 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-4096 4096 1 3 4 0 2 16384 0 malloc-2048 2048 2 6 2 0 8 16384 0 malloc-1024 1024 2 14 2 0 16 16384 0 malloc-512 512 0 32 1 0 30 16384 0 SMR CPU 32 7 504 7 0 254 16352 0 vtnet_tx_hdr 24 0 668 967094 0 254 16032 0 malloc-16 16 76 924 155128 0 254 16000 0 malloc-16 16 508 492 1771 0 254 16000 0 AIO 208 0 76 156 0 62 15808 0 kenv 258 15 45 1045 0 30 15480 0 mqnode 416 3 33 3 0 30 14976 0 vmem 1856 1 7 1 0 8 14848 0 domainset 40 0 315 3 0 254 12600 0 SMR SHARED 24 7 504 7 0 254 12264 0 tcp_log_bucket 176 0 69 1 0 62 12144 0 malloc-32 32 6 372 6 0 254 12096 0 malloc-16 16 31 719 145 0 254 12000 0 malloc-16 16 122 628 97152 0 254 12000 0 malloc-16 16 2 748 3 0 254 12000 0 L VFS Cache 320 0 36 1 0 30 11520 0 malloc-384 384 1 29 1 0 30 11520 0 malloc-384 384 28 2 28 0 30 11520 0 malloc-8192 8192 1 0 1 0 1 8192 0 pcpu-16 16 14 498 14 0 254 8192 0 malloc-16 16 7 493 9 0 254 8000 0 UMA Slabs 1 176 9 13 9 0 62 3872 0 KMAP ENTRY 96 12 27 14 0 0 3744 0 FFS1 dinode 128 0 0 0 0 126 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 62 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 312 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 48 0 0 0 0 254 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 72 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 tcpreass 48 0 0 0 0 254 0 0 ripcb ports 32 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 itimer 352 0 0 0 0 30 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 TMPFS node 224 0 0 0 0 62 0 0 NCLNODE 608 0 0 0 0 16 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 cryptop 280 0 0 0 0 30 0 0 linux_dma_object 32 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-512 512 0 0 0 0 30 0 0 malloc-128 128 0 0 0 0 126 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0