panic: seq_out not found rack:0xfffffe0099a00700 tp:0xfffffe0092734450 cpuid = 0 time = 1651394620 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0xc7/frame 0xfffffe0053b23510 kdb_backtrace() at kdb_backtrace+0xd3/frame 0xfffffe0053b23670 vpanic() at vpanic+0x2b8/frame 0xfffffe0053b23750 panic() at panic+0xb5/frame 0xfffffe0053b23810 rack_log_output() at rack_log_output+0x1296/frame 0xfffffe0053b239e0 rack_output() at rack_output+0x9915/frame 0xfffffe0053b23ff0 rack_do_segment_nounlock() at rack_do_segment_nounlock+0x57e2/frame 0xfffffe0053b24830 rack_do_segment() at rack_do_segment+0x1a8/frame 0xfffffe0053b24920 tcp_input_with_port() at tcp_input_with_port+0x1cb0/frame 0xfffffe0053b24bb0 tcp_input() at tcp_input+0x1f/frame 0xfffffe0053b24bd0 ip_input() at ip_input+0x80e/frame 0xfffffe0053b24cf0 swi_net() at swi_net+0x2f0/frame 0xfffffe0053b24d90 ithread_loop() at ithread_loop+0x4f1/frame 0xfffffe0053b24ef0 fork_exit() at fork_exit+0xd0/frame 0xfffffe0053b24f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0053b24f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- KDB: enter: panic [ thread pid 12 tid 100031 ] Stopped at kdb_enter+0x6b: movq $0,0x27044ea(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0x28 rax 0x12 rcx 0x472b1fc2e6669a0f rdx 0x1 rbx 0 rsp 0xfffffe0053b23650 rbp 0xfffffe0053b23670 rsi 0 rdi 0xffffffff817829ba vprintf+0x35a r8 0 r9 0xffffffff r10 0 r11 0 r12 0xfffffe0053ea8560 r13 0xfffffe0053b23601 r14 0xffffffff82bc9b00 .str.26 r15 0xffffffff82bc9b00 .str.26 rip 0xffffffff81775f2b kdb_enter+0x6b rflags 0x46 kdb_enter+0x6b: movq $0,0x27044ea(%rip) db> show proc Process 12 (intr) at 0xfffffe0053def548: state: NORMAL uid: 0 gids: 0 parent: pid 0 at 0xffffffff83e1da40 ABI: null flag: 0x10000284 flag2: 0 reaper: 0xffffffff83e1da40 reapsubtree: 12 sigparent: 20 vmspace: 0xffffffff83e1e9e0 (map 0xffffffff83e1e9e0) (map.pmap 0xffffffff83e1eaa0) (pmap 0xffffffff83e1eb08) threads: 22 100015 I [swi5: fast taskq] 100018 I [swi6: task queue] 100019 I [swi6: Giant taskq] 100031 Run CPU 0 [swi1: netisr 0] 100032 I [swi1: hpts] 100033 I [swi1: hpts] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq33: virtio_pci2] 100060 I [irq34: virtio_pci2] 100061 I [irq35: virtio_pci2] 100063 I [irq1: atkbd0] 100064 I [irq12: psm0] 100065 I [swi0: uart uart++] 100069 I [swi1: pf send] db> ps pid ppid pgrp uid state wmesg wchan cmd 9126 778 778 0 R (threaded) syz-executor.1 109057 Run CPU 1 syz-executor.1 109760 S connec 0xfffffe0058c3849a syz-executor.1 9124 777 777 0 R (threaded) syz-executor.0 109680 RunQ syz-executor.0 109759 S connec 0xfffffe0058c380da syz-executor.0 9123 835 835 0 R (threaded) syz-executor.2 109752 RunQ syz-executor.2 109757 S uwait 0xfffffe0057892500 syz-executor.2 9122 780 780 0 R syz-executor.3 8188 1 8188 65 Ss select 0xfffffe0099da7040 dhclient 7054 1 7054 0 Ss select 0xfffffe0058884d40 dhclient 7044 1 7044 0 Ss select 0xfffffe00588d5040 dhclient 6993 1 6993 65 Ss select 0xfffffe0099da6040 dhclient 6790 1 780 0 S uwait 0xfffffe0056fe4c00 syz-executor.3 6778 1 780 0 S uwait 0xfffffe0099b0a480 syz-executor.3 6769 1 780 0 S uwait 0xfffffe0057893d00 syz-executor.3 6757 1 780 0 S uwait 0xfffffe0057893880 syz-executor.3 6755 1 780 0 S uwait 0xfffffe0058bac600 syz-executor.3 6753 1 780 0 S uwait 0xfffffe0058baf700 syz-executor.3 6751 1 777 0 SV uwait 0xfffffe0099b0a380 syz-executor.0 6750 1 780 0 S uwait 0xfffffe0057892c00 syz-executor.3 6745 1 780 0 S uwait 0xfffffe0057895480 syz-executor.3 6739 1 777 0 SV uwait 0xfffffe0056fe4d00 syz-executor.0 6734 1 780 0 S uwait 0xfffffe0058bad500 syz-executor.3 6732 1 780 0 S uwait 0xfffffe0058bac200 syz-executor.3 6730 1 780 0 S uwait 0xfffffe0099b0a680 syz-executor.3 6728 1 780 0 S uwait 0xfffffe0057892a00 syz-executor.3 6726 1 777 0 SV uwait 0xfffffe0058bacc80 syz-executor.0 6717 1 777 0 SV uwait 0xfffffe0057893280 syz-executor.0 6707 1 777 0 SV uwait 0xfffffe0057892b00 syz-executor.0 4820 1 4820 0 Ss select 0xfffffe0099da6540 dhclient 4811 1 4811 0 Ss select 0xfffffe0099da65c0 dhclient 4763 1 4763 65 Ss select 0xfffffe0099da66c0 dhclient 3983 1 778 0 SV uwait 0xfffffe0057894280 syz-executor.1 3974 1 778 0 SV uwait 0xfffffe0057892600 syz-executor.1 3968 1 778 0 SV uwait 0xfffffe0058bac500 syz-executor.1 3949 1 778 0 SV uwait 0xfffffe0057892800 syz-executor.1 3940 1 778 0 SV uwait 0xfffffe0057894580 syz-executor.1 3935 1 778 0 SV uwait 0xfffffe0057895e80 syz-executor.1 3218 1 3218 0 Ss select 0xfffffe00588d5440 dhclient 3215 1 3215 0 Ss select 0xfffffe009235fe40 dhclient 3196 1 3196 65 Ss select 0xfffffe00588d5b40 dhclient 1973 0 0 0 DL - 0xffffffff83f677c0 [soaiod4] 1972 0 0 0 DL - 0xffffffff83f677c0 [soaiod3] 1971 0 0 0 DL - 0xffffffff83f677c0 [soaiod2] 1970 0 0 0 DL - 0xffffffff83f677c0 [soaiod1] 1658 1652 780 0 SV uwait 0xfffffe0057893f00 syz-executor.3 1652 1 780 0 DV ppwait 0xfffffe0094851f88 syz-executor.3 1640 1 780 0 SV uwait 0xfffffe0058bacb80 syz-executor.3 1628 1 780 0 SV uwait 0xfffffe0058bacd80 syz-executor.3 1513 1511 835 0 SV uwait 0xfffffe0058badb80 syz-executor.2 1511 1 835 0 DV ppwait 0xfffffe0058deaa40 syz-executor.2 1499 1 835 0 SV uwait 0xfffffe0058bace80 syz-executor.2 1494 1 835 0 SV uwait 0xfffffe0057895980 syz-executor.2 1468 1 835 0 S uwait 0xfffffe0057895a80 syz-executor.2 1463 1 835 0 S uwait 0xfffffe0057893380 syz-executor.2 1461 1 835 0 S uwait 0xfffffe0057894380 syz-executor.2 1455 1 835 0 S uwait 0xfffffe0057893480 syz-executor.2 1278 1 1278 0 Ss select 0xfffffe00588d56c0 dhclient 1275 1 1275 0 Ss select 0xfffffe00588d55c0 dhclient 879 0 0 0 DL aiordy 0xfffffe00927c9000 [aiod4] 878 0 0 0 DL aiordy 0xfffffe0056f9c548 [aiod3] 877 0 0 0 DL aiordy 0xfffffe0092be3548 [aiod2] 875 0 0 0 DL aiordy 0xfffffe0056f9ca90 [aiod1] 835 774 835 0 Rs syz-executor.2 780 774 780 0 Rs syz-executor.3 778 774 778 0 Ss nanslp 0xffffffff83e47a81 syz-executor.1 777 774 777 0 Ss nanslp 0xffffffff83e47a81 syz-executor.0 774 772 772 0 R (threaded) syz-fuzzer 100103 S uwait 0xfffffe0058baf400 syz-fuzzer 100113 RunQ syz-fuzzer 100114 S uwait 0xfffffe0058badd80 syz-fuzzer 100115 S kqread 0xfffffe0058af7300 syz-fuzzer 100116 S uwait 0xfffffe0058bad000 syz-fuzzer 100117 S uwait 0xfffffe0058bad100 syz-fuzzer 100118 S uwait 0xfffffe0058bad200 syz-fuzzer 100119 S uwait 0xfffffe0058bae000 syz-fuzzer 100121 RunQ syz-fuzzer 772 770 772 0 Ss pause 0xfffffe0058c665f8 csh 770 688 770 0 Ss select 0xfffffe00588d5ac0 sshd 754 1 754 0 Ss+ ttyin 0xfffffe00574774b0 getty 753 1 753 0 Ss+ ttyin 0xfffffe00586c34b0 getty 752 1 752 0 Ss+ ttyin 0xfffffe005748d8b0 getty 751 1 751 0 Ss+ ttyin 0xfffffe005748e0b0 getty 750 1 750 0 Ss+ ttyin 0xfffffe00586c3cb0 getty 749 1 749 0 Ss+ ttyin 0xfffffe005748e8b0 getty 748 1 748 0 Ss+ ttyin 0xfffffe0053f414b0 getty 747 1 747 0 Ss+ ttyin 0xfffffe0053f41cb0 getty 746 1 746 0 Ss+ ttyin 0xfffffe0053f424b0 getty 692 1 692 0 Ss nanslp 0xffffffff83e47a81 cron 688 1 688 0 Ss select 0xfffffe00588d5dc0 sshd 501 1 501 0 Ss select 0xfffffe00588d67c0 syslogd 430 1 430 0 Ss select 0xfffffe0092360bc0 devd 429 1 429 65 Ss select 0xfffffe00588d68c0 dhclient 344 1 344 0 Ss select 0xfffffe00588d69c0 dhclient 341 1 341 0 Ss select 0xfffffe00923605c0 dhclient 17 0 0 0 DL vlruwt 0xfffffe0056f9ea90 [vnlru] 16 0 0 0 DL syncer 0xffffffff83f6d2e0 [syncer] 15 0 0 0 DL (threaded) [bufdaemon] 100079 D psleep 0xffffffff83f6b8e0 [bufdaemon] 100082 D - 0xffffffff83211f80 [bufspacedaemon-0] 100095 D sdflush 0xfffffe0058c88ce8 [/ worker] 9 0 0 0 DL psleep 0xffffffff83f9f400 [vmdaemon] 8 0 0 0 DL (threaded) [pagedaemon] 100077 D psleep 0xffffffff83f932b8 [dom0] 100080 D launds 0xffffffff83f932c4 [laundry: dom0] 100081 D umarcl 0xffffffff81eb2060 [uma] 7 0 0 0 DL - 0xffffffff83c03788 [rand_harvestq] 6 0 0 0 DL pftm 0xffffffff8486a530 [pf purge] 5 0 0 0 DL waiting 0xffffffff84b9b4a0 [sctp_iterator] 4 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff83aa56c0 [doneq0] 100045 D - 0xffffffff83aa5640 [async] 100076 D - 0xffffffff83aa54c0 [scanner] 14 0 0 0 DL seqstat 0xfffffe0056f01c88 [sequencer 00] 3 0 0 0 DL (threaded) [crypto] 100040 D crypto_ 0xffffffff83f8eac0 [crypto] 100041 D crypto_ 0xfffffe0053f73030 [crypto returns 0] 100042 D crypto_ 0xfffffe0053f73080 [crypto returns 1] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff83e1d000 [g_event] 100036 D - 0xffffffff83e1d020 [g_up] 100037 D - 0xffffffff83e1d040 [g_down] 2 0 0 0 WL (threaded) [clock] 100029 I [clock (0)] 100030 I [clock (1)] 12 0 0 0 RL (threaded) [intr] 100015 I [swi5: fast taskq] 100018 I [swi6: task queue] 100019 I [swi6: Giant taskq] 100031 Run CPU 0 [swi1: netisr 0] 100032 I [swi1: hpts] 100033 I [swi1: hpts] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq33: virtio_pci2] 100060 I [irq34: virtio_pci2] 100061 I [irq35: virtio_pci2] 100063 I [irq1: atkbd0] 100064 I [irq12: psm0] 100065 I [swi0: uart uart++] 100069 I [swi1: pf send] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffffe0053df0000 [init] 10 0 0 0 DL audit_w 0xffffffff83f8f5c0 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff83e1da40 [swapper] 100005 D - 0xfffffe0008191000 [if_config_tqg_0] 100006 D - 0xfffffe0008190e00 [softirq_0] 100007 D - 0xfffffe0008190d00 [softirq_1] 100008 D - 0xfffffe0008190c00 [if_io_tqg_0] 100009 D - 0xfffffe0008190b00 [if_io_tqg_1] 100010 D - 0xfffffe0008190a00 [inm_free taskq] 100011 D - 0xfffffe0008190900 [linuxkpi_irq_wq] 100012 D - 0xfffffe0008190800 [in6m_free taskq] 100013 D - 0xfffffe0008190700 [deferred_unmount ta] 100014 D - 0xfffffe0008190600 [thread taskq] 100016 D - 0xfffffe0008190400 [kqueue_ctx taskq] 100017 D - 0xfffffe0008190300 [pci_hp taskq] 100020 D - 0xfffffe0008190000 [aiod_kick taskq] 100021 D - 0xfffffe000818fe00 [linuxkpi_short_wq_0] 100022 D - 0xfffffe000818fe00 [linuxkpi_short_wq_1] 100023 D - 0xfffffe000818fe00 [linuxkpi_short_wq_2] 100024 D - 0xfffffe000818fe00 [linuxkpi_short_wq_3] 100025 D - 0xfffffe000818fd00 [linuxkpi_long_wq_0] 100026 D - 0xfffffe000818fd00 [linuxkpi_long_wq_1] 100027 D - 0xfffffe000818fd00 [linuxkpi_long_wq_2] 100028 D - 0xfffffe000818fd00 [linuxkpi_long_wq_3] 100034 D - 0xfffffe000818fc00 [firmware taskq] 100038 D - 0xfffffe000818f200 [crypto_0] 100039 D - 0xfffffe000818f200 [crypto_1] 100055 D - 0xfffffe000818e900 [vtnet0 rxq 0] 100056 D - 0xfffffe000818e800 [vtnet0 txq 0] 100057 D - 0xfffffe000818e700 [vtnet0 rxq 1] 100058 D - 0xfffffe000818e600 [vtnet0 txq 1] 100062 D vtbslp 0xfffffe0056f4be00 [virtio_balloon] 100066 D - 0xffffffff82bcf981 [deadlkres] 100070 D - 0xfffffe0008191100 [mca taskq] 100072 D - 0xfffffe000818eb00 [acpi_task_0] 100073 D - 0xfffffe000818eb00 [acpi_task_1] 100074 D - 0xfffffe000818eb00 [acpi_task_2] 100075 D - 0xfffffe000818ea00 [CAM taskq] db> show all locks Process 9124 (syz-executor.0) thread 0xfffffe0058d403a0 (109680) exclusive sleep mutex umtxql (umtxql) r = 0 (0xffffffff83e66930) locked @ /syzkaller/managers/main/kernel/sys/kern/kern_umtx.c:1281 Process 12 (intr) thread 0xfffffe0053ea8560 (100031) exclusive rw tcpinp (tcpinp) r = 0 (0xfffffe00926eb508) locked @ /syzkaller/managers/main/kernel/sys/netinet/in_pcb.c:1551 db> show malloc Type InUse MemUse Requests pf_hash 5 11524K 5 tcp_hpts 7 4801K 7 devbuf 4217 4323K 4245 sysctloid 35322 2081K 35393 vtbuf 24 1968K 46 kobj 328 1312K 489 newblk 76 1043K 20306 vfscache 3 1025K 3 filedesc 88 698K 16311 linker 380 572K 453 pcb 47 564K 14066 inodedep 19 519K 8905 ufs_quota 1 512K 1 vfs_hash 1 512K 1 callout 2 512K 2 intr 4 472K 4 subproc 221 446K 9248 vmem 3 274K 6 acpica 1674 184K 57552 vnet_data 1 168K 1 tidhash 3 141K 3 pagedep 9 130K 8133 tfo_ccache 1 128K 1 DEVFS1 109 109K 126 sem 4 106K 4 BPF 52 91K 1774 bus 994 81K 5207 filemon 9 72K 26 mtx_pool 2 72K 2 syncache 1 68K 1 module 514 65K 514 acpitask 1 64K 1 ddb_capture 1 64K 1 umtx 462 58K 462 kdtrace 300 57K 18888 temp 34 33K 2577 DEVFS3 128 32K 138 hostcache 1 32K 1 shm 1 32K 10 msg 4 30K 4 gtaskqueue 18 26K 18 kbdmux 6 22K 6 ifaddr 71 21K 73 DEVFS_RULE 56 20K 56 ufs_mount 4 17K 5 proc 3 17K 3 tty 16 16K 16 routetbl 135 16K 427 ithread 97 16K 97 sctp_timw 59 15K 59 bus-sc 34 15K 1681 lltable 44 14K 64 ether_multi 157 13K 167 KTRACE 100 13K 100 ifnet 7 13K 7 kenv 95 12K 95 eventhandler 134 12K 134 freework 43 11K 13101 rman 88 11K 431 GEOM 61 11K 489 CAM queue 5 11K 1528 in6_multi 71 9K 71 kqueue 102 9K 9139 bmsafemap 2 9K 8813 UART 12 9K 12 devstat 4 9K 4 ksem 1 8K 465 rpc 2 8K 2 shmfd 1 8K 23 pfs_vncache 1 8K 1 pfs_nodes 20 8K 20 audit_evclass 237 8K 296 sctp_atcl 18 7K 7891 cred 27 7K 238 taskqueue 63 7K 63 pwddesc 100 7K 9127 sglist 5 7K 5 sctp_stro 6 6K 912 CAM DEV 3 6K 510 plimit 24 6K 421 DEVFSP 83 6K 1790 ufs_dirhash 24 5K 24 UMA 272 5K 272 session 35 5K 1695 pf_ifnet 10 5K 19 vt 11 5K 11 memdesc 1 4K 1 MCA 32 4K 32 evdev 4 4K 4 proc-args 127 4K 10441 acpisem 28 4K 28 selfd 55 4K 145497 lockf 32 4K 54 hhook 15 4K 17 kcovinfo 52 4K 52 terminal 11 3K 11 select 19 3K 73 uidinfo 3 3K 10 local_apic 1 2K 1 io_apic 1 2K 1 fpukern_ctx 2 2K 2 ipsec-saq 2 2K 2 ip6ndp 12 2K 15 sctp_ifa 14 2K 15 dirrem 7 2K 8510 Unitno 29 2K 911 CAM XPT 22 2K 543 msi 12 2K 12 freeblks 6 2K 8648 in_multi 6 2K 8 freefile 10 2K 8476 ipsecpolicy 2 2K 2 acpidev 20 2K 20 clone 9 2K 9 tun 7 2K 7 softdep 1 1K 1 mkdir 8 1K 16230 sahead 1 1K 1 secasvar 1 1K 1 nhops 6 1K 8 vnodemarker 2 1K 38 NFSD session 1 1K 1 sctp_atky 24 1K 8816 CAM periph 4 1K 271 sctp_ifn 6 1K 15 ipsec 3 1K 3 indirdep 3 1K 7041 mld 6 1K 6 igmp 6 1K 6 toponodes 6 1K 6 isadev 6 1K 6 mount 16 1K 89 pci_link 10 1K 10 crypto 4 1K 4 inpcbpolicy 20 1K 6248 encap_export_host 12 1K 12 procdesc 5 1K 18 newdirblk 4 1K 8115 diradd 4 1K 8549 CC Mem 8 1K 5752 pfil 4 1K 4 cdev 2 1K 2 osd 12 1K 5763 chacha20random 1 1K 1 sctp_athm 18 1K 7904 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 DEVFS 9 1K 10 vnodes 1 1K 1 CAM SIM 2 1K 2 feeder 7 1K 7 sctp_map 12 1K 1824 tcpfunc 3 1K 3 loginclass 3 1K 6 prison 6 1K 6 lkpikmalloc 5 1K 6 aesni_data 2 1K 2 soname 6 1K 22806 tcp_fsb 1 1K 2397 cryptodev 2 1K 204 nexusdev 8 1K 8 apmdev 1 1K 1 atkbddev 2 1K 2 CAM dev queue 2 1K 2 CAM I/O Scheduler 1 1K 1 CAM path 4 1K 1034 pmchooks 1 1K 1 filecaps 5 1K 112 sctp_vrf 1 1K 1 vnet 1 1K 1 entropy 2 1K 42 pmc 1 1K 1 acpiintr 1 1K 1 cpus 2 1K 2 vnet_data_free 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 ext2_mount 0 0K 0 ext2_node 0 0K 0 ext2_extents 0 0K 0 sctp_mcore 0 0K 0 sctp_socko 0 0K 2541 sctp_iter 0 0K 13 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 0 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 13 sctp_aadr 0 0K 0 sctp_stri 0 0K 26 pf_table 0 0K 0 pf_rule 0 0K 0 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_temp 0 0K 0 tcp_do 0 0K 0 ipcomp 0 0K 0 esp 0 0K 0 ah 0 0K 0 mqdata 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 ixl 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 NFSCL lckown 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 iavf 0 0K 0 axgbe 0 0K 0 NFSCL client 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 NFSD V4lock 0 0K 0 xen_intr 0 0K 0 NFSD V4state 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 bounce 0 0K 0 busdma 0 0K 0 qpidrv 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 DEVFS4 0 0K 0 vm_fictitious 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xenstore 0 0K 0 scsi_pass 0 0K 0 ciss_data 0 0K 0 xnb 0 0K 0 xen_acpi 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 UMAHash 0 0K 0 Balloon 0 0K 0 sysmouse 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 8556 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 16 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 2 allocindir 0 0K 0 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 vtfont 0 0K 0 BACKLIGHT 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 pvscsi 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 scsi_cd 0 0K 0 ktls_ocf 0 0K 0 AHCI driver 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS_RX 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 lkpiskb 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpindev 0 0K 0 lkpifw 0 0K 0 lkpi80211 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 27 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 LRO 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 0 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 0 VN POLL 0 0K 0 agp 0 0K 0 statfs 0 0K 8307 namei_tracker 0 0K 0 export_host 0 0K 0 cl_savebuf 0 0K 2 nvme_da 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 twsbuf 0 0K 0 twe_commands 0 0K 0 tcp_log_dev 0 0K 961 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 523 biobuf 0 0K 0 aios 0 0K 39 lio 0 0K 13 acl 0 0K 0 mbuf_tag 0 0K 124 ktls 0 0K 0 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 tempbuff 0 0K 0 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 accf 0 0K 0 pts 0 0K 0 iov 0 0K 19836 ioctlops 0 0K 369 eventfd 0 0K 9 Witness 0 0K 0 stack 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 sbuf 0 0K 288 mpr_user 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 MPRSAS 0 0K 0 SWAP 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 sysctltmp 0 0K 828 sysctl 0 0K 3 md_sectors 0 0K 0 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 md_disk 0 0K 0 malodev 0 0K 0 LED 0 0K 0 ix_sriov 0 0K 0 cache 0 0K 0 aacraidcam 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 1 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 aacraid_buf 0 0K 0 aaccam 0 0K 0 boottrace 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8320 1078 1570871 0 254 38494208 0 RADIX NODE 144 17965 398 202534 0 62 2644272 0 mbuf 256 8750 913 2505499 0 254 2473728 0 malloc-384 384 4135 1295 13022 0 30 2085120 0 pbuf 2624 0 794 0 0 2 2083456 0 BUF TRIE 144 189 11599 13623 0 62 1697472 0 mbuf_cluster 2048 762 0 762 0 254 1560576 0 malloc-128 128 11641 232 36485 0 126 1519744 0 malloc-4096 4096 330 4 501 0 2 1368064 0 UMA Slabs 0 112 11046 0 11046 0 126 1237152 0 sctp_asoc 2256 6 504 912 0 254 1150560 0 vmem btag 56 18386 85 18386 0 254 1034376 0 VM OBJECT 264 2927 163 203868 0 30 815760 0 malloc-16384 16384 44 4 8154 0 1 786432 0 socket 960 57 707 14560 0 254 733440 0 FFS inode 1160 547 34 9023 0 8 673960 0 sctp_ep 1208 12 498 6966 0 254 616080 0 tcpcb 1104 8 503 5752 0 254 564144 0 tcp_bbr_map 128 5 3901 56458 0 126 499968 0 MAP ENTRY 96 4408 380 508998 0 126 459648 0 malloc-4096 4096 105 5 9134 0 2 450560 0 256 Bucket 2048 200 18 6857 0 8 446464 0 THREAD 1808 199 32 9760 0 8 417648 0 malloc-65536 65536 6 0 6 0 1 393216 0 malloc-2048 2048 21 171 6975 0 8 393216 0 lkpimm 168 1 2327 1 0 62 391104 0 lkpicurr 168 2 2326 2 0 62 391104 0 sctp_raddr 736 6 511 912 0 254 380512 0 malloc-256 256 70 1370 8765 0 62 368640 0 malloc-256 256 47 1393 17287 0 62 368640 0 malloc-65536 65536 1 4 137 0 1 327680 0 VNODE 448 588 105 9066 0 30 310464 0 malloc-64 64 3998 349 158472 0 254 278208 0 malloc-16 16 14617 383 20432 0 254 240000 0 ertt_txseginfo 40 0 5858 74447 0 254 234320 0 malloc-32768 32768 0 7 6670 0 1 229376 0 DEVCTL 1024 0 220 155 0 0 225280 0 VMSPACE 2552 73 11 9084 0 4 214368 0 tcp_log 416 0 513 7382 0 254 213408 0 FPU_save_area 832 201 42 13921 0 16 202176 0 malloc-65536 65536 1 2 197 0 1 196608 0 mbuf_packet 256 25 737 9007 0 254 195072 0 malloc-128 128 45 1474 8666 0 126 194432 0 malloc-32 32 5564 358 5649 0 254 189504 0 UMA Zones 768 244 0 244 0 16 187392 0 malloc-128 128 1216 179 7254 0 126 178560 0 malloc-128 128 1242 153 27174 0 126 178560 0 malloc-256 256 274 416 20767 0 62 176640 0 malloc-1024 1024 137 23 154 0 16 163840 0 PROC 1352 99 21 9126 0 8 162240 0 FFS2 dinode 256 547 83 9023 0 62 161280 0 128 Bucket 1024 103 44 1549 0 16 150528 0 S VFS Cache 104 1025 379 11186 0 126 146016 0 filedesc0 1072 100 26 9127 0 8 135072 0 unpcb 256 20 490 1307 0 254 130560 0 ksiginfo 112 111 933 5290 0 126 116928 0 malloc-256 256 296 139 10266 0 62 111360 0 malloc-4096 4096 21 5 95 0 2 106496 0 malloc-384 384 49 221 7942 0 30 103680 0 64 Bucket 512 139 61 12404 0 30 102400 0 malloc-32768 32768 0 3 120 0 1 98304 0 malloc-8192 8192 9 3 38 0 1 98304 0 malloc-8192 8192 10 2 27 0 1 98304 0 malloc-256 256 121 254 29003 0 62 96000 0 UMA Kegs 384 230 3 230 0 30 89472 0 syncache 168 0 528 4 0 254 88704 0 malloc-128 128 374 277 11535 0 126 83328 0 malloc-16384 16384 0 5 160 0 1 81920 0 malloc-4096 4096 20 0 34 0 2 81920 0 malloc-2048 2048 5 35 1228 0 8 81920 0 malloc-384 384 173 37 174 0 30 80640 0 32 Bucket 256 135 180 17365 0 62 80640 0 malloc-512 512 18 134 1561 0 30 77824 0 malloc-8192 8192 8 1 10 0 1 73728 0 g_bio 408 0 180 96281 0 30 73440 0 malloc-64 64 578 493 2653 0 254 68544 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-65536 65536 0 1 8 0 1 65536 0 malloc-32768 32768 2 0 2 0 1 65536 0 malloc-32768 32768 2 0 2 0 1 65536 0 sctp_stream_msg_out 112 0 540 35 0 254 60480 0 udp_inpcb 424 6 120 295 0 30 53424 0 udplite_inpcb 424 0 126 104 0 30 53424 0 malloc-64 64 566 253 1111 0 254 52416 0 malloc-64 64 180 639 1892 0 254 52416 0 Files 80 251 399 36665 0 126 52000 0 malloc-256 256 7 188 9621 0 62 49920 0 DIRHASH 1024 35 13 35 0 16 49152 0 NAMEI 1024 0 48 58051 0 16 49152 0 malloc-16384 16384 3 0 3 0 1 49152 0 malloc-16384 16384 2 1 17 0 1 49152 0 malloc-1024 1024 9 39 928 0 16 49152 0 malloc-1024 1024 5 43 1986 0 16 49152 0 pcpu-64 64 496 272 496 0 254 49152 0 pcpu-8 8 4750 1394 15707 0 254 49152 0 tcp_rack_pcb 896 1 53 2397 0 16 48384 0 pipe 744 23 42 502 0 16 48360 0 tcp_bbr_pcb 832 2 52 2517 0 16 44928 0 TURNSTILE 136 232 83 232 0 62 42840 0 clpbuf 2624 0 16 28 0 16 41984 0 sctp_chunk 152 0 260 16 0 254 39520 0 Mountpoints 2752 2 12 2 0 4 38528 0 tcp_inpcb 424 9 81 5752 0 30 38160 0 ripcb 424 5 85 97 0 30 38160 0 hostcache 64 2 565 2 0 254 36288 0 PWD 32 54 1080 8200 0 254 36288 0 malloc-64 64 142 425 9229 0 254 36288 0 malloc-64 64 37 530 16594 0 254 36288 0 malloc-64 64 134 433 15460 0 254 36288 0 malloc-64 64 224 343 247 0 254 36288 0 16 Bucket 144 79 173 1282 0 62 36288 0 malloc-128 128 36 243 173 0 126 35712 0 malloc-128 128 29 250 1382 0 126 35712 0 malloc-128 128 47 232 47 0 126 35712 0 routing nhops 256 27 108 35 0 62 34560 0 ttyoutq 256 72 63 160 0 62 34560 0 malloc-384 384 54 36 56 0 30 34560 0 malloc-256 256 80 55 313 0 62 34560 0 malloc-256 256 31 104 1744 0 62 34560 0 SLEEPQUEUE 88 232 152 232 0 126 33792 0 malloc-16384 16384 2 0 2 0 1 32768 0 malloc-8192 8192 2 2 103 0 1 32768 0 malloc-2048 2048 3 13 49 0 8 32768 0 malloc-2048 2048 10 6 10 0 8 32768 0 malloc-2048 2048 3 13 91 0 8 32768 0 malloc-2048 2048 0 16 50 0 8 32768 0 malloc-1024 1024 8 24 9 0 16 32768 0 malloc-1024 1024 5 27 5 0 16 32768 0 malloc-1024 1024 3 29 160 0 16 32768 0 malloc-1024 1024 9 23 9 0 16 32768 0 malloc-1024 1024 3 29 20 0 16 32768 0 malloc-512 512 0 64 76 0 30 32768 0 malloc-512 512 0 64 132 0 30 32768 0 malloc-512 512 2 62 51 0 30 32768 0 malloc-512 512 5 59 41 0 30 32768 0 malloc-512 512 0 64 27 0 30 32768 0 malloc-512 512 8 56 8 0 30 32768 0 KNOTE 160 28 172 105563 0 62 32000 0 ttyinq 160 135 65 300 0 62 32000 0 tcp_rack_map 120 3 261 5256 0 126 31680 0 cpuset 104 7 272 1716 0 126 29016 0 sctp_laddr 48 24 564 3296 0 254 28224 0 tcp_inpcb ports 32 5 877 4805 0 254 28224 0 malloc-32 32 203 679 26279 0 254 28224 0 4 Bucket 48 7 581 236 0 254 28224 0 2 Bucket 32 73 809 2383 0 254 28224 0 AIO 208 7 126 27 0 62 27664 0 AIOCB 552 0 49 86 0 16 27048 0 da_ccb 544 0 49 24175 0 16 26656 0 malloc-8192 8192 3 0 3 0 1 24576 0 malloc-4096 4096 2 4 8932 0 2 24576 0 malloc-4096 4096 2 4 5 0 2 24576 0 rtentry 176 31 107 35 0 62 24288 0 PGRP 88 35 241 1754 0 126 24288 0 rl_entry 40 88 518 88 0 254 24240 0 8 Bucket 80 65 235 2498 0 126 24000 0 malloc-384 384 3 57 105 0 30 23040 0 malloc-384 384 20 40 360 0 30 23040 0 domainset 40 0 567 1277 0 254 22680 0 udpcb 32 6 624 399 0 254 20160 0 udp_inpcb ports 32 3 627 33 0 254 20160 0 AIOP 32 4 626 4 0 254 20160 0 ertt 72 8 272 5752 0 126 20160 0 malloc-32 32 111 519 993 0 254 20160 0 malloc-32 32 26 604 292 0 254 20160 0 malloc-32 32 59 571 1087 0 254 20160 0 malloc-32 32 56 574 357 0 254 20160 0 AIOLIO 272 0 70 13 0 30 19040 0 epoch_record pcpu 256 4 60 4 0 62 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-2048 2048 3 5 3 0 8 16384 0 malloc-2048 2048 3 5 3 0 8 16384 0 malloc-512 512 1 31 1 0 30 16384 0 SMR CPU 32 7 504 7 0 254 16352 0 vtnet_tx_hdr 24 0 668 796121 0 254 16032 0 malloc-16 16 107 893 21346 0 254 16000 0 malloc-16 16 285 715 17072 0 254 16000 0 kenv 258 15 45 1053 0 30 15480 0 mqnode 416 3 33 3 0 30 14976 0 vmem 1856 1 7 1 0 8 14848 0 SMR SHARED 24 7 504 7 0 254 12264 0 malloc-32 32 121 257 6945 0 254 12096 0 malloc-32 32 5 373 9 0 254 12096 0 malloc-16 16 14 736 1684 0 254 12000 0 malloc-16 16 40 710 167 0 254 12000 0 malloc-16 16 291 459 26510 0 254 12000 0 malloc-16 16 10 740 12 0 254 12000 0 malloc-384 384 1 29 1 0 30 11520 0 malloc-384 384 2 28 2 0 30 11520 0 malloc-8192 8192 1 0 1 0 1 8192 0 malloc-8192 8192 1 0 1 0 1 8192 0 malloc-4096 4096 0 2 1 0 2 8192 0 pcpu-16 16 14 498 14 0 254 8192 0 malloc-16 16 0 500 2 0 254 8000 0 UMA Slabs 1 176 12 10 12 0 62 3872 0 KMAP ENTRY 96 15 24 20 0 0 3744 0 pcpu-4 4 1 511 1 0 254 2048 0 FFS1 dinode 128 0 0 0 0 126 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 254 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 312 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 56 0 0 0 0 254 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 72 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 sctp_readq 152 0 0 0 0 254 0 0 tcp_log_id_node 120 0 0 0 0 126 0 0 tcp_log_id_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 ripcb ports 32 0 0 0 0 254 0 0 udplite_inpcb ports 32 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 itimer 352 0 0 0 0 30 0 0 TMPFS node 224 0 0 0 0 62 0 0 NCLNODE 608 0 0 0 0 16 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 cryptop 280 0 0 0 0 30 0 0 linux_dma_object 32 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-4096 4096 0 0 0 0 2 0 0 pcpu-32 32 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0