panic: Don't own TCB send lock cpuid = 1 time = 1648223937 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0xc7/frame 0xfffffe0054150810 kdb_backtrace() at kdb_backtrace+0xd3/frame 0xfffffe0054150970 vpanic() at vpanic+0x2b8/frame 0xfffffe0054150a50 panic() at panic+0xb5/frame 0xfffffe0054150b10 sctp_ss_default_add() at sctp_ss_default_add+0x11d/frame 0xfffffe0054150b50 sctp_msg_append() at sctp_msg_append+0x646/frame 0xfffffe0054150bb0 sctp_sendall_iterator() at sctp_sendall_iterator+0x207/frame 0xfffffe0054150d70 sctp_iterator_worker() at sctp_iterator_worker+0xbb1/frame 0xfffffe0054150ed0 sctp_iterator_thread() at sctp_iterator_thread+0x5e/frame 0xfffffe0054150ef0 fork_exit() at fork_exit+0xd0/frame 0xfffffe0054150f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0054150f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- KDB: enter: panic [ thread pid 5 tid 100068 ] Stopped at kdb_enter+0x6b: movq $0,0x270a50a(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0x28 rax 0x12 rcx 0x5c11d128e078a4a rdx 0x1 rbx 0 rsp 0xfffffe0054150950 rbp 0xfffffe0054150970 rsi 0 rdi 0xffffffff8177a78a vprintf+0x35a r8 0 r9 0xffffffff r10 0 r11 0xfffffe0099913510 r12 0xfffffe005786a720 r13 0xfffffe0054150901 r14 0xffffffff82bb9ce0 .str.26 r15 0xffffffff82bb9ce0 .str.26 rip 0xffffffff8176dd0b kdb_enter+0x6b rflags 0x46 kdb_enter+0x6b: movq $0,0x270a50a(%rip) db> show proc Process 5 (sctp_iterator) at 0xfffffe0057994a90: state: NORMAL uid: 0 gids: 0 parent: pid 0 at 0xffffffff83e1b8c0 ABI: null flag: 0x10000204 flag2: 0 reaper: 0xffffffff83e1b8c0 reapsubtree: 5 sigparent: 20 vmspace: 0xffffffff83e1c860 (map 0xffffffff83e1c860) (map.pmap 0xffffffff83e1c920) (pmap 0xffffffff83e1c988) threads: 1 100068 Run CPU 1 [sctp_iterator] db> ps pid ppid pgrp uid state wmesg wchan cmd 49930 779 779 0 R (threaded) syz-executor.0 155780 RunQ syz-executor.0 155844 RunQ syz-executor.0 49929 790 790 0 R (threaded) syz-executor.2 155675 S nanslp 0xffffffff83e458c1 syz-executor.2 155842 RunQ syz-executor.2 49927 825 825 0 R (threaded) syz-executor.3 155703 Run CPU 0 syz-executor.3 155843 RunQ syz-executor.3 35536 1 790 0 S uwait 0xfffffe0057876f00 syz-executor.2 30077 1 779 0 SV uwait 0xfffffe0057876880 syz-executor.0 30072 1 779 0 SV sigwait 0xfffffe0099f0bb40 syz-executor.0 20081 1 825 0 SV uwait 0xfffffe0099c3d080 syz-executor.3 20070 20069 825 0 SV uwait 0xfffffe0057877800 syz-executor.3 20069 1 825 0 DV ppwait 0xfffffe00998d5a40 syz-executor.3 20061 1 825 0 SV uwait 0xfffffe008fe20e80 syz-executor.3 18683 1 790 0 S uwait 0xfffffe0057876580 syz-executor.2 18675 1 790 0 S uwait 0xfffffe0057878580 syz-executor.2 15210 1 15206 0 S uwait 0xfffffe0057445600 syz-executor.3 15209 1 15206 0 S uwait 0xfffffe0057878680 syz-executor.3 15203 1 15199 0 S uwait 0xfffffe0099c3d380 syz-executor.3 15194 1 15190 0 S uwait 0xfffffe0057878380 syz-executor.3 11198 1 11198 65 Ss select 0xfffffe0099c3c940 dhclient 9072 1 9072 0 Ss select 0xfffffe0057879640 dhclient 9068 1 9068 0 Ss select 0xfffffe0099c3c9c0 dhclient 9045 1 9045 65 Ss select 0xfffffe0099c3ca40 dhclient 6052 1 6052 0 Ss select 0xfffffe0057879940 dhclient 6049 1 6049 0 Ss select 0xfffffe0057448f40 dhclient 6022 1 6022 65 Ss select 0xfffffe0057879740 dhclient 3947 1 3947 0 Ss select 0xfffffe00578770c0 dhclient 3944 1 3944 0 Ss select 0xfffffe0057877040 dhclient 3925 1 3925 65 Ss select 0xfffffe00578796c0 dhclient 2400 0 0 0 DL aiordy 0xfffffe0092d7c548 [aiod4] 2399 0 0 0 DL aiordy 0xfffffe0092de8548 [aiod3] 2398 0 0 0 DL aiordy 0xfffffe0092d7c000 [aiod2] 2397 0 0 0 DL aiordy 0xfffffe0092d7ba90 [aiod1] 1025 1 1025 0 Ss select 0xfffffe00578797c0 dhclient 1022 1 1022 0 Ss select 0xfffffe0057879840 dhclient 825 774 825 0 Ss nanslp 0xffffffff83e458c1 syz-executor.3 790 774 790 0 Ss nanslp 0xffffffff83e458c1 syz-executor.2 780 774 780 0 Rs syz-executor.1 779 774 779 0 Ss nanslp 0xffffffff83e458c1 syz-executor.0 774 772 772 0 S (threaded) syz-fuzzer 100113 S uwait 0xfffffe008fe22980 syz-fuzzer 100118 S uwait 0xfffffe008fe23200 syz-fuzzer 100119 S kqread 0xfffffe000796fa00 syz-fuzzer 100120 S uwait 0xfffffe008fe22780 syz-fuzzer 100121 S uwait 0xfffffe008fe22e80 syz-fuzzer 100122 S uwait 0xfffffe008fe22200 syz-fuzzer 100123 S uwait 0xfffffe008fe21f00 syz-fuzzer 100125 S uwait 0xfffffe008fe21d80 syz-fuzzer 100126 S uwait 0xfffffe008fe23280 syz-fuzzer 100127 S uwait 0xfffffe008fe23180 syz-fuzzer 772 770 772 0 Ss pause 0xfffffe0058c32b40 csh 770 688 770 0 Ss select 0xfffffe008fe21d40 sshd 754 1 754 0 Ss+ ttyin 0xfffffe0057466cb0 getty 753 1 753 0 Ss+ ttyin 0xfffffe0057464cb0 getty 752 1 752 0 Ss+ ttyin 0xfffffe00574650b0 getty 751 1 751 0 Ss+ ttyin 0xfffffe0057a5ecb0 getty 750 1 750 0 Ss+ ttyin 0xfffffe00574654b0 getty 749 1 749 0 Ss+ ttyin 0xfffffe00574658b0 getty 748 1 748 0 Ss+ ttyin 0xfffffe0057465cb0 getty 747 1 747 0 Ss+ ttyin 0xfffffe0057a5f0b0 getty 746 1 746 0 Ss+ ttyin 0xfffffe0057a5f4b0 getty 692 1 692 0 Ss nanslp 0xffffffff83e458c1 cron 688 1 688 0 Ss select 0xfffffe008fe22140 sshd 501 1 501 0 Ss select 0xfffffe008fe22040 syslogd 430 1 430 0 Ss select 0xfffffe008fe22740 devd 429 1 429 65 Ss select 0xfffffe008fe220c0 dhclient 344 1 344 0 Ss select 0xfffffe008fe226c0 dhclient 341 1 341 0 Ss select 0xfffffe0057878740 dhclient 17 0 0 0 DL vlruwt 0xfffffe0056f7e548 [vnlru] 16 0 0 0 DL syncer 0xffffffff83f6b0e0 [syncer] 15 0 0 0 DL (threaded) [bufdaemon] 100080 D psleep 0xffffffff83f696e0 [bufdaemon] 100083 D - 0xffffffff83211f80 [bufspacedaemon-0] 100095 D sdflush 0xfffffe0057463ce8 [/ worker] 9 0 0 0 DL psleep 0xffffffff83f9d140 [vmdaemon] 8 0 0 0 DL (threaded) [pagedaemon] 100078 D psleep 0xffffffff83f90ff8 [dom0] 100081 D launds 0xffffffff83f91004 [laundry: dom0] 100082 D umarcl 0xffffffff81ea4800 [uma] 7 0 0 0 DL - 0xffffffff83c01688 [rand_harvestq] 6 0 0 0 DL pftm 0xffffffff847cd530 [pf purge] 5 0 0 0 RL CPU 1 [sctp_iterator] 4 0 0 0 DL (threaded) [cam] 100045 D - 0xffffffff83aa35c0 [doneq0] 100046 D - 0xffffffff83aa3540 [async] 100077 D - 0xffffffff83aa33c0 [scanner] 14 0 0 0 DL seqstat 0xfffffe0053fbb488 [sequencer 00] 3 0 0 0 DL (threaded) [crypto] 100041 D crypto_ 0xffffffff83f8c840 [crypto] 100042 D crypto_ 0xfffffe0053c8ad30 [crypto returns 0] 100043 D crypto_ 0xfffffe0053c8ad80 [crypto returns 1] 13 0 0 0 DL (threaded) [geom] 100036 D - 0xffffffff83e1ae80 [g_event] 100037 D - 0xffffffff83e1aea0 [g_up] 100038 D - 0xffffffff83e1aec0 [g_down] 2 0 0 0 WL (threaded) [clock] 100030 I [clock (0)] 100031 I [clock (1)] 12 0 0 0 WL (threaded) [intr] 100010 I [swi6: task queue] 100011 I [swi6: Giant taskq] 100018 I [swi5: fast taskq] 100029 I [swi1: netisr 0] 100032 I [swi3: busdma] 100033 I [swi1: hpts] 100034 I [swi1: hpts] 100047 I [irq24: virtio_pci0] 100048 I [irq25: virtio_pci0] 100049 I [irq26: virtio_pci0] 100050 I [irq27: virtio_pci0] 100051 I [irq28: virtio_pci1] 100052 I [irq29: virtio_pci1] 100053 I [irq30: virtio_pci1] 100054 I [irq31: virtio_pci1] 100055 I [irq32: virtio_pci1] 100060 I [irq33: virtio_pci2] 100061 I [irq34: virtio_pci2] 100062 I [irq35: virtio_pci2] 100064 I [irq1: atkbd0] 100065 I [irq12: psm0] 100066 I [swi0: uart uart++] 100070 I [swi1: pf send] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffffe0053dda000 [init] 10 0 0 0 DL audit_w 0xffffffff83f8d340 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff83e1b8c0 [swapper] 100005 D - 0xfffffe0053e01000 [if_config_tqg_0] 100006 D - 0xfffffe0053e00e00 [softirq_0] 100007 D - 0xfffffe0053e00d00 [softirq_1] 100008 D - 0xfffffe0053e00c00 [if_io_tqg_0] 100009 D - 0xfffffe0053e00b00 [if_io_tqg_1] 100012 D - 0xfffffe000796d200 [aiod_kick taskq] 100013 D - 0xfffffe000796d000 [inm_free taskq] 100014 D - 0xfffffe000796cd00 [linuxkpi_irq_wq] 100015 D - 0xfffffe000796cb00 [in6m_free taskq] 100016 D - 0xfffffe000796c900 [deferred_unmount ta] 100017 D - 0xfffffe000796c700 [thread taskq] 100019 D - 0xfffffe000796c300 [kqueue_ctx taskq] 100020 D - 0xfffffe000796c100 [pci_hp taskq] 100021 D - 0xfffffe000796be00 [linuxkpi_short_wq_0] 100022 D - 0xfffffe000796be00 [linuxkpi_short_wq_1] 100023 D - 0xfffffe000796be00 [linuxkpi_short_wq_2] 100024 D - 0xfffffe000796be00 [linuxkpi_short_wq_3] 100025 D - 0xfffffe000796b900 [linuxkpi_long_wq_0] 100026 D - 0xfffffe000796b900 [linuxkpi_long_wq_1] 100027 D - 0xfffffe000796b900 [linuxkpi_long_wq_2] 100028 D - 0xfffffe000796b900 [linuxkpi_long_wq_3] 100035 D - 0xfffffe0053ed0200 [firmware taskq] 100039 D - 0xfffffe0053ecfc00 [crypto_0] 100040 D - 0xfffffe0053ecfc00 [crypto_1] 100056 D - 0xfffffe0053eced00 [vtnet0 rxq 0] 100057 D - 0xfffffe0053ecec00 [vtnet0 txq 0] 100058 D - 0xfffffe0053eceb00 [vtnet0 rxq 1] 100059 D - 0xfffffe0053ecea00 [vtnet0 txq 1] 100063 D vtbslp 0xfffffe0053fdc380 [virtio_balloon] 100067 D - 0xffffffff82bbfb60 [deadlkres] 100071 D - 0xfffffe005787d100 [acpi_task_0] 100072 D - 0xfffffe005787d100 [acpi_task_1] 100073 D - 0xfffffe005787d100 [acpi_task_2] 100074 D - 0xfffffe000796e200 [mca taskq] 100076 D - 0xfffffe0053ecf500 [CAM taskq] 49928 780 780 0 Z syz-executor.1 db> show all locks Process 5 (sctp_iterator) thread 0xfffffe005786a720 (100068) exclusive sleep mutex sctp-tcb (tcb) r = 0 (0xfffffe00996e11a0) locked @ /syzkaller/managers/main/kernel/sys/netinet/sctputil.c:1523 exclusive sleep mutex sctp-inp (inp) r = 0 (0xfffffe00996c8248) locked @ /syzkaller/managers/main/kernel/sys/netinet/sctputil.c:1463 exclusive sleep mutex sctp-it (iterator) r = 0 (0xffffffff84af6560) locked @ /syzkaller/managers/main/kernel/sys/netinet/sctputil.c:1460 shared rw sctp-info (sctp-info) r = 0 (0xfffffe0053c43d48) locked @ /syzkaller/managers/main/kernel/sys/netinet/sctputil.c:1459 db> show malloc Type InUse MemUse Requests pf_hash 5 11524K 5 tcp_hpts 6 4801K 6 devbuf 4217 4323K 4245 sysctloid 35306 2080K 35377 pcb 1390 2074K 171768 vtbuf 24 1968K 46 kobj 327 1308K 488 inodedep 1970 1251K 49830 newblk 10 1027K 50918 vfscache 3 1025K 3 sctp_stro 682 682K 19698 ufs_quota 1 512K 1 vfs_hash 1 512K 1 callout 2 512K 2 sctp_atcl 1365 512K 105065 dirrem 1960 490K 49730 intr 4 472K 4 subproc 165 331K 50022 filedesc 42 329K 97787 vmem 3 274K 6 freefile 1960 245K 49710 acpica 1674 184K 56014 vnet_data 1 168K 1 tidhash 3 141K 3 linker 358 134K 410 pagedep 9 130K 48909 tfo_ccache 1 128K 1 DEVFS1 109 109K 126 sem 4 106K 4 BPF 46 88K 65 sctp_atky 2047 86K 126712 bus 991 81K 5140 mtx_pool 2 72K 2 syncache 1 68K 1 module 512 64K 512 acpitask 1 64K 1 ddb_capture 1 64K 1 sctp_timw 224 56K 224 umtx 418 53K 418 kdtrace 244 48K 105776 shm 3 36K 11 temp 34 33K 3099 DEVFS3 128 32K 138 hostcache 1 32K 1 msg 4 30K 4 gtaskqueue 18 26K 18 kbdmux 6 22K 6 sctp_athm 1365 22K 105689 sctp_map 1364 22K 39396 ifaddr 71 21K 73 DEVFS_RULE 56 20K 56 ufs_mount 4 17K 5 proc 3 17K 3 tty 16 16K 16 ithread 100 16K 100 routetbl 135 16K 427 bus-sc 34 15K 1651 lltable 45 14K 154 KTRACE 101 13K 574 ether_multi 157 13K 167 ifnet 7 13K 7 kenv 95 12K 95 eventhandler 134 12K 134 rman 88 11K 429 GEOM 61 11K 490 CAM queue 5 11K 1528 in6_multi 71 9K 71 bmsafemap 2 9K 49844 UART 12 9K 12 devstat 4 9K 4 ksem 1 8K 172 rpc 2 8K 2 shmfd 1 8K 342 pfs_vncache 1 8K 1 pfs_nodes 20 8K 20 audit_evclass 237 8K 296 cred 28 7K 337 kqueue 76 7K 50049 taskqueue 63 7K 63 sglist 5 7K 5 CAM DEV 3 6K 510 plimit 24 6K 529 session 38 5K 98 ufs_dirhash 24 5K 24 pwddesc 73 5K 49931 UMA 272 5K 272 pf_ifnet 10 5K 19 DEVFSP 68 5K 718 vt 11 5K 11 memdesc 1 4K 1 MCA 32 4K 32 evdev 4 4K 4 proc-args 104 4K 51415 acpisem 28 4K 28 selfd 55 4K 670238 lockf 32 4K 69 hhook 15 4K 17 kcovinfo 52 4K 52 terminal 11 3K 11 select 19 3K 178 uidinfo 3 3K 19 local_apic 1 2K 1 io_apic 1 2K 1 fpukern_ctx 2 2K 2 ipsec-saq 2 2K 2 ip6ndp 12 2K 16 sctp_ifa 14 2K 15 Unitno 29 2K 827 CAM XPT 22 2K 543 msi 12 2K 12 in_multi 6 2K 8 ipsecpolicy 2 2K 2 acpidev 20 2K 20 clone 9 2K 9 tun 7 2K 7 freework 5 2K 49417 softdep 1 1K 1 mkdir 8 1K 97772 freeblks 4 1K 49318 sahead 1 1K 1 secasvar 1 1K 1 nhops 6 1K 8 vnodemarker 2 1K 136 NFSD session 1 1K 1 CAM periph 4 1K 271 ipsec 3 1K 3 sctp_ifn 6 1K 15 mld 6 1K 6 igmp 6 1K 6 toponodes 6 1K 6 isadev 6 1K 6 mount 16 1K 125 pci_link 10 1K 10 crypto 4 1K 80 inpcbpolicy 20 1K 14272 encap_export_host 12 1K 12 procdesc 5 1K 18 newdirblk 4 1K 48886 diradd 4 1K 49772 CC Mem 8 1K 11830 pfil 4 1K 4 cdev 2 1K 2 osd 12 1K 11790 chacha20random 1 1K 1 sctp_iter 1 1K 25 sctp_cpal 1 1K 12 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 DEVFS 9 1K 10 vnodes 1 1K 1 CAM SIM 2 1K 2 feeder 7 1K 7 tcpfunc 3 1K 3 loginclass 3 1K 6 prison 6 1K 6 lkpikmalloc 5 1K 6 aesni_data 2 1K 2 cryptodev 2 1K 833 nexusdev 8 1K 8 apmdev 1 1K 1 atkbddev 2 1K 2 CAM dev queue 2 1K 2 CAM I/O Scheduler 1 1K 1 CAM path 4 1K 1034 pmchooks 1 1K 1 soname 4 1K 76674 filecaps 5 1K 131 sctp_vrf 1 1K 1 vnet 1 1K 1 entropy 2 1K 49 pmc 1 1K 1 acpiintr 1 1K 1 cpus 2 1K 2 vnet_data_free 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 ipcomp 0 0K 0 esp 0 0K 0 ah 0 0K 0 sctp_mcore 0 0K 0 sctp_socko 0 0K 52029 sctp_mvrf 0 0K 0 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 13 sctp_aadr 0 0K 0 sctp_stri 0 0K 0 pf_table 0 0K 0 pf_rule 0 0K 0 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_temp 0 0K 0 tcp_do 0 0K 0 tcp_fsb 0 0K 3257 mqdata 0 0K 0 filemon 0 0K 39 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 ixl 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 iavf 0 0K 0 axgbe 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 xen_intr 0 0K 0 NFSD srvcache 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 bounce 0 0K 0 busdma 0 0K 0 qpidrv 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 msdosfs_node 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 DEVFS4 0 0K 0 xenbus 0 0K 0 DEVFS2 0 0K 0 vm_fictitious 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xenstore 0 0K 0 scsi_pass 0 0K 0 ciss_data 0 0K 0 xnb 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 sysmouse 0 0K 0 UMAHash 0 0K 0 vtfont 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 36305 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 65 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 233 allocindir 0 0K 0 indirdep 0 0K 124 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 BACKLIGHT 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 pvscsi 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 scsi_cd 0 0K 0 ktls_ocf 0 0K 0 AHCI driver 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS_RX 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 lkpiskb 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpindev 0 0K 0 lkpifw 0 0K 0 lkpi80211 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 238 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 2 tcplog 0 0K 0 tcp_hwpace 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 LRO 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 1 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 5 VN POLL 0 0K 0 agp 0 0K 0 statfs 0 0K 49451 namei_tracker 0 0K 0 export_host 0 0K 0 cl_savebuf 0 0K 3 nvme_da 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 twsbuf 0 0K 0 twe_commands 0 0K 0 tcp_log_dev 0 0K 1943 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 523 biobuf 0 0K 0 aios 0 0K 9 lio 0 0K 25 acl 0 0K 0 mbuf_tag 0 0K 193 ktls 0 0K 0 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 tempbuff 0 0K 0 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 accf 0 0K 0 pts 0 0K 0 iov 0 0K 27598 ioctlops 0 0K 945 eventfd 0 0K 317 Witness 0 0K 0 stack 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 sbuf 0 0K 290 mpr_user 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 MPRSAS 0 0K 0 SWAP 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 sysctltmp 0 0K 1047 sysctl 0 0K 3 md_sectors 0 0K 0 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 md_disk 0 0K 0 malodev 0 0K 0 LED 0 0K 0 ix_sriov 0 0K 0 cache 0 0K 0 aacraidcam 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 1 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 aacraid_buf 0 0K 0 aaccam 0 0K 0 boottrace 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8328 1070 1756311 0 254 38494208 0 tcp_log 416 0 7371 110443 0 254 3066336 0 mbuf 256 9817 1361 3358868 0 254 2861568 0 RADIX NODE 144 19405 442 968234 0 63 2857968 0 sctp_asoc 2288 682 338 19698 0 254 2333760 0 pbuf 2624 0 778 0 0 2 2041472 0 malloc-2048 2048 686 170 86084 0 8 1753088 0 BUF TRIE 144 189 11599 2041 0 62 1697472 0 malloc-384 384 4117 53 4458 0 30 1601280 0 mbuf_cluster 2048 762 0 762 0 254 1560576 0 malloc-128 128 11689 308 12210 0 126 1535616 0 sctp_ep 1208 683 587 85367 0 254 1534160 0 malloc-4096 4096 327 3 650 0 2 1351680 0 malloc-256 256 2172 2913 142032 0 62 1301760 0 UMA Slabs 0 112 11484 30 11484 0 126 1289568 0 vmem btag 56 20413 146 20413 0 254 1151304 0 malloc-384 384 1984 656 49866 0 30 1013760 0 sctp_raddr 736 682 341 19698 0 254 752928 0 malloc-1024 1024 683 37 19711 0 16 737280 0 socket 960 57 707 101327 0 254 733440 0 VM OBJECT 264 2443 137 1066870 0 30 681120 0 FFS inode 1160 531 29 50243 0 8 649600 0 malloc-384 384 1365 195 105133 0 30 599040 0 tcpcb 1104 8 503 11779 0 254 564144 0 256 Bucket 2048 247 21 9067 0 8 548864 0 malloc-16384 16384 25 5 49071 0 1 491520 0 malloc-65536 65536 4 3 612 0 1 458752 0 lkpimm 168 1 2327 1 0 62 391104 0 lkpicurr 168 2 2326 2 0 62 391104 0 THREAD 1808 169 40 55844 0 8 377872 0 MAP ENTRY 96 3340 440 2509612 0 126 362880 0 malloc-128 128 2058 732 246296 0 126 357120 0 malloc-8192 8192 6 37 175 0 1 352256 0 malloc-4096 4096 74 6 49933 0 2 327680 0 VNODE 448 572 94 50286 0 30 298368 0 ertt_txseginfo 40 0 6666 369384 0 254 266640 0 malloc-64 64 3871 224 4538 0 254 262080 0 tcp_bbr_map 128 0 1891 77935 0 126 242048 0 malloc-16 16 14613 387 14801 0 254 240000 0 malloc-256 256 311 619 51598 0 62 238080 0 malloc-256 256 691 239 85379 0 62 238080 0 DEVCTL 1024 0 220 155 0 0 225280 0 malloc-65536 65536 1 2 41 0 1 196608 0 mbuf_packet 256 123 639 79533 0 254 195072 0 sctp_chunk 152 955 319 1771 0 254 193648 0 malloc-32 32 5398 524 6101 0 254 189504 0 UMA Zones 768 244 0 244 0 16 187392 0 FPU_save_area 832 171 54 63159 0 16 187200 0 malloc-128 128 1228 167 26332 0 126 178560 0 FFS2 dinode 256 531 99 50242 0 62 161280 0 S VFS Cache 104 1035 486 51545 0 126 158184 0 VMSPACE 2552 50 10 49905 0 4 153120 0 128 Bucket 1024 92 55 11399 0 16 150528 0 malloc-1024 1024 125 19 1681 0 16 147456 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-32768 32768 0 4 120 0 1 131072 0 malloc-128 128 933 90 2497 0 126 130944 0 unpcb 256 20 490 1624 0 254 130560 0 PROC 1352 73 17 49930 0 8 121680 0 64 Bucket 512 132 100 23731 0 30 118784 0 ksiginfo 112 80 964 8681 0 126 116928 0 filedesc0 1072 73 25 49931 0 8 105056 0 malloc-128 128 519 256 5629 0 126 99200 0 malloc-4096 4096 18 4 656 0 2 90112 0 malloc-4096 4096 22 0 22 0 2 90112 0 UMA Kegs 384 230 3 230 0 30 89472 0 pipe 744 23 92 1186 0 16 85560 0 clpbuf 2624 0 32 41 0 16 83968 0 malloc-256 256 244 71 53438 0 62 80640 0 32 Bucket 256 115 200 21602 0 62 80640 0 sctp_readq 152 0 520 8 0 254 79040 0 malloc-32 32 1631 763 178222 0 254 76608 0 tcp_inpcb 424 8 172 11779 0 30 76320 0 malloc-8192 8192 9 0 9 0 1 73728 0 g_bio 408 0 180 12052 0 30 73440 0 malloc-64 64 682 389 38204 0 254 68544 0 malloc-64 64 729 342 51868 0 254 68544 0 malloc-64 64 529 542 13398 0 254 68544 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-2048 2048 4 28 535 0 8 65536 0 ripcb 424 6 147 1227 0 30 64872 0 Files 80 237 563 172783 0 126 64000 0 sctp_stream_msg_out 112 42 498 2247 0 254 60480 0 malloc-384 384 113 37 222 0 30 57600 0 udp_inpcb 424 6 120 833 0 30 53424 0 malloc-64 64 253 566 670473 0 254 52416 0 malloc-64 64 364 455 100654 0 254 52416 0 malloc-256 256 48 147 99969 0 62 49920 0 DIRHASH 1024 34 14 34 0 16 49152 0 NAMEI 1024 0 48 219179 0 16 49152 0 malloc-1024 1024 14 34 522 0 16 49152 0 tcp_rack_pcb 896 0 54 3257 0 16 48384 0 malloc-384 384 96 24 99 0 30 46080 0 tcp_bbr_pcb 832 0 54 5185 0 16 44928 0 syncache 168 0 264 4 0 254 44352 0 malloc-8192 8192 5 0 5 0 1 40960 0 pcpu-8 8 4705 415 4973 0 254 40960 0 malloc-16 16 1921 579 93633 0 254 40000 0 Mountpoints 2752 2 12 2 0 4 38528 0 udplite_inpcb 424 0 90 433 0 30 38160 0 da_ccb 544 0 70 3129 0 16 38080 0 hostcache 64 2 565 2 0 254 36288 0 malloc-64 64 4 563 21 0 254 36288 0 malloc-64 64 6 561 255 0 254 36288 0 16 Bucket 144 76 176 725 0 62 36288 0 malloc-16 16 1367 883 128211 0 254 36000 0 8 Bucket 80 66 384 10988 0 126 36000 0 malloc-128 128 77 202 3352 0 126 35712 0 malloc-128 128 2 277 376 0 126 35712 0 malloc-128 128 22 257 2044 0 126 35712 0 routing nhops 256 27 108 35 0 62 34560 0 ttyoutq 256 72 63 160 0 62 34560 0 malloc-256 256 37 98 49895 0 62 34560 0 malloc-256 256 54 81 365 0 62 34560 0 malloc-256 256 20 115 14209 0 62 34560 0 TURNSTILE 136 210 42 210 0 62 34272 0 SLEEPQUEUE 88 210 174 210 0 126 33792 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-4096 4096 3 5 49458 0 2 32768 0 malloc-2048 2048 3 13 467 0 8 32768 0 malloc-2048 2048 3 13 3 0 8 32768 0 malloc-2048 2048 9 7 202 0 8 32768 0 malloc-2048 2048 10 6 22 0 8 32768 0 malloc-2048 2048 3 13 13 0 8 32768 0 malloc-1024 1024 4 28 348 0 16 32768 0 malloc-1024 1024 19 13 40 0 16 32768 0 malloc-1024 1024 8 24 34 0 16 32768 0 malloc-1024 1024 1 31 77 0 16 32768 0 malloc-512 512 4 60 185 0 30 32768 0 malloc-512 512 9 55 9 0 30 32768 0 malloc-512 512 2 62 137 0 30 32768 0 malloc-512 512 10 54 27 0 30 32768 0 pcpu-64 64 492 20 492 0 254 32768 0 KNOTE 160 28 172 493783 0 62 32000 0 ttyinq 160 135 65 300 0 62 32000 0 tcp_rack_map 120 0 264 6091 0 126 31680 0 cpuset 104 7 272 1587 0 126 29016 0 sctp_laddr 48 12 576 777 0 254 28224 0 tcp_inpcb ports 32 2 880 7941 0 254 28224 0 ertt 72 8 384 11779 0 126 28224 0 PWD 32 31 851 48973 0 254 28224 0 malloc-32 32 293 589 50105 0 254 28224 0 4 Bucket 48 7 581 1095 0 254 28224 0 AIO 208 0 133 93 0 62 27664 0 malloc-8192 8192 2 1 4 0 1 24576 0 malloc-4096 4096 6 0 6 0 2 24576 0 rtentry 176 31 107 35 0 62 24288 0 PGRP 88 38 238 98 0 126 24288 0 rl_entry 40 87 519 87 0 254 24240 0 malloc-384 384 27 33 27 0 30 23040 0 malloc-384 384 0 60 252 0 30 23040 0 domainset 40 0 567 1190 0 254 22680 0 udpcb 32 6 624 1266 0 254 20160 0 udplite_inpcb ports 32 0 630 28 0 254 20160 0 udp_inpcb ports 32 3 627 61 0 254 20160 0 AIOP 32 4 626 4 0 254 20160 0 malloc-32 32 15 615 818 0 254 20160 0 malloc-32 32 35 595 3793 0 254 20160 0 malloc-32 32 12 618 25 0 254 20160 0 malloc-32 32 65 565 1705 0 254 20160 0 malloc-32 32 14 616 3420 0 254 20160 0 2 Bucket 32 73 557 4962 0 254 20160 0 cryptop 280 0 70 6 0 30 19600 0 AIOCB 552 0 35 186 0 16 19320 0 AIOLIO 272 0 70 25 0 30 19040 0 epoch_record pcpu 256 4 60 4 0 62 16384 0 malloc-16384 16384 0 1 5 0 1 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-8192 8192 2 0 2 0 1 16384 0 malloc-8192 8192 0 2 28 0 1 16384 0 malloc-2048 2048 2 6 2 0 8 16384 0 malloc-512 512 0 32 136 0 30 16384 0 malloc-512 512 0 32 1 0 30 16384 0 malloc-512 512 1 31 1 0 30 16384 0 SMR CPU 32 7 504 7 0 254 16352 0 tcpreass 48 0 336 1 0 254 16128 0 vtnet_tx_hdr 24 0 668 1148015 0 254 16032 0 kenv 258 15 45 1039 0 30 15480 0 mqnode 416 3 33 3 0 30 14976 0 vmem 1856 1 7 1 0 8 14848 0 SMR SHARED 24 7 504 7 0 254 12264 0 malloc-16 16 25 725 80 0 254 12000 0 malloc-16 16 31 719 37240 0 254 12000 0 malloc-16 16 8 742 121 0 254 12000 0 malloc-16 16 63 687 106 0 254 12000 0 malloc-16 16 5 745 32880 0 254 12000 0 L VFS Cache 320 0 36 1 0 30 11520 0 malloc-384 384 1 29 1 0 30 11520 0 malloc-8192 8192 1 0 1 0 1 8192 0 malloc-4096 4096 1 1 1 0 2 8192 0 malloc-4096 4096 1 1 1 0 2 8192 0 pcpu-16 16 14 498 14 0 254 8192 0 UMA Slabs 1 176 9 13 9 0 62 3872 0 KMAP ENTRY 96 12 27 14 0 0 3744 0 FFS1 dinode 128 0 0 0 0 126 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 62 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 312 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 56 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 72 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 tcp_log_node 120 0 0 0 0 126 0 0 tcp_log_bucket 176 0 0 0 0 62 0 0 ripcb ports 32 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 itimer 352 0 0 0 0 30 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 TMPFS node 224 0 0 0 0 62 0 0 NCLNODE 608 0 0 0 0 16 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 linux_dma_object 32 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-1024 1024 0 0 0 0 16 0 0 malloc-512 512 0 0 0 0 30 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0