panic: Don't own TCB send lock cpuid = 1 time = 1648492482 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0xc7/frame 0xfffffe0054150810 kdb_backtrace() at kdb_backtrace+0xd3/frame 0xfffffe0054150970 vpanic() at vpanic+0x2b8/frame 0xfffffe0054150a50 panic() at panic+0xb5/frame 0xfffffe0054150b10 sctp_ss_default_add() at sctp_ss_default_add+0x11d/frame 0xfffffe0054150b50 sctp_msg_append() at sctp_msg_append+0x646/frame 0xfffffe0054150bb0 sctp_sendall_iterator() at sctp_sendall_iterator+0x207/frame 0xfffffe0054150d70 sctp_iterator_worker() at sctp_iterator_worker+0xbb1/frame 0xfffffe0054150ed0 sctp_iterator_thread() at sctp_iterator_thread+0x5e/frame 0xfffffe0054150ef0 fork_exit() at fork_exit+0xd0/frame 0xfffffe0054150f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0054150f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- KDB: enter: panic [ thread pid 5 tid 100068 ] Stopped at kdb_enter+0x6b: movq $0,0x270a69a(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0 rax 0x12 rcx 0xd5043d2d314208c rdx 0x1 rbx 0 rsp 0xfffffe0054150950 rbp 0xfffffe0054150970 rsi 0 rdi 0xffffffff8177a5ea vprintf+0x35a r8 0 r9 0xffffffff r10 0 r11 0xfffffe0099fcbc50 r12 0xfffffe005786a720 r13 0xfffffe0054150901 r14 0xffffffff82bb9d20 .str.26 r15 0xffffffff82bb9d20 .str.26 rip 0xffffffff8176db7b kdb_enter+0x6b rflags 0x46 kdb_enter+0x6b: movq $0,0x270a69a(%rip) db> show proc Process 5 (sctp_iterator) at 0xfffffe0057994a90: state: NORMAL uid: 0 gids: 0 parent: pid 0 at 0xffffffff83e1b8c0 ABI: null flag: 0x10000204 flag2: 0 reaper: 0xffffffff83e1b8c0 reapsubtree: 5 sigparent: 20 vmspace: 0xffffffff83e1c860 (map 0xffffffff83e1c860) (map.pmap 0xffffffff83e1c920) (pmap 0xffffffff83e1c988) threads: 1 100068 Run CPU 1 [sctp_iterator] db> ps pid ppid pgrp uid state wmesg wchan cmd 7124 789 789 0 R syz-executor.2 7123 794 794 0 R (threaded) syz-executor.3 104231 Run CPU 0 syz-executor.3 107265 RunQ syz-executor.3 7122 780 780 0 R syz-executor.1 7121 779 779 0 R (threaded) syz-executor.0 105150 RunQ syz-executor.0 107264 RunQ syz-executor.0 6510 1 6510 65 Ss select 0xfffffe0099fcd640 dhclient 5272 1 5272 0 Ss select 0xfffffe009a0ff640 dhclient 5265 1 5265 0 Ss select 0xfffffe0099fce040 dhclient 5245 1 5245 65 Ss select 0xfffffe009a0ff740 dhclient 4167 1 794 0 S uwait 0xfffffe0057877480 syz-executor.3 3919 1 789 0 S uwait 0xfffffe0057448980 syz-executor.2 3910 1 794 0 S uwait 0xfffffe0057448880 syz-executor.3 3645 1 3645 0 Ss select 0xfffffe0057879ec0 dhclient 3638 1 3638 0 Ss select 0xfffffe0057879e40 dhclient 3601 1 3601 65 Ss select 0xfffffe008fe89040 dhclient 3555 1 780 0 S uwait 0xfffffe008fe89700 syz-executor.1 3519 1 794 0 S uwait 0xfffffe0099fcf600 syz-executor.3 3421 1 789 0 S uwait 0xfffffe0056f48580 syz-executor.2 3415 1 789 0 S umtxn 0xfffffe0099fcf500 syz-executor.2 3411 1 789 0 S uwait 0xfffffe0057876800 syz-executor.2 3406 1 789 0 S uwait 0xfffffe008fe8b500 syz-executor.2 3400 1 789 0 S uwait 0xfffffe0057876e00 syz-executor.2 3396 1 789 0 S uwait 0xfffffe008fe8ad80 syz-executor.2 2906 1 789 0 S uwait 0xfffffe008fe8b600 syz-executor.2 2899 1 789 0 S uwait 0xfffffe008fe89800 syz-executor.2 2897 1 789 0 S uwait 0xfffffe0057448a80 syz-executor.2 2875 1 780 0 S uwait 0xfffffe008fe89e00 syz-executor.1 2470 1 2470 0 Ss select 0xfffffe008fe892c0 dhclient 2467 1 2467 0 Ss select 0xfffffe008fe89140 dhclient 2442 1 2442 65 Ss select 0xfffffe008fe89440 dhclient 2426 1 780 0 SV uwait 0xfffffe0057877680 syz-executor.1 2420 1 780 0 SV uwait 0xfffffe008fe8b200 syz-executor.1 2412 1 780 0 SV uwait 0xfffffe008fe89d00 syz-executor.1 2402 1 780 0 SV uwait 0xfffffe008fe8ac80 syz-executor.1 2393 1 780 0 SV uwait 0xfffffe008fe8a100 syz-executor.1 2383 1 780 0 SV uwait 0xfffffe0057876600 syz-executor.1 2376 1 780 0 SV uwait 0xfffffe008fe8b000 syz-executor.1 2367 1 780 0 SV uwait 0xfffffe008fe89a00 syz-executor.1 2355 1 780 0 SV uwait 0xfffffe0057878180 syz-executor.1 868 0 0 0 DL aiordy 0xfffffe009294fa90 [aiod4] 867 0 0 0 DL aiordy 0xfffffe0092d80000 [aiod3] 866 0 0 0 DL aiordy 0xfffffe0092950000 [aiod2] 865 0 0 0 DL aiordy 0xfffffe0092950548 [aiod1] 820 1 820 0 Ss select 0xfffffe008fe8aa40 dhclient 813 1 813 0 Ss select 0xfffffe0057876f40 dhclient 794 774 794 0 Rs syz-executor.3 789 774 789 0 Rs syz-executor.2 780 774 780 0 Ss nanslp 0xffffffff83e458c1 syz-executor.1 779 774 779 0 Rs syz-executor.0 774 772 772 0 S (threaded) syz-fuzzer 100090 S uwait 0xfffffe0057878480 syz-fuzzer 100115 S uwait 0xfffffe0056f48980 syz-fuzzer 100116 S uwait 0xfffffe0056f48a80 syz-fuzzer 100117 S uwait 0xfffffe0056f48b80 syz-fuzzer 100118 S uwait 0xfffffe0056f48c80 syz-fuzzer 100119 S uwait 0xfffffe0057878280 syz-fuzzer 100120 S uwait 0xfffffe0057878380 syz-fuzzer 100122 S uwait 0xfffffe0057877c00 syz-fuzzer 101869 S kqread 0xfffffe000796fe00 syz-fuzzer 772 770 772 0 Ss pause 0xfffffe0092d815f8 csh 770 688 770 0 Ss select 0xfffffe008fe8bc40 sshd 754 1 754 0 Ss+ ttyin 0xfffffe0057466cb0 getty 753 1 753 0 Ss+ ttyin 0xfffffe00574654b0 getty 752 1 752 0 Ss+ ttyin 0xfffffe00579ae4b0 getty 751 1 751 0 Ss+ ttyin 0xfffffe00574658b0 getty 750 1 750 0 Ss+ ttyin 0xfffffe0057465cb0 getty 749 1 749 0 Ss+ ttyin 0xfffffe00579ae8b0 getty 748 1 748 0 Ss+ ttyin 0xfffffe00579aecb0 getty 747 1 747 0 Ss+ ttyin 0xfffffe00579af0b0 getty 746 1 746 0 Ss+ ttyin 0xfffffe00579af4b0 getty 692 1 692 0 Ss nanslp 0xffffffff83e458c1 cron 688 1 688 0 Ss select 0xfffffe008fe8c1c0 sshd 501 1 501 0 Ss select 0xfffffe0057877a40 syslogd 430 1 430 0 Ss select 0xfffffe008fe8c440 devd 429 1 429 65 Ss select 0xfffffe00578779c0 dhclient 344 1 344 0 Ss select 0xfffffe008fe8c140 dhclient 341 1 341 0 Ss select 0xfffffe00574460c0 dhclient 17 0 0 0 DL vlruwt 0xfffffe0057990a90 [vnlru] 16 0 0 0 DL syncer 0xffffffff83f6b0e0 [syncer] 15 0 0 0 DL (threaded) [bufdaemon] 100080 D psleep 0xffffffff83f696e0 [bufdaemon] 100081 D - 0xffffffff83211f80 [bufspacedaemon-0] 100092 D sdflush 0xfffffe00574650e8 [/ worker] 9 0 0 0 DL psleep 0xffffffff83f9d180 [vmdaemon] 8 0 0 0 DL (threaded) [pagedaemon] 100078 D psleep 0xffffffff83f91038 [dom0] 100082 D launds 0xffffffff83f91044 [laundry: dom0] 100083 D umarcl 0xffffffff81ea4500 [uma] 7 0 0 0 DL - 0xffffffff83c01688 [rand_harvestq] 6 0 0 0 DL pftm 0xffffffff84590530 [pf purge] 5 0 0 0 RL CPU 1 [sctp_iterator] 4 0 0 0 DL (threaded) [cam] 100045 D - 0xffffffff83aa35c0 [doneq0] 100046 D - 0xffffffff83aa3540 [async] 100077 D - 0xffffffff83aa33c0 [scanner] 14 0 0 0 DL seqstat 0xfffffe0053fbb488 [sequencer 00] 3 0 0 0 DL (threaded) [crypto] 100041 D crypto_ 0xffffffff83f8c840 [crypto] 100042 D crypto_ 0xfffffe0053c8ad30 [crypto returns 0] 100043 D crypto_ 0xfffffe0053c8ad80 [crypto returns 1] 13 0 0 0 DL (threaded) [geom] 100036 D - 0xffffffff83e1ae80 [g_event] 100037 D - 0xffffffff83e1aea0 [g_up] 100038 D - 0xffffffff83e1aec0 [g_down] 2 0 0 0 WL (threaded) [clock] 100030 I [clock (0)] 100031 I [clock (1)] 12 0 0 0 WL (threaded) [intr] 100010 I [swi6: task queue] 100011 I [swi6: Giant taskq] 100018 I [swi5: fast taskq] 100029 I [swi1: netisr 0] 100032 I [swi3: busdma] 100033 I [swi1: hpts] 100034 I [swi1: hpts] 100047 I [irq24: virtio_pci0] 100048 I [irq25: virtio_pci0] 100049 I [irq26: virtio_pci0] 100050 I [irq27: virtio_pci0] 100051 I [irq28: virtio_pci1] 100052 I [irq29: virtio_pci1] 100053 I [irq30: virtio_pci1] 100054 I [irq31: virtio_pci1] 100055 I [irq32: virtio_pci1] 100060 I [irq33: virtio_pci2] 100061 I [irq34: virtio_pci2] 100062 I [irq35: virtio_pci2] 100064 I [irq1: atkbd0] 100065 I [irq12: psm0] 100066 I [swi0: uart uart++] 100070 I [swi1: pf send] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffffe0053dda000 [init] 10 0 0 0 DL audit_w 0xffffffff83f8d340 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff83e1b8c0 [swapper] 100005 D - 0xfffffe0053e01000 [if_config_tqg_0] 100006 D - 0xfffffe0053e00e00 [softirq_0] 100007 D - 0xfffffe0053e00d00 [softirq_1] 100008 D - 0xfffffe0053e00c00 [if_io_tqg_0] 100009 D - 0xfffffe0053e00b00 [if_io_tqg_1] 100012 D - 0xfffffe000796d200 [aiod_kick taskq] 100013 D - 0xfffffe000796d000 [inm_free taskq] 100014 D - 0xfffffe000796cd00 [linuxkpi_irq_wq] 100015 D - 0xfffffe000796cb00 [in6m_free taskq] 100016 D - 0xfffffe000796c900 [deferred_unmount ta] 100017 D - 0xfffffe000796c700 [thread taskq] 100019 D - 0xfffffe000796c300 [kqueue_ctx taskq] 100020 D - 0xfffffe000796c100 [pci_hp taskq] 100021 D - 0xfffffe000796be00 [linuxkpi_short_wq_0] 100022 D - 0xfffffe000796be00 [linuxkpi_short_wq_1] 100023 D - 0xfffffe000796be00 [linuxkpi_short_wq_2] 100024 D - 0xfffffe000796be00 [linuxkpi_short_wq_3] 100025 D - 0xfffffe000796b900 [linuxkpi_long_wq_0] 100026 D - 0xfffffe000796b900 [linuxkpi_long_wq_1] 100027 D - 0xfffffe000796b900 [linuxkpi_long_wq_2] 100028 D - 0xfffffe000796b900 [linuxkpi_long_wq_3] 100035 D - 0xfffffe0053ed0200 [firmware taskq] 100039 D - 0xfffffe0053ecfc00 [crypto_0] 100040 D - 0xfffffe0053ecfc00 [crypto_1] 100056 D - 0xfffffe0053eced00 [vtnet0 rxq 0] 100057 D - 0xfffffe0053ecec00 [vtnet0 txq 0] 100058 D - 0xfffffe0053eceb00 [vtnet0 rxq 1] 100059 D - 0xfffffe0053ecea00 [vtnet0 txq 1] 100063 D vtbslp 0xfffffe0053fdc380 [virtio_balloon] 100067 D - 0xffffffff82bbfba0 [deadlkres] 100071 D - 0xfffffe005787d100 [acpi_task_0] 100072 D - 0xfffffe005787d100 [acpi_task_1] 100073 D - 0xfffffe005787d100 [acpi_task_2] 100074 D - 0xfffffe000796e200 [mca taskq] 100076 D - 0xfffffe0053ecf500 [CAM taskq] db> show all locks Process 5 (sctp_iterator) thread 0xfffffe005786a720 (100068) exclusive sleep mutex sctp-tcb (tcb) r = 0 (0xfffffe00996d41a0) locked @ /syzkaller/managers/i386/kernel/sys/netinet/sctputil.c:1523 exclusive sleep mutex sctp-inp (inp) r = 0 (0xfffffe009960ed90) locked @ /syzkaller/managers/i386/kernel/sys/netinet/sctputil.c:1463 exclusive sleep mutex sctp-it (iterator) r = 0 (0xffffffff84b84560) locked @ /syzkaller/managers/i386/kernel/sys/netinet/sctputil.c:1460 shared rw sctp-info (sctp-info) r = 0 (0xfffffe0053c44980) locked @ /syzkaller/managers/i386/kernel/sys/netinet/sctputil.c:1459 db> show malloc Type InUse MemUse Requests pf_hash 5 11524K 5 tcp_hpts 6 4801K 6 devbuf 4217 4323K 4245 sysctloid 35306 2080K 35377 vtbuf 24 1968K 46 kobj 327 1308K 488 newblk 9 1026K 28792 vfscache 3 1025K 3 pcb 52 562K 1777 inodedep 67 537K 8476 filedesc 66 521K 12473 ufs_quota 1 512K 1 vfs_hash 1 512K 1 callout 2 512K 2 intr 4 472K 4 subproc 191 380K 7231 vmem 3 274K 6 acpica 1674 184K 56014 vnet_data 1 168K 1 tidhash 3 141K 3 linker 358 134K 386 pagedep 10 131K 5949 tfo_ccache 1 128K 1 DEVFS1 109 109K 126 sem 4 106K 4 BPF 46 88K 52 bus 991 81K 5140 mtx_pool 2 72K 2 syncache 1 68K 1 module 512 64K 512 acpitask 1 64K 1 ddb_capture 1 64K 1 umtx 418 53K 418 kdtrace 269 52K 14391 temp 34 33K 2963 DEVFS3 128 32K 138 hostcache 1 32K 1 shm 1 32K 4 msg 4 30K 4 gtaskqueue 18 26K 18 kbdmux 6 22K 6 ifaddr 70 20K 72 DEVFS_RULE 56 20K 56 ufs_mount 4 17K 5 proc 3 17K 3 tty 16 16K 16 ithread 100 16K 100 routetbl 128 16K 411 dirrem 58 15K 7305 bus-sc 34 15K 1651 KTRACE 100 13K 100 lltable 41 13K 79 ifnet 7 13K 7 ether_multi 152 13K 162 kenv 95 12K 95 eventhandler 134 12K 134 rman 88 11K 429 GEOM 61 11K 490 CAM queue 5 11K 1528 in6_multi 65 9K 65 bmsafemap 2 9K 7909 UART 12 9K 12 devstat 4 9K 4 sctp_timw 32 8K 32 ksem 1 8K 197 rpc 2 8K 2 shmfd 1 8K 22 pfs_vncache 1 8K 1 kqueue 87 8K 7129 pfs_nodes 20 8K 20 audit_evclass 237 8K 296 freefile 58 8K 7279 taskqueue 63 7K 63 cred 26 7K 244 sglist 5 7K 5 CAM DEV 3 6K 510 plimit 24 6K 424 pwddesc 85 6K 7125 pf_rule 40 5K 82 ufs_dirhash 24 5K 24 sctp_atcl 12 5K 975 UMA 272 5K 272 DEVFSP 70 5K 710 session 35 5K 51 pf_ifnet 10 5K 19 vt 11 5K 11 memdesc 1 4K 1 MCA 32 4K 32 evdev 4 4K 4 proc-args 116 4K 8391 acpisem 28 4K 28 selfd 55 4K 105687 lockf 32 4K 54 hhook 15 4K 17 kcovinfo 52 4K 52 terminal 11 3K 11 select 19 3K 65 uidinfo 3 3K 9 local_apic 1 2K 1 io_apic 1 2K 1 fpukern_ctx 2 2K 2 ipsec-saq 2 2K 2 tcp_fsb 15 2K 1863 ip6ndp 12 2K 13 Unitno 30 2K 3105 sctp_ifa 13 2K 14 CAM XPT 22 2K 543 msi 12 2K 12 CC Mem 24 2K 5017 in_multi 6 2K 8 inpcbpolicy 42 2K 6014 ipsecpolicy 2 2K 2 acpidev 20 2K 20 clone 9 2K 9 tun 7 2K 7 freework 5 2K 15087 sctp_stro 1 1K 135 softdep 1 1K 1 mkdir 8 1K 11862 freeblks 4 1K 7226 sahead 1 1K 1 secasvar 1 1K 1 nhops 6 1K 8 vnodemarker 2 1K 42 NFSD session 1 1K 1 CAM periph 4 1K 271 sctp_ifn 6 1K 14 ipsec 3 1K 3 mld 6 1K 6 igmp 6 1K 6 toponodes 6 1K 6 isadev 6 1K 6 mount 16 1K 89 pci_link 10 1K 10 osd 28 1K 5023 newdirblk 5 1K 5931 crypto 4 1K 4 encap_export_host 12 1K 12 procdesc 5 1K 14 diradd 4 1K 7345 pfil 4 1K 4 cdev 2 1K 2 sctp_atky 13 1K 1136 chacha20random 1 1K 1 sctp_iter 1 1K 13 sctp_cpal 1 1K 1 cryptodev 4 1K 625 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 DEVFS 9 1K 10 vnodes 1 1K 1 CAM SIM 2 1K 2 feeder 7 1K 7 sctp_athm 12 1K 998 tcpfunc 3 1K 3 loginclass 3 1K 6 prison 6 1K 6 lkpikmalloc 5 1K 6 aesni_data 2 1K 2 nexusdev 8 1K 8 apmdev 1 1K 1 atkbddev 2 1K 2 CAM dev queue 2 1K 2 CAM I/O Scheduler 1 1K 1 CAM path 4 1K 1034 pmchooks 1 1K 1 soname 4 1K 12420 filecaps 5 1K 98 sctp_vrf 1 1K 1 vnet 1 1K 1 entropy 2 1K 42 pmc 1 1K 1 acpiintr 1 1K 1 sctp_map 2 1K 270 cpus 2 1K 2 vnet_data_free 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 sctp_mcore 0 0K 0 sctp_socko 0 0K 605 sctp_mvrf 0 0K 0 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 12 sctp_aadr 0 0K 0 sctp_stri 0 0K 46 ipcomp 0 0K 0 esp 0 0K 0 ah 0 0K 0 tcp_do 0 0K 0 filemon 0 0K 149 pf_table 0 0K 0 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_temp 0 0K 0 mqdata 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 ixl 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 iavf 0 0K 0 axgbe 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 xen_intr 0 0K 0 NFSD srvcache 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 bounce 0 0K 0 busdma 0 0K 0 qpidrv 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 msdosfs_node 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 DEVFS4 0 0K 0 xenbus 0 0K 0 DEVFS2 0 0K 0 vm_fictitious 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xenstore 0 0K 0 scsi_pass 0 0K 0 ciss_data 0 0K 0 xnb 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 sysmouse 0 0K 0 UMAHash 0 0K 0 vtfont 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 23123 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 18 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 2 allocindir 0 0K 0 indirdep 0 0K 13765 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 BACKLIGHT 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 pvscsi 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 scsi_cd 0 0K 0 ktls_ocf 0 0K 0 AHCI driver 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS_RX 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 lkpiskb 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpindev 0 0K 0 lkpifw 0 0K 0 lkpi80211 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 9 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 LRO 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 0 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 0 VN POLL 0 0K 0 agp 0 0K 0 statfs 0 0K 6113 namei_tracker 0 0K 0 export_host 0 0K 0 cl_savebuf 0 0K 3 nvme_da 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 twsbuf 0 0K 0 twe_commands 0 0K 0 tcp_log_dev 0 0K 849 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 523 biobuf 0 0K 0 aios 0 0K 0 lio 0 0K 0 acl 0 0K 0 mbuf_tag 0 0K 136 ktls 0 0K 0 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 tempbuff 0 0K 0 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 accf 0 0K 0 pts 0 0K 0 iov 0 0K 19072 ioctlops 0 0K 356 eventfd 0 0K 0 Witness 0 0K 0 stack 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 sbuf 0 0K 288 mpr_user 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 MPRSAS 0 0K 0 SWAP 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 sysctltmp 0 0K 821 sysctl 0 0K 3 md_sectors 0 0K 0 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 md_disk 0 0K 0 malodev 0 0K 0 LED 0 0K 0 ix_sriov 0 0K 0 cache 0 0K 0 aacraidcam 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 1 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 aacraid_buf 0 0K 0 aaccam 0 0K 0 boottrace 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8320 1078 1385713 0 254 38494208 0 tcp_log 416 0 10161 88556 0 254 4226976 0 mbuf 256 8637 1026 2367214 0 254 2473728 0 RADIX NODE 144 14263 208 223298 0 62 2083824 0 pbuf 2624 0 794 0 0 2 2083456 0 BUF TRIE 144 186 11602 28139 0 62 1697472 0 malloc-384 384 4117 53 5111 0 30 1601280 0 mbuf_cluster 2048 762 0 762 0 254 1560576 0 malloc-128 128 11689 184 11835 0 126 1519744 0 malloc-4096 4096 327 3 492 0 2 1351680 0 UMA Slabs 0 112 10952 22 10952 0 126 1229088 0 sctp_asoc 2288 1 509 135 0 254 1166880 0 vmem btag 56 19028 91 19028 0 254 1070664 0 VM OBJECT 264 2547 153 117588 0 30 712800 0 malloc-16384 16384 37 4 6414 0 1 671744 0 FFS inode 1160 540 34 7820 0 8 665840 0 sctp_ep 1208 11 499 817 0 254 616080 0 tcpcb 1104 24 487 5012 0 254 564144 0 256 Bucket 2048 240 18 5604 0 8 528384 0 socket 960 78 430 8177 0 254 487680 0 malloc-65536 65536 4 2 328 0 1 393216 0 lkpimm 168 1 2327 1 0 62 391104 0 lkpicurr 168 2 2326 2 0 62 391104 0 malloc-256 256 284 1216 37737 0 62 384000 0 sctp_raddr 736 1 516 140 0 254 380512 0 THREAD 1808 183 26 7265 0 8 377872 0 malloc-4096 4096 85 5 7127 0 2 368640 0 MAP ENTRY 96 3279 501 281793 0 126 362880 0 malloc-384 384 81 789 8511 0 30 334080 0 malloc-2048 2048 14 146 1534 0 8 327680 0 VNODE 448 579 114 7861 0 30 310464 0 tcp_bbr_map 128 0 2418 16279 0 126 309504 0 ertt_txseginfo 40 0 6868 68074 0 254 274720 0 malloc-64 64 3873 222 4447 0 254 262080 0 malloc-16 16 14613 387 14683 0 254 240000 0 malloc-256 256 309 621 29434 0 62 238080 0 DEVCTL 1024 0 220 154 0 0 225280 0 VMSPACE 2552 64 14 7101 0 4 199056 0 malloc-65536 65536 1 2 10 0 1 196608 0 malloc-32768 32768 0 6 13100 0 1 196608 0 mbuf_packet 256 1 761 6600 0 254 195072 0 UMA Zones 768 244 0 244 0 16 187392 0 FPU_save_area 832 185 40 10344 0 16 187200 0 malloc-32 32 5390 280 5616 0 254 181440 0 malloc-128 128 1228 167 26361 0 126 178560 0 FFS2 dinode 256 540 150 7819 0 62 176640 0 malloc-32768 32768 0 5 120 0 1 163840 0 malloc-1024 1024 125 19 1490 0 16 147456 0 malloc-128 128 971 176 2279 0 126 146816 0 S VFS Cache 104 1023 381 8443 0 126 146016 0 PROC 1352 84 21 7124 0 8 141960 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-128 128 153 870 32520 0 126 130944 0 unpcb 256 20 490 1309 0 254 130560 0 128 Bucket 1024 86 29 839 0 16 117760 0 ksiginfo 112 95 949 3811 0 126 116928 0 filedesc0 1072 85 20 7125 0 8 112560 0 malloc-128 128 519 256 4692 0 126 99200 0 malloc-8192 8192 6 6 285 0 1 98304 0 malloc-2048 2048 4 44 535 0 8 98304 0 malloc-4096 4096 18 4 656 0 2 90112 0 malloc-4096 4096 22 0 22 0 2 90112 0 UMA Kegs 384 230 3 230 0 30 89472 0 64 Bucket 512 111 57 10595 0 30 86016 0 malloc-384 384 12 198 1009 0 30 80640 0 malloc-256 256 19 296 828 0 62 80640 0 sctp_chunk 152 0 520 50 0 254 79040 0 malloc-8192 8192 9 0 9 0 1 73728 0 g_bio 408 0 180 189658 0 30 73440 0 malloc-384 384 124 56 162 0 30 69120 0 malloc-64 64 741 330 8979 0 254 68544 0 malloc-64 64 545 526 6627 0 254 68544 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-256 256 49 206 15126 0 62 65280 0 32 Bucket 256 113 142 7771 0 62 65280 0 sctp_stream_msg_out 112 1 539 13 0 254 60480 0 tcp_inpcb 424 24 102 5012 0 30 53424 0 ripcb 424 12 114 95 0 30 53424 0 udp_inpcb 424 6 120 900 0 30 53424 0 malloc-64 64 1 818 17140 0 254 52416 0 malloc-64 64 253 566 105909 0 254 52416 0 malloc-64 64 386 433 15013 0 254 52416 0 Files 80 260 390 24639 0 126 52000 0 malloc-256 256 52 143 16474 0 62 49920 0 malloc-256 256 37 158 7958 0 62 49920 0 DIRHASH 1024 34 14 34 0 16 49152 0 NAMEI 1024 0 48 45647 0 16 49152 0 malloc-1024 1024 14 34 522 0 16 49152 0 tcp_rack_pcb 896 15 39 1863 0 16 48384 0 malloc-384 384 95 25 97 0 30 46080 0 da_ccb 544 0 84 47526 0 16 45696 0 tcp_bbr_pcb 832 0 54 1947 0 16 44928 0 syncache 168 0 264 4 0 254 44352 0 TURNSTILE 136 210 105 210 0 62 42840 0 clpbuf 2624 0 16 28 0 16 41984 0 malloc-8192 8192 1 4 8 0 1 40960 0 malloc-8192 8192 5 0 5 0 1 40960 0 pcpu-8 8 4701 419 4899 0 254 40960 0 sctp_readq 152 0 260 5 0 254 39520 0 pipe 744 23 27 383 0 16 37200 0 malloc-64 64 4 563 21 0 254 36288 0 malloc-64 64 6 561 11 0 254 36288 0 16 Bucket 144 71 181 2369 0 62 36288 0 malloc-128 128 92 187 1947 0 126 35712 0 malloc-128 128 2 277 144 0 126 35712 0 malloc-128 128 22 257 285 0 126 35712 0 routing nhops 256 27 108 34 0 62 34560 0 ttyoutq 256 72 63 160 0 62 34560 0 malloc-256 256 52 83 272 0 62 34560 0 malloc-256 256 20 115 668 0 62 34560 0 SLEEPQUEUE 88 210 174 210 0 126 33792 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-8192 8192 2 2 4 0 1 32768 0 malloc-4096 4096 3 5 6119 0 2 32768 0 malloc-2048 2048 3 13 251 0 8 32768 0 malloc-2048 2048 3 13 3 0 8 32768 0 malloc-2048 2048 9 7 144 0 8 32768 0 malloc-2048 2048 10 6 11 0 8 32768 0 malloc-2048 2048 1 15 6 0 8 32768 0 malloc-1024 1024 2 30 144 0 16 32768 0 malloc-1024 1024 4 28 28 0 16 32768 0 malloc-1024 1024 19 13 39 0 16 32768 0 malloc-1024 1024 8 24 16 0 16 32768 0 malloc-512 512 0 64 46 0 30 32768 0 malloc-512 512 4 60 188 0 30 32768 0 malloc-512 512 9 55 9 0 30 32768 0 malloc-512 512 2 62 42 0 30 32768 0 malloc-512 512 10 54 17 0 30 32768 0 pcpu-64 64 492 20 492 0 254 32768 0 KNOTE 160 28 172 97502 0 62 32000 0 ttyinq 160 135 65 300 0 62 32000 0 tcp_rack_map 120 30 234 2755 0 126 31680 0 cpuset 104 7 272 6890 0 126 29016 0 sctp_laddr 48 0 588 65 0 254 28224 0 tcp_inpcb ports 32 2 880 3276 0 254 28224 0 PWD 32 46 836 6027 0 254 28224 0 malloc-32 32 300 582 9591 0 254 28224 0 malloc-32 32 299 583 7424 0 254 28224 0 4 Bucket 48 7 581 180 0 254 28224 0 2 Bucket 32 61 821 4240 0 254 28224 0 AIO 208 0 133 8 0 62 27664 0 udplite_inpcb 424 0 63 7 0 30 26712 0 malloc-4096 4096 6 0 6 0 2 24576 0 rtentry 176 30 108 34 0 62 24288 0 PGRP 88 35 241 51 0 126 24288 0 rl_entry 40 105 501 105 0 254 24240 0 8 Bucket 80 62 238 1853 0 126 24000 0 malloc-384 384 27 33 27 0 30 23040 0 domainset 40 0 567 5354 0 254 22680 0 hostcache 64 1 314 1 0 254 20160 0 udpcb 32 6 624 907 0 254 20160 0 udp_inpcb ports 32 3 627 62 0 254 20160 0 AIOP 32 4 626 4 0 254 20160 0 ertt 72 24 256 5012 0 126 20160 0 malloc-32 32 16 614 3093 0 254 20160 0 malloc-32 32 41 589 2860 0 254 20160 0 malloc-32 32 12 618 25 0 254 20160 0 malloc-32 32 65 565 1661 0 254 20160 0 malloc-32 32 14 616 123 0 254 20160 0 malloc-16 16 571 679 11489 0 254 20000 0 AIOCB 552 0 35 8 0 16 19320 0 Mountpoints 2752 2 5 2 0 4 19264 0 epoch_record pcpu 256 4 60 4 0 62 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-8192 8192 2 0 2 0 1 16384 0 malloc-2048 2048 2 6 2 0 8 16384 0 malloc-1024 1024 1 15 1 0 16 16384 0 malloc-1024 1024 0 16 2 0 16 16384 0 malloc-512 512 0 32 1 0 30 16384 0 malloc-512 512 1 31 1 0 30 16384 0 SMR CPU 32 7 504 7 0 254 16352 0 vtnet_tx_hdr 24 0 668 696425 0 254 16032 0 kenv 258 15 45 1035 0 30 15480 0 mqnode 416 3 33 3 0 30 14976 0 vmem 1856 1 7 1 0 8 14848 0 SMR SHARED 24 7 504 7 0 254 12264 0 malloc-16 16 14 736 8810 0 254 12000 0 malloc-16 16 25 725 64 0 254 12000 0 malloc-16 16 47 703 30473 0 254 12000 0 malloc-16 16 8 742 120 0 254 12000 0 malloc-16 16 63 687 94 0 254 12000 0 malloc-16 16 5 745 371 0 254 12000 0 malloc-384 384 1 29 1 0 30 11520 0 malloc-4096 4096 1 1 1 0 2 8192 0 malloc-4096 4096 1 1 1 0 2 8192 0 pcpu-16 16 7 249 7 0 254 4096 0 UMA Slabs 1 176 9 13 9 0 62 3872 0 KMAP ENTRY 96 12 27 14 0 0 3744 0 FFS1 dinode 128 0 0 0 0 126 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 62 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 312 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 56 0 0 0 0 254 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 72 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 tcp_log_node 120 0 0 0 0 126 0 0 tcp_log_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 ripcb ports 32 0 0 0 0 254 0 0 udplite_inpcb ports 32 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 itimer 352 0 0 0 0 30 0 0 AIOLIO 272 0 0 0 0 30 0 0 TMPFS node 224 0 0 0 0 62 0 0 NCLNODE 608 0 0 0 0 16 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 cryptop 280 0 0 0 0 30 0 0 linux_dma_object 32 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-512 512 0 0 0 0 30 0 0 malloc-384 384 0 0 0 0 30 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0