panic: Don't own TCB send lock cpuid = 1 time = 1648197937 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0xc7/frame 0xfffffe0054150810 kdb_backtrace() at kdb_backtrace+0xd3/frame 0xfffffe0054150970 vpanic() at vpanic+0x2b8/frame 0xfffffe0054150a50 panic() at panic+0xb5/frame 0xfffffe0054150b10 sctp_ss_default_add() at sctp_ss_default_add+0x11d/frame 0xfffffe0054150b50 sctp_msg_append() at sctp_msg_append+0x646/frame 0xfffffe0054150bb0 sctp_sendall_iterator() at sctp_sendall_iterator+0x207/frame 0xfffffe0054150d70 sctp_iterator_worker() at sctp_iterator_worker+0xbb1/frame 0xfffffe0054150ed0 sctp_iterator_thread() at sctp_iterator_thread+0x5e/frame 0xfffffe0054150ef0 fork_exit() at fork_exit+0xd0/frame 0xfffffe0054150f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0054150f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- KDB: enter: panic [ thread pid 5 tid 100068 ] Stopped at kdb_enter+0x6b: movq $0,0x270a50a(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0x28 rax 0x12 rcx 0x6ad165556c856621 rdx 0x1 rbx 0 rsp 0xfffffe0054150950 rbp 0xfffffe0054150970 rsi 0 rdi 0xffffffff8177a78a vprintf+0x35a r8 0 r9 0xffffffff r10 0 r11 0xfffffe00997376f0 r12 0xfffffe005786a720 r13 0xfffffe0054150901 r14 0xffffffff82bb9ce0 .str.26 r15 0xffffffff82bb9ce0 .str.26 rip 0xffffffff8176dd0b kdb_enter+0x6b rflags 0x46 kdb_enter+0x6b: movq $0,0x270a50a(%rip) db> show proc Process 5 (sctp_iterator) at 0xfffffe0057994a90: state: NORMAL uid: 0 gids: 0 parent: pid 0 at 0xffffffff83e1b8c0 ABI: null flag: 0x10000204 flag2: 0 reaper: 0xffffffff83e1b8c0 reapsubtree: 5 sigparent: 20 vmspace: 0xffffffff83e1c860 (map 0xffffffff83e1c860) (map.pmap 0xffffffff83e1c920) (pmap 0xffffffff83e1c988) threads: 1 100068 Run CPU 1 [sctp_iterator] db> ps pid ppid pgrp uid state wmesg wchan cmd 24705 796 796 0 R (threaded) syz-executor.2 125443 RunQ syz-executor.2 125495 S connec 0xfffffe0099f990da syz-executor.2 125498 RunQ syz-executor.2 24703 777 777 0 R (threaded) syz-executor.0 125469 Run CPU 0 syz-executor.0 125493 S select 0xfffffe0057879a40 syz-executor.0 11782 1 11782 65 Ss select 0xfffffe009a5450c0 dhclient 7801 1 7801 0 Ss select 0xfffffe0057448ac0 dhclient 7792 1 7792 0 Ss select 0xfffffe0057448840 dhclient 7742 1 7742 65 Ss select 0xfffffe0057448b40 dhclient 6182 1 6182 0 Ss select 0xfffffe0057876740 dhclient 6179 1 6179 0 Ss select 0xfffffe00999918c0 dhclient 6149 1 6149 65 Ss select 0xfffffe0099991940 dhclient 5110 1 796 0 S uwait 0xfffffe0092317480 syz-executor.2 5100 1 796 0 S uwait 0xfffffe0099993380 syz-executor.2 5091 1 796 0 S uwait 0xfffffe0057877b80 syz-executor.2 5084 1 796 0 S uwait 0xfffffe0092318300 syz-executor.2 5079 1 796 0 S uwait 0xfffffe0092316b00 syz-executor.2 5073 1 796 0 S uwait 0xfffffe0057877a80 syz-executor.2 5065 1 796 0 S uwait 0xfffffe0092317380 syz-executor.2 5059 1 796 0 S uwait 0xfffffe0099993680 syz-executor.2 5054 1 796 0 S uwait 0xfffffe0092316800 syz-executor.2 5049 1 796 0 S uwait 0xfffffe0092316380 syz-executor.2 5044 1 796 0 S uwait 0xfffffe0099993580 syz-executor.2 5041 1 796 0 S uwait 0xfffffe0092317c80 syz-executor.2 5036 1 796 0 S uwait 0xfffffe0092318200 syz-executor.2 5031 1 796 0 S uwait 0xfffffe0057879b00 syz-executor.2 5025 1 796 0 S uwait 0xfffffe0092317580 syz-executor.2 5023 1 796 0 S uwait 0xfffffe0092316080 syz-executor.2 5018 1 796 0 S uwait 0xfffffe0092318100 syz-executor.2 5014 1 796 0 S uwait 0xfffffe0057877980 syz-executor.2 5012 1 796 0 S uwait 0xfffffe0057877e80 syz-executor.2 5006 1 796 0 S uwait 0xfffffe0092317e80 syz-executor.2 4998 1 796 0 S uwait 0xfffffe0057879880 syz-executor.2 4991 1 796 0 S uwait 0xfffffe0092316f00 syz-executor.2 4984 1 796 0 S uwait 0xfffffe0092317080 syz-executor.2 4974 1 796 0 S uwait 0xfffffe0092318980 syz-executor.2 3984 1 3984 0 Ss select 0xfffffe00923166c0 dhclient 3981 1 3981 0 Ss select 0xfffffe00999930c0 dhclient 3963 1 3963 65 Ss select 0xfffffe00999931c0 dhclient 2137 1 778 0 SV uwait 0xfffffe0099993480 syz-executor.1 2128 1 778 0 SV uwait 0xfffffe0057879780 syz-executor.1 2117 1 778 0 SV uwait 0xfffffe0092316c00 syz-executor.1 1482 1 778 0 S uwait 0xfffffe0057877c80 syz-executor.1 1478 1 778 0 S uwait 0xfffffe0092316180 syz-executor.1 1475 1 778 0 S uwait 0xfffffe0092316480 syz-executor.1 1472 1 778 0 S uwait 0xfffffe0057879f00 syz-executor.1 1089 1 1089 0 Ss select 0xfffffe00923167c0 dhclient 1086 1 1086 0 Ss select 0xfffffe00578767c0 dhclient 816 0 0 0 DL aiordy 0xfffffe0092622a90 [aiod4] 815 0 0 0 DL aiordy 0xfffffe009277e000 [aiod3] 814 0 0 0 DL aiordy 0xfffffe009277e548 [aiod2] 813 0 0 0 DL aiordy 0xfffffe0058df3000 [aiod1] 809 774 809 0 Rs syz-executor.3 796 774 796 0 Ss nanslp 0xffffffff83e458c1 syz-executor.2 778 774 778 0 Rs syz-executor.1 777 774 777 0 Ss nanslp 0xffffffff83e458c1 syz-executor.0 774 772 772 0 S (threaded) syz-fuzzer 100108 S uwait 0xfffffe0057445080 syz-fuzzer 100112 S uwait 0xfffffe0057878280 syz-fuzzer 100113 S uwait 0xfffffe0057878380 syz-fuzzer 100114 S uwait 0xfffffe0057878480 syz-fuzzer 100115 S uwait 0xfffffe0057445180 syz-fuzzer 100116 S uwait 0xfffffe0057878580 syz-fuzzer 100117 S uwait 0xfffffe0057878680 syz-fuzzer 100118 S kqread 0xfffffe0058de7900 syz-fuzzer 100119 S uwait 0xfffffe0092319280 syz-fuzzer 772 770 772 0 Ss pause 0xfffffe0092d655f8 csh 770 688 770 0 Ss select 0xfffffe0092318940 sshd 754 1 754 0 Ss+ ttyin 0xfffffe0057466cb0 getty 753 1 753 0 Ss+ ttyin 0xfffffe00579a68b0 getty 752 1 752 0 Ss+ ttyin 0xfffffe00579a6cb0 getty 751 1 751 0 Ss+ ttyin 0xfffffe00579a70b0 getty 750 1 750 0 Ss+ ttyin 0xfffffe00579a74b0 getty 749 1 749 0 Ss+ ttyin 0xfffffe00579a78b0 getty 748 1 748 0 Ss+ ttyin 0xfffffe00579a7cb0 getty 747 1 747 0 Ss+ ttyin 0xfffffe00579a80b0 getty 746 1 746 0 Ss+ ttyin 0xfffffe00579a84b0 getty 692 1 692 0 Ss nanslp 0xffffffff83e458c1 cron 688 1 688 0 Ss select 0xfffffe0092318b40 sshd 501 1 501 0 Ss select 0xfffffe0057877f40 syslogd 430 1 430 0 Ss select 0xfffffe00923190c0 devd 429 1 429 65 Ss select 0xfffffe0092318c40 dhclient 344 1 344 0 Ss select 0xfffffe0092318dc0 dhclient 341 1 341 0 Ss select 0xfffffe0092319240 dhclient 17 0 0 0 DL vlruwt 0xfffffe0056f7e548 [vnlru] 16 0 0 0 DL syncer 0xffffffff83f6b0e0 [syncer] 15 0 0 0 DL (threaded) [bufdaemon] 100080 D psleep 0xffffffff83f696e0 [bufdaemon] 100083 D - 0xffffffff83211f80 [bufspacedaemon-0] 100092 D sdflush 0xfffffe0057465ce8 [/ worker] 9 0 0 0 DL psleep 0xffffffff83f9d140 [vmdaemon] 8 0 0 0 DL (threaded) [pagedaemon] 100078 D psleep 0xffffffff83f90ff8 [dom0] 100081 D launds 0xffffffff83f91004 [laundry: dom0] 100082 D umarcl 0xffffffff81ea4800 [uma] 7 0 0 0 DL - 0xffffffff83c01688 [rand_harvestq] 6 0 0 0 DL pftm 0xffffffff8486c530 [pf purge] 5 0 0 0 RL CPU 1 [sctp_iterator] 4 0 0 0 DL (threaded) [cam] 100045 D - 0xffffffff83aa35c0 [doneq0] 100046 D - 0xffffffff83aa3540 [async] 100077 D - 0xffffffff83aa33c0 [scanner] 14 0 0 0 DL seqstat 0xfffffe0053fbb488 [sequencer 00] 3 0 0 0 DL (threaded) [crypto] 100041 D crypto_ 0xffffffff83f8c840 [crypto] 100042 D crypto_ 0xfffffe0053c8ad30 [crypto returns 0] 100043 D crypto_ 0xfffffe0053c8ad80 [crypto returns 1] 13 0 0 0 DL (threaded) [geom] 100036 D - 0xffffffff83e1ae80 [g_event] 100037 D - 0xffffffff83e1aea0 [g_up] 100038 D - 0xffffffff83e1aec0 [g_down] 2 0 0 0 WL (threaded) [clock] 100030 I [clock (0)] 100031 I [clock (1)] 12 0 0 0 WL (threaded) [intr] 100010 I [swi6: task queue] 100011 I [swi6: Giant taskq] 100018 I [swi5: fast taskq] 100029 I [swi1: netisr 0] 100032 I [swi3: busdma] 100033 I [swi1: hpts] 100034 I [swi1: hpts] 100047 I [irq24: virtio_pci0] 100048 I [irq25: virtio_pci0] 100049 I [irq26: virtio_pci0] 100050 I [irq27: virtio_pci0] 100051 I [irq28: virtio_pci1] 100052 I [irq29: virtio_pci1] 100053 I [irq30: virtio_pci1] 100054 I [irq31: virtio_pci1] 100055 I [irq32: virtio_pci1] 100060 I [irq33: virtio_pci2] 100061 I [irq34: virtio_pci2] 100062 I [irq35: virtio_pci2] 100064 I [irq1: atkbd0] 100065 I [irq12: psm0] 100066 I [swi0: uart uart++] 100070 I [swi1: pf send] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffffe0053dda000 [init] 10 0 0 0 DL audit_w 0xffffffff83f8d340 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff83e1b8c0 [swapper] 100005 D - 0xfffffe0053e01000 [if_config_tqg_0] 100006 D - 0xfffffe0053e00e00 [softirq_0] 100007 D - 0xfffffe0053e00d00 [softirq_1] 100008 D - 0xfffffe0053e00c00 [if_io_tqg_0] 100009 D - 0xfffffe0053e00b00 [if_io_tqg_1] 100012 D - 0xfffffe000796d200 [aiod_kick taskq] 100013 D - 0xfffffe000796d000 [inm_free taskq] 100014 D - 0xfffffe000796cd00 [linuxkpi_irq_wq] 100015 D - 0xfffffe000796cb00 [in6m_free taskq] 100016 D - 0xfffffe000796c900 [deferred_unmount ta] 100017 D - 0xfffffe000796c700 [thread taskq] 100019 D - 0xfffffe000796c300 [kqueue_ctx taskq] 100020 D - 0xfffffe000796c100 [pci_hp taskq] 100021 D - 0xfffffe000796be00 [linuxkpi_short_wq_0] 100022 D - 0xfffffe000796be00 [linuxkpi_short_wq_1] 100023 D - 0xfffffe000796be00 [linuxkpi_short_wq_2] 100024 D - 0xfffffe000796be00 [linuxkpi_short_wq_3] 100025 D - 0xfffffe000796b900 [linuxkpi_long_wq_0] 100026 D - 0xfffffe000796b900 [linuxkpi_long_wq_1] 100027 D - 0xfffffe000796b900 [linuxkpi_long_wq_2] 100028 D - 0xfffffe000796b900 [linuxkpi_long_wq_3] 100035 D - 0xfffffe0053ed0200 [firmware taskq] 100039 D - 0xfffffe0053ecfc00 [crypto_0] 100040 D - 0xfffffe0053ecfc00 [crypto_1] 100056 D - 0xfffffe0053eced00 [vtnet0 rxq 0] 100057 D - 0xfffffe0053ecec00 [vtnet0 txq 0] 100058 D - 0xfffffe0053eceb00 [vtnet0 rxq 1] 100059 D - 0xfffffe0053ecea00 [vtnet0 txq 1] 100063 D vtbslp 0xfffffe0053fdc380 [virtio_balloon] 100067 D - 0xffffffff82bbfb60 [deadlkres] 100071 D - 0xfffffe005787d100 [acpi_task_0] 100072 D - 0xfffffe005787d100 [acpi_task_1] 100073 D - 0xfffffe005787d100 [acpi_task_2] 100074 D - 0xfffffe000796e200 [mca taskq] 100076 D - 0xfffffe0053ecf500 [CAM taskq] 24707 778 778 0 Z syz-executor.1 db> show all locks Process 5 (sctp_iterator) thread 0xfffffe005786a720 (100068) exclusive sleep mutex sctp-tcb (tcb) r = 0 (0xfffffe009bb09a90) locked @ /syzkaller/managers/main/kernel/sys/netinet/sctputil.c:1523 exclusive sleep mutex sctp-inp (inp) r = 0 (0xfffffe009b7c2420) locked @ /syzkaller/managers/main/kernel/sys/netinet/sctputil.c:1463 exclusive sleep mutex sctp-it (iterator) r = 0 (0xffffffff846a0560) locked @ /syzkaller/managers/main/kernel/sys/netinet/sctputil.c:1460 shared rw sctp-info (sctp-info) r = 0 (0xfffffe0053c402f8) locked @ /syzkaller/managers/main/kernel/sys/netinet/sctputil.c:1459 db> show malloc Type InUse MemUse Requests pf_hash 5 11524K 5 pcb 4235 5275K 55474 tcp_hpts 6 4801K 6 devbuf 4217 4323K 4245 sctp_stro 2106 2106K 13375 sysctloid 35306 2080K 35377 vtbuf 24 1968K 46 sctp_atcl 4212 1580K 40982 kobj 327 1308K 488 inodedep 1564 1099K 25166 newblk 16 1028K 25973 vfscache 3 1025K 3 filedesc 76 602K 47463 ufs_quota 1 512K 1 vfs_hash 1 512K 1 callout 2 512K 2 intr 4 472K 4 subproc 206 408K 24826 dirrem 1552 388K 25052 vmem 3 274K 6 sctp_atky 6318 264K 54789 freefile 1552 194K 25016 acpica 1674 184K 56014 vnet_data 1 168K 1 tidhash 3 141K 3 linker 358 134K 386 pagedep 9 130K 24675 tfo_ccache 1 128K 1 DEVFS1 109 109K 126 sem 4 106K 4 BPF 46 88K 49 bus 991 81K 5140 mtx_pool 2 72K 2 syncache 1 68K 1 sctp_athm 4212 66K 41020 sctp_map 4212 66K 26750 module 512 64K 512 acpitask 1 64K 1 ddb_capture 1 64K 1 umtx 440 55K 440 kdtrace 284 54K 50207 sctp_timw 206 52K 206 shm 4 38K 7 temp 34 33K 3253 DEVFS3 128 32K 138 hostcache 1 32K 1 msg 4 30K 4 gtaskqueue 18 26K 18 kbdmux 6 22K 6 ifaddr 70 20K 72 DEVFS_RULE 56 20K 56 ufs_mount 4 17K 5 proc 3 17K 3 tty 16 16K 16 ithread 100 16K 100 routetbl 130 16K 410 bus-sc 34 15K 1651 cred 53 14K 284 KTRACE 101 13K 8162 ifnet 7 13K 7 ether_multi 152 13K 162 lltable 40 12K 77 kenv 95 12K 95 eventhandler 134 12K 134 ip6opt 74 11K 349 rman 88 11K 429 GEOM 61 11K 490 CAM queue 5 11K 1528 in6_multi 65 9K 65 bmsafemap 2 9K 25230 UART 12 9K 12 devstat 4 9K 4 ksem 1 8K 41 rpc 2 8K 2 shmfd 1 8K 11 pfs_vncache 1 8K 1 kqueue 93 8K 24715 pfs_nodes 20 8K 20 audit_evclass 237 8K 296 taskqueue 63 7K 63 sglist 5 7K 5 CAM DEV 3 6K 510 plimit 24 6K 440 pwddesc 90 6K 24708 ufs_dirhash 24 5K 24 UMA 272 5K 272 session 35 5K 63 pf_ifnet 10 5K 19 DEVFSP 68 5K 430 vt 11 5K 11 memdesc 1 4K 1 MCA 32 4K 32 evdev 4 4K 4 proc-args 121 4K 26048 selfd 57 4K 339282 acpisem 28 4K 28 lockf 32 4K 57 hhook 15 4K 17 kcovinfo 52 4K 52 freework 12 3K 24834 freeblks 11 3K 24831 terminal 11 3K 11 select 20 3K 65 uidinfo 3 3K 11 local_apic 1 2K 1 io_apic 1 2K 1 fpukern_ctx 2 2K 2 ipsec-saq 2 2K 2 ip6ndp 12 2K 14 sctp_ifa 13 2K 14 Unitno 27 2K 47 CAM XPT 22 2K 543 msi 12 2K 12 in_multi 6 2K 8 ipsecpolicy 2 2K 2 acpidev 20 2K 20 clone 9 2K 9 tun 7 2K 7 softdep 1 1K 1 mkdir 8 1K 49312 sahead 1 1K 1 secasvar 1 1K 1 nhops 6 1K 8 vnodemarker 2 1K 46 NFSD session 1 1K 1 CAM periph 4 1K 271 ipsec 3 1K 3 pf_osfp 7 1K 7 sctp_ifn 6 1K 14 mld 6 1K 6 igmp 6 1K 6 toponodes 6 1K 6 isadev 6 1K 6 mount 16 1K 113 pci_link 10 1K 10 crypto 4 1K 4 encap_export_host 12 1K 12 procdesc 5 1K 18 newdirblk 4 1K 24656 diradd 4 1K 25092 pfil 4 1K 4 cdev 2 1K 2 inpcbpolicy 15 1K 7163 osd 8 1K 6285 chacha20random 1 1K 1 sctp_iter 1 1K 540 sctp_cpal 1 1K 598 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 DEVFS 9 1K 10 CC Mem 4 1K 6274 vnodes 1 1K 1 CAM SIM 2 1K 2 feeder 7 1K 7 tcpfunc 3 1K 3 loginclass 3 1K 9 prison 6 1K 6 lkpikmalloc 5 1K 6 aesni_data 2 1K 2 soname 5 1K 41039 cryptodev 2 1K 279 nexusdev 8 1K 8 apmdev 1 1K 1 atkbddev 2 1K 2 CAM dev queue 2 1K 2 CAM I/O Scheduler 1 1K 1 CAM path 4 1K 1034 pmchooks 1 1K 1 filecaps 5 1K 116 sctp_vrf 1 1K 1 vnet 1 1K 1 entropy 2 1K 45 pmc 1 1K 1 acpiintr 1 1K 1 cpus 2 1K 2 vnet_data_free 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 ipcomp 0 0K 0 esp 0 0K 0 ah 0 0K 0 mqdata 0 0K 0 tcp_do 0 0K 0 tcp_fsb 0 0K 509 filemon 0 0K 154 pf_table 0 0K 0 pf_rule 0 0K 0 pf_altq 0 0K 0 pf_temp 0 0K 0 sctp_mcore 0 0K 0 sctp_socko 0 0K 19011 sctp_mvrf 0 0K 0 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 12 sctp_aadr 0 0K 0 sctp_stri 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 ixl 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 iavf 0 0K 0 axgbe 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 xen_intr 0 0K 0 NFSD srvcache 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 bounce 0 0K 0 busdma 0 0K 0 qpidrv 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 msdosfs_node 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 DEVFS4 0 0K 0 xenbus 0 0K 0 DEVFS2 0 0K 0 vm_fictitious 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xenstore 0 0K 0 scsi_pass 0 0K 0 ciss_data 0 0K 0 xnb 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 sysmouse 0 0K 0 UMAHash 0 0K 0 vtfont 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 15037 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 21 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 14 allocindir 0 0K 0 indirdep 0 0K 343 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 BACKLIGHT 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 pvscsi 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 scsi_cd 0 0K 0 ktls_ocf 0 0K 0 AHCI driver 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS_RX 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 lkpiskb 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpindev 0 0K 0 lkpifw 0 0K 0 lkpi80211 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 LRO 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 0 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 360 VN POLL 0 0K 0 agp 0 0K 0 statfs 0 0K 24847 namei_tracker 0 0K 3 export_host 0 0K 0 cl_savebuf 0 0K 5 nvme_da 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 twsbuf 0 0K 0 twe_commands 0 0K 0 tcp_log_dev 0 0K 1781 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 523 biobuf 0 0K 0 aios 0 0K 3 lio 0 0K 11 acl 0 0K 0 mbuf_tag 0 0K 134 ktls 0 0K 0 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 tempbuff 0 0K 0 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 accf 0 0K 0 pts 0 0K 0 iov 0 0K 33105 ioctlops 0 0K 3363 eventfd 0 0K 103 Witness 0 0K 0 stack 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 sbuf 0 0K 288 mpr_user 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 MPRSAS 0 0K 0 SWAP 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 sysctltmp 0 0K 844 sysctl 0 0K 3 md_sectors 0 0K 0 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 md_disk 0 0K 0 malodev 0 0K 0 LED 0 0K 0 ix_sriov 0 0K 0 cache 0 0K 0 aacraidcam 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 1 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 aacraid_buf 0 0K 0 aaccam 0 0K 0 boottrace 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 9358 548 1439432 0 254 40574976 0 sctp_asoc 2288 2106 434 13375 0 254 5811520 0 malloc-2048 2048 2109 131 28324 0 8 4587520 0 mbuf 256 11571 628 2404335 0 254 3122944 0 sctp_ep 1208 2106 434 27607 0 254 3068320 0 RADIX NODE 144 15448 451 488266 0 63 2289456 0 tcp_log 416 0 5337 80601 0 254 2220192 0 malloc-1024 1024 2107 21 13352 0 16 2179072 0 pbuf 2624 0 794 0 0 2 2083456 0 mbuf_cluster 2048 1016 0 1016 0 254 2080768 0 sctp_raddr 736 2106 435 13375 0 254 1870176 0 BUF TRIE 144 205 11583 2187 0 62 1697472 0 malloc-384 384 4212 138 41058 0 30 1670400 0 malloc-384 384 4117 53 4458 0 30 1601280 0 malloc-256 256 1787 4168 65633 0 62 1524480 0 malloc-128 128 11689 184 11923 0 126 1519744 0 UMA Slabs 0 112 13180 26 13180 0 126 1479072 0 malloc-4096 4096 327 3 691 0 2 1351680 0 vmem btag 56 23630 97 23630 0 254 1328712 0 malloc-384 384 1578 1812 25215 0 30 1301760 0 FFS inode 1160 550 493 25566 0 8 1209880 0 tcpcb 1104 4 759 6274 0 254 842352 0 VM OBJECT 264 2796 204 532096 0 30 792000 0 malloc-16384 16384 42 6 23909 0 1 786432 0 socket 960 43 721 36127 0 254 733440 0 malloc-256 256 2114 181 27619 0 62 587520 0 VNODE 448 591 552 25609 0 30 512064 0 256 Bucket 2048 229 21 7278 0 8 512000 0 malloc-128 128 1646 1888 124191 0 126 452352 0 MAP ENTRY 96 4199 463 1268022 0 126 447552 0 tcp_bbr_map 128 0 3286 61323 0 126 420608 0 malloc-4096 4096 91 7 24710 0 2 401408 0 THREAD 1808 192 28 25498 0 8 397760 0 malloc-65536 65536 4 2 340 0 1 393216 0 lkpimm 168 1 2327 1 0 62 391104 0 lkpicurr 168 2 2326 2 0 62 391104 0 ertt_txseginfo 40 0 8989 152603 0 254 359560 0 FFS2 dinode 256 550 575 25566 0 62 288000 0 malloc-64 64 3871 224 5655 0 254 262080 0 mbuf_packet 256 409 607 37079 0 254 260096 0 malloc-16 16 14613 387 18236 0 254 240000 0 malloc-256 256 316 614 26661 0 62 238080 0 sctp_stream_msg_out 112 1766 286 4221 0 254 229824 0 DEVCTL 1024 0 220 154 0 0 225280 0 VMSPACE 2552 69 12 24685 0 4 206712 0 FPU_save_area 832 194 49 30282 0 16 202176 0 UMA Zones 768 244 0 244 0 16 187392 0 malloc-32 32 5393 277 8432 0 254 181440 0 malloc-128 128 1233 162 26222 0 126 178560 0 128 Bucket 1024 106 57 6220 0 16 166912 0 malloc-64 64 2106 477 30541 0 254 165312 0 malloc-32 32 4474 692 76185 0 254 165312 0 sctp_chunk 152 917 123 959 0 254 158080 0 PROC 1352 90 24 24707 0 8 154128 0 malloc-1024 1024 125 19 1559 0 16 147456 0 malloc-128 128 954 193 2262 0 126 146816 0 S VFS Cache 104 1023 381 26798 0 126 146016 0 64 Bucket 512 159 105 13927 0 30 135168 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-65536 65536 1 1 9 0 1 131072 0 unpcb 256 20 490 1317 0 254 130560 0 filedesc0 1072 90 22 24708 0 8 120064 0 ksiginfo 112 103 941 5362 0 126 116928 0 malloc-2048 2048 4 52 535 0 8 114688 0 malloc-128 128 519 256 6368 0 126 99200 0 malloc-32768 32768 0 3 120 0 1 98304 0 malloc-8192 8192 6 6 290 0 1 98304 0 malloc-256 256 233 142 29917 0 62 96000 0 32 Bucket 256 173 202 12181 0 62 96000 0 malloc-4096 4096 18 4 656 0 2 90112 0 malloc-4096 4096 22 0 22 0 2 90112 0 UMA Kegs 384 230 3 230 0 30 89472 0 g_bio 408 0 210 9635 0 30 85680 0 pipe 744 23 92 1059 0 16 85560 0 malloc-64 64 527 796 7891 0 254 84672 0 malloc-16 16 4786 464 56351 0 254 84000 0 malloc-16 16 4214 786 64352 0 254 80000 0 malloc-8192 8192 9 0 9 0 1 73728 0 malloc-384 384 132 48 169 0 30 69120 0 malloc-64 64 744 327 26300 0 254 68544 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-32768 32768 0 2 3 0 1 65536 0 tcp_inpcb 424 4 149 6274 0 30 64872 0 ripcb 424 5 148 432 0 30 64872 0 Files 80 225 575 78859 0 126 64000 0 udplite_inpcb 424 0 126 175 0 30 53424 0 malloc-64 64 255 564 339507 0 254 52416 0 malloc-64 64 398 421 50205 0 254 52416 0 malloc-256 256 37 158 25282 0 62 49920 0 malloc-256 256 55 140 51222 0 62 49920 0 DIRHASH 1024 34 14 34 0 16 49152 0 NAMEI 1024 0 48 123672 0 16 49152 0 malloc-2048 2048 10 14 12 0 8 49152 0 malloc-1024 1024 14 34 522 0 16 49152 0 malloc-384 384 95 25 97 0 30 46080 0 tcp_bbr_pcb 832 0 54 5070 0 16 44928 0 syncache 168 0 264 4 0 254 44352 0 clpbuf 2624 0 16 25 0 16 41984 0 malloc-8192 8192 5 0 5 0 1 40960 0 malloc-4096 4096 3 7 24932 0 2 40960 0 pcpu-8 8 4701 419 4878 0 254 40960 0 sctp_readq 152 0 260 4 0 254 39520 0 Mountpoints 2752 2 12 2 0 4 38528 0 udp_inpcb 424 6 84 282 0 30 38160 0 da_ccb 544 0 70 2527 0 16 38080 0 malloc-64 64 4 563 21 0 254 36288 0 malloc-64 64 6 561 60 0 254 36288 0 8 Bucket 80 121 329 5259 0 126 36000 0 malloc-128 128 77 202 598 0 126 35712 0 malloc-128 128 2 277 156 0 126 35712 0 malloc-128 128 22 257 379 0 126 35712 0 routing nhops 256 27 108 34 0 62 34560 0 ttyoutq 256 72 63 160 0 62 34560 0 malloc-256 256 79 56 315 0 62 34560 0 malloc-256 256 57 78 2226 0 62 34560 0 TURNSTILE 136 221 31 221 0 62 34272 0 SLEEPQUEUE 88 221 163 221 0 126 33792 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-4096 4096 6 2 6 0 2 32768 0 malloc-2048 2048 3 13 1812 0 8 32768 0 malloc-2048 2048 3 13 9 0 8 32768 0 malloc-2048 2048 9 7 177 0 8 32768 0 malloc-1024 1024 4 28 17 0 16 32768 0 malloc-1024 1024 19 13 43 0 16 32768 0 malloc-512 512 0 64 44 0 30 32768 0 malloc-512 512 4 60 185 0 30 32768 0 malloc-512 512 9 55 9 0 30 32768 0 malloc-512 512 2 62 46 0 30 32768 0 malloc-512 512 10 54 14 0 30 32768 0 pcpu-64 64 492 20 492 0 254 32768 0 tcp_rack_pcb 896 0 36 509 0 16 32256 0 KNOTE 160 28 172 195154 0 62 32000 0 ttyinq 160 135 65 300 0 62 32000 0 tcp_rack_map 120 0 264 497 0 126 31680 0 cpuset 104 7 272 7 0 126 29016 0 sctp_laddr 48 0 588 12 0 254 28224 0 tcp_inpcb ports 32 2 880 5119 0 254 28224 0 ertt 72 4 388 6274 0 126 28224 0 PWD 32 50 832 23784 0 254 28224 0 malloc-32 32 304 578 24926 0 254 28224 0 16 Bucket 144 85 111 1068 0 62 28224 0 4 Bucket 48 7 581 533 0 254 28224 0 AIO 208 0 133 29 0 62 27664 0 malloc-8192 8192 2 1 4 0 1 24576 0 rtentry 176 30 108 34 0 62 24288 0 PGRP 88 35 241 63 0 126 24288 0 rl_entry 40 75 531 75 0 254 24240 0 itimer 352 0 66 8 0 30 23232 0 malloc-384 384 27 33 27 0 30 23040 0 malloc-384 384 0 60 16 0 30 23040 0 hostcache 64 1 314 1 0 254 20160 0 udpcb 32 6 624 457 0 254 20160 0 udplite_inpcb ports 32 0 630 7 0 254 20160 0 udp_inpcb ports 32 3 627 37 0 254 20160 0 malloc-32 32 13 617 44 0 254 20160 0 malloc-32 32 34 596 2972 0 254 20160 0 malloc-32 32 12 618 25 0 254 20160 0 malloc-32 32 65 565 2030 0 254 20160 0 malloc-32 32 51 579 2467 0 254 20160 0 2 Bucket 32 61 569 2477 0 254 20160 0 AIOCB 552 0 35 104 0 16 19320 0 AIOLIO 272 0 70 11 0 30 19040 0 epoch_record pcpu 256 4 60 4 0 62 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-8192 8192 2 0 2 0 1 16384 0 malloc-8192 8192 0 2 3 0 1 16384 0 malloc-2048 2048 2 6 2 0 8 16384 0 malloc-2048 2048 4 4 9 0 8 16384 0 malloc-1024 1024 8 8 8 0 16 16384 0 malloc-1024 1024 1 15 1 0 16 16384 0 malloc-512 512 0 32 7 0 30 16384 0 malloc-512 512 1 31 1 0 30 16384 0 SMR CPU 32 7 504 7 0 254 16352 0 vtnet_tx_hdr 24 0 668 735501 0 254 16032 0 malloc-16 16 27 973 31735 0 254 16000 0 kenv 258 15 45 1039 0 30 15480 0 mqnode 416 3 33 3 0 30 14976 0 vmem 1856 1 7 1 0 8 14848 0 SMR SHARED 24 7 504 7 0 254 12264 0 AIOP 32 4 374 4 0 254 12096 0 malloc-16 16 25 725 64 0 254 12000 0 malloc-16 16 8 742 120 0 254 12000 0 malloc-16 16 63 687 94 0 254 12000 0 malloc-16 16 5 745 15057 0 254 12000 0 malloc-384 384 1 29 1 0 30 11520 0 malloc-8192 8192 1 0 1 0 1 8192 0 malloc-4096 4096 1 1 1 0 2 8192 0 malloc-4096 4096 1 1 1 0 2 8192 0 pcpu-16 16 14 498 14 0 254 8192 0 UMA Slabs 1 176 9 13 9 0 62 3872 0 KMAP ENTRY 96 12 27 14 0 0 3744 0 FFS1 dinode 128 0 0 0 0 126 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 62 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 312 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 56 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 72 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 tcp_log_node 120 0 0 0 0 126 0 0 tcp_log_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 ripcb ports 32 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 NCLNODE 608 0 0 0 0 16 0 0 TMPFS node 224 0 0 0 0 62 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 cryptop 280 0 0 0 0 30 0 0 linux_dma_object 32 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 domainset 40 0 0 0 0 254 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-1024 1024 0 0 0 0 16 0 0 malloc-512 512 0 0 0 0 30 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0