panic: thread_lock() of sleep mutex `Ίκ @ /syzkaller/managers/main/kernel/sys/kern/kern_switch.c:238 cpuid = 0 time = 1622273862 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x47/frame 0xfffffe005168e2a0 vpanic() at vpanic+0x1c7/frame 0xfffffe005168e300 panic() at panic+0x43/frame 0xfffffe005168e360 thread_lock_flags_() at thread_lock_flags_/frame 0xfffffe005168e3a0 _thread_lock() at _thread_lock+0x55/frame 0xfffffe005168e3e0 critical_exit_preempt() at critical_exit_preempt+0x69/frame 0xfffffe005168e410 mb_free_ext() at mb_free_ext+0x230/frame 0xfffffe005168e450 m_free() at m_free+0x1be/frame 0xfffffe005168e490 m_freem() at m_freem+0x38/frame 0xfffffe005168e4b0 tcp_do_segment() at tcp_do_segment+0x5eb1/frame 0xfffffe005168e5b0 tcp_input_with_port() at tcp_input_with_port+0x13d5/frame 0xfffffe005168e710 tcp_input() at tcp_input+0x1f/frame 0xfffffe005168e730 ip_input() at ip_input+0x38b/frame 0xfffffe005168e7f0 netisr_dispatch_src() at netisr_dispatch_src+0x107/frame 0xfffffe005168e850 ether_demux() at ether_demux+0x288/frame 0xfffffe005168e8a0 ether_nh_input() at ether_nh_input+0x7b0/frame 0xfffffe005168e920 netisr_dispatch_src() at netisr_dispatch_src+0x107/frame 0xfffffe005168e980 ether_input() at ether_input+0xce/frame 0xfffffe005168e9f0 vtnet_rxq_eof() at vtnet_rxq_eof+0xf4d/frame 0xfffffe005168ead0 vtnet_rx_vq_process() at vtnet_rx_vq_process+0xe1/frame 0xfffffe005168eb10 ithread_loop() at ithread_loop+0x33f/frame 0xfffffe005168ebb0 fork_exit() at fork_exit+0xb3/frame 0xfffffe005168ebf0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe005168ebf0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- KDB: enter: panic [ thread pid 1572864 tid 367606664 ] Stopped at kdb_enter+0x67: movq $0,0x1636a5e(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0x28 rax 0x12 rcx 0x80 rdx 0xffffffff819c9b1f rbx 0 rsp 0xfffffe005168e280 rbp 0xfffffe005168e2a0 rsi 0x1 rdi 0 r8 0 r9 0x8080808080808080 r10 0xfffffe005168e170 r11 0x1ffaefff59c r12 0xffffffff82267b90 ddb_dbbe r13 0 r14 0xffffffff81a7b78f r15 0xffffffff81a7b78f rip 0xffffffff81132727 kdb_enter+0x67 rflags 0x82 kdb_enter+0x67: movq $0,0x1636a5e(%rip) db> show proc Process 1572864 () at 0xffffffff8272eb60: state: NEW uid: 71532544 gids: KDB: reentering KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x47/frame 0xfffffe005168dc10 kdb_reenter() at kdb_reenter+0x4d/frame 0xfffffe005168dc30 trap() at trap+0xc6/frame 0xfffffe005168dd50 calltrap() at calltrap+0x8/frame 0xfffffe005168dd50 --- trap 0x9, rip = 0xffffffff8059026a, rsp = 0xfffffe005168de20, rbp = 0xfffffe005168de50 --- db_show_proc() at db_show_proc+0x1ba/frame 0xfffffe005168de50 db_command() at db_command+0x5e0/frame 0xfffffe005168df50 db_command_loop() at db_command_loop+0x82/frame 0xfffffe005168df70 db_trap() at db_trap+0x15d/frame 0xfffffe005168e010 kdb_trap() at kdb_trap+0x198/frame 0xfffffe005168e090 trap() at trap+0x30b/frame 0xfffffe005168e1b0 calltrap() at calltrap+0x8/frame 0xfffffe005168e1b0 --- trap 0x3, rip = 0xffffffff81132727, rsp = 0xfffffe005168e280, rbp = 0xfffffe005168e2a0 --- kdb_enter() at kdb_enter+0x67/frame 0xfffffe005168e2a0 vpanic() at vpanic+0x221/frame 0xfffffe005168e300 panic() at panic+0x43/frame 0xfffffe005168e360 thread_lock_flags_() at thread_lock_flags_/frame 0xfffffe005168e3a0 _thread_lock() at _thread_lock+0x55/frame 0xfffffe005168e3e0 critical_exit_preempt() at critical_exit_preempt+0x69/frame 0xfffffe005168e410 mb_free_ext() at mb_free_ext+0x230/frame 0xfffffe005168e450 m_free() at m_free+0x1be/frame 0xfffffe005168e490 m_freem() at m_freem+0x38/frame 0xfffffe005168e4b0 tcp_do_segment() at tcp_do_segment+0x5eb1/frame 0xfffffe005168e5b0 tcp_input_with_port() at tcp_input_with_port+0x13d5/frame 0xfffffe005168e710 tcp_input() at tcp_input+0x1f/frame 0xfffffe005168e730 ip_input() at ip_input+0x38b/frame 0xfffffe005168e7f0 netisr_dispatch_src() at netisr_dispatch_src+0x107/frame 0xfffffe005168e850 ether_demux() at ether_demux+0x288/frame 0xfffffe005168e8a0 ether_nh_input() at ether_nh_input+0x7b0/frame 0xfffffe005168e920 netisr_dispatch_src() at netisr_dispatch_src+0x107/frame 0xfffffe005168e980 ether_input() at ether_input+0xce/frame 0xfffffe005168e9f0 vtnet_rxq_eof() at vtnet_rxq_eof+0xf4d/frame 0xfffffe005168ead0 vtnet_rx_vq_process() at vtnet_rx_vq_process+0xe1/frame 0xfffffe005168eb10 ithread_loop() at ithread_loop+0x33f/frame 0xfffffe005168ebb0 fork_exit() at fork_exit+0xb3/frame 0xfffffe005168ebf0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe005168ebf0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- db> ps pid ppid pgrp uid state wmesg wchan cmd 3394 786 784 0 T (threaded) syz-executor6207262 100108 s syz-executor6207262 102723 Run CPU 1 syz-executor6207262 786 784 784 0 S nanslp 0xffffffff8273c8e1 syz-executor6207262 784 782 784 0 Ss pause 0xfffff80015eec5e0 csh 782 694 782 0 Ss select 0xfffff8001571d740 sshd 760 1 760 0 Ss+ ttyin 0xfffff800154a3cb0 getty 759 1 759 0 Ss+ ttyin 0xfffff80015b2d4b0 getty 758 1 758 0 Ss+ ttyin 0xfffff80015b2dcb0 getty 757 1 757 0 Ss+ ttyin 0xfffff80015b314b0 getty 756 1 756 0 Ss+ ttyin 0xfffff80015b31cb0 getty 755 1 755 0 Ss+ ttyin 0xfffff80015ab74b0 getty 754 1 754 0 Ss+ ttyin 0xfffff80015ab7cb0 getty 753 1 753 0 Ss+ ttyin 0xfffff80015abd4b0 getty 752 1 752 0 Ss+ ttyin 0xfffff80015abdcb0 getty 698 1 698 0 Ss nanslp 0xffffffff8273c8e0 cron 694 1 694 0 Ss select 0xfffff8004931c140 sshd 507 1 507 0 Ss select 0xfffff80015d642c0 syslogd 436 1 436 0 Ss select 0xfffff8004931c2c0 devd 435 1 435 65 Ss select 0xfffff80015d752c0 dhclient 350 1 350 0 Ss select 0xfffff8004927a8c0 dhclient 347 1 347 0 Ss select 0xfffff8004931c440 dhclient 23 0 0 0 DL vlruwt 0xfffff80015cd1530 [vnlru] 22 0 0 0 DL syncer 0xffffffff8282bcd0 [syncer] 21 0 0 0 DL (threaded) [bufdaemon] 100081 D qsleep 0xffffffff8282ada0 [bufdaemon] 100088 D - 0xffffffff8220ae80 [bufspacedaemon-0] 100098 D sdflush 0xfffff80015ed4ce8 [/ worker] 20 0 0 0 DL psleep 0xffffffff82852c08 [vmdaemon] 19 0 0 0 DL (threaded) [pagedaemon] 100079 D psleep 0xffffffff82847078 [dom0] 100086 D launds 0xffffffff82847084 [laundry: dom0] 100087 D umarcl 0xffffffff815cfbe0 [uma] 18 0 0 0 DL - 0xffffffff82570c88 [rand_harvestq] 17 0 0 0 DL waiting 0xffffffff830e3828 [sctp_iterator] 16 0 0 0 DL pftm 0xffffffff82ec83c0 [pf purge] 15 0 0 0 DL - 0xffffffff8282845c [soaiod4] 9 0 0 0 DL - 0xffffffff8282845c [soaiod3] 8 0 0 0 DL - 0xffffffff8282845c [soaiod2] 7 0 0 0 DL - 0xffffffff8282845c [soaiod1] 6 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff82448140 [doneq0] 100045 D - 0xffffffff824480c0 [async] 100078 D - 0xffffffff82447f90 [scanner] 14 0 0 0 DL seqstat 0xfffff80004dec488 [sequencer 00] 5 0 0 0 DL crypto_ 0xfffff80004db6f80 [crypto returns 1] 4 0 0 0 DL crypto_ 0xfffff80004db6f30 [crypto returns 0] 3 0 0 0 DL crypto_ 0xffffffff82844560 [crypto] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff8271c120 [g_event] 100036 D - 0xffffffff8271c128 [g_up] 100037 D - 0xffffffff8271c130 [g_down] 2 0 0 0 DL (threaded) [KTLS] 100028 D - 0xfffff80004440900 [thr_0] 100029 D - 0xfffff80004440980 [thr_1] 12 0 0 0 RL (threaded) [intr] 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 I [swi3: vm] 100031 I [swi4: clock (0)] 100032 I [swi4: clock (1)] 100033 I [swi1: netisr 0] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 Run CPU 0 [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100084 I [swi1: hpts] 100085 I [swi1: hpts] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffff80004bd7530 [init] 10 0 0 0 DL audit_w 0xffffffff82844a70 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff8271c6b0 [swapper] 100005 D - 0xfffff80004c76d00 [if_io_tqg_0] 100006 D - 0xfffff80004c76900 [if_io_tqg_1] 100007 D - 0xfffff80004c76500 [if_config_tqg_0] 100008 D - 0xfffff80004c76100 [softirq_0] 100009 D - 0xfffff80004c73d00 [softirq_1] 100010 D - 0xfffff80004c72d00 [aiod_kick taskq] 100012 D - 0xfffff80004c72500 [kqueue_ctx taskq] 100013 D - 0xfffff80004c72100 [pci_hp taskq] 100015 D - 0xfffff80004c6a900 [inm_free taskq] 100017 D - 0xfffff80004c6a100 [linuxkpi_irq_wq] 100018 D - 0xfffff80004c65d00 [thread taskq] 100019 D - 0xfffff80004c65900 [in6m_free taskq] 100020 D - 0xfffff80004c65500 [linuxkpi_short_wq_0] 100021 D - 0xfffff80004c65500 [linuxkpi_short_wq_1] 100022 D - 0xfffff80004c65500 [linuxkpi_short_wq_2] 100023 D - 0xfffff80004c65500 [linuxkpi_short_wq_3] 100024 D - 0xfffff80004c65100 [linuxkpi_long_wq_0] 100025 D - 0xfffff80004c65100 [linuxkpi_long_wq_1] 100026 D - 0xfffff80004c65100 [linuxkpi_long_wq_2] 100027 D - 0xfffff80004c65100 [linuxkpi_long_wq_3] 100034 D - 0xfffff80004cccd00 [firmware taskq] 100038 D - 0xfffff80004ccc900 [crypto_0] 100039 D - 0xfffff80004ccc900 [crypto_1] 100055 D - 0xfffff80004ccc100 [vtnet0 rxq 0] 100056 D - 0xfffff800153e1d00 [vtnet0 txq 0] 100057 D - 0xfffff800153e1900 [vtnet0 rxq 1] 100058 D - 0xfffff800153e1500 [vtnet0 txq 1] 100060 D vtbslp 0xfffff8001545eb00 [virtio_balloon] 100064 D - 0xfffff800153e1100 [mca taskq] 100066 D - 0xffffffff81e28400 [deadlkres] 100074 D - 0xfffff80015ac4d00 [acpi_task_0] 100075 D - 0xfffff80015ac4d00 [acpi_task_1] 100076 D - 0xfffff80015ac4d00 [acpi_task_2] 100077 D - 0xfffff80004ccc500 [CAM taskq] db> show all locks Process 12 (intr) thread 0xfffffe0051980740 (100053) exclusive rw tcpinp (tcpinp) r = 0 (0xfffff800493213f0) locked @ /syzkaller/managers/main/kernel/sys/netinet/in_pcb.c:2524 exclusive sleep mutex vtnet0-rx1 (vtnet0-rx1) r = 0 (0xfffff80004ec7540) locked @ /syzkaller/managers/main/kernel/sys/dev/virtio/network/if_vtnet.c:2181 db> show malloc Type InUse MemUse Requests sysctloid 34288 12858K 34355 pf_hash 5 11560K 5 devbuf 4216 6982K 4244 tcp_hpts 5 3219K 5 kobj 328 2624K 488 vtbuf 24 2064K 46 newblk 541 1302K 602 vfscache 3 1035K 3 acpica 1674 649K 54692 pcb 23 612K 77 inodedep 9 528K 71 callout 2 528K 2 ufs_quota 1 520K 1 vfs_hash 1 520K 1 intr 4 480K 4 subproc 96 404K 3447 bus 990 378K 3485 linker 348 270K 397 DEVFS1 103 206K 112 module 512 192K 512 vnet_data 1 176K 1 tidhash 3 164K 3 pagedep 3 137K 18 tfo_ccache 1 136K 1 kdtrace 161 134K 6119 sem 4 120K 4 umtx 264 116K 264 UMA 270 102K 270 audit_evclass 236 89K 294 mtx_pool 2 80K 2 syncache 1 76K 1 temp 18 71K 1622 msg 4 68K 4 BPF 10 68K 10 acpitask 1 64K 1 ddb_capture 1 64K 1 DEVFS3 122 61K 132 gtaskqueue 18 57K 18 vmem 3 56K 4 DEVFS_RULE 56 54K 56 kenv 95 52K 95 eventhandler 133 50K 133 ithread 99 43K 99 rman 84 42K 425 ifaddr 30 40K 32 KTRACE 100 38K 100 taskqueue 60 36K 60 proc 3 34K 3 ufs_mount 5 34K 6 bus-sc 33 34K 1696 routetbl 50 34K 176 devstat 4 33K 4 hostcache 1 32K 1 tty 16 32K 16 shm 1 32K 1 GEOM 60 29K 489 kbdmux 6 28K 6 cred 22 22K 234 CAM queue 5 21K 1528 pfs_nodes 20 20K 20 UART 12 18K 12 kqueue 44 17K 3397 pwddesc 44 17K 3395 ksem 1 16K 1 rpc 2 16K 2 bmsafemap 1 16K 41 shmfd 1 16K 1 plimit 16 16K 323 pfs_vncache 1 16K 1 ether_multi 40 15K 50 ufs_dirhash 24 14K 24 proc-args 36 14K 491 sglist 5 13K 5 MCA 32 12K 32 CAM DEV 3 12K 510 vt 11 11K 11 in6_multi 25 11K 25 acpisem 28 11K 28 CAM XPT 22 11K 543 selfd 27 11K 43068 Unitno 27 11K 39 session 19 10K 31 lltable 11 9K 11 uidinfo 3 9K 8 ifnet 3 9K 3 memdesc 1 8K 1 ipsec-saq 2 8K 2 evdev 4 8K 4 filedesc 1 8K 1 acpidev 20 8K 20 hhook 15 8K 17 mount 16 7K 90 pf_ifnet 5 6K 6 fpukern_ctx 3 6K 3 lockf 15 6K 22 terminal 11 6K 11 inpcbpolicy 14 6K 137 ipsecpolicy 2 5K 2 encap_export_host 12 5K 12 clone 9 5K 9 osd 11 5K 19 local_apic 1 4K 1 io_apic 1 4K 1 sahead 1 4K 1 secasvar 1 4K 1 pci_link 10 4K 10 msi 9 4K 9 DEVFS 9 4K 10 ipsec 3 3K 3 nhops 6 3K 6 nexusdev 7 3K 7 diradd 7 3K 36 feeder 7 3K 7 select 7 3K 29 toponodes 6 3K 6 prison 6 3K 6 isadev 6 3K 6 softdep 1 2K 1 vnodemarker 2 2K 14 NFSD session 1 2K 1 sctp_ifa 5 2K 6 linux 5 2K 6 CAM periph 4 2K 271 crypto 4 2K 4 ip6ndp 4 2K 5 DEVFSP 4 2K 9 indirdep 3 2K 3 pfil 4 2K 4 CAM path 4 2K 1034 soname 4 2K 3231 tcpfunc 3 2K 3 tun 3 2K 3 loginclass 3 2K 7 in_multi 2 1K 4 chacha20random 1 1K 1 vnodes 1 1K 1 CAM SIM 2 1K 2 ktls 1 1K 1 cdev 2 1K 2 aesni_data 2 1K 2 sctp_ifn 2 1K 6 cpus 2 1K 2 atkbddev 2 1K 2 CAM dev queue 2 1K 2 xform 2 1K 49 mld 2 1K 2 igmp 2 1K 2 entropy 2 1K 35 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 procdesc 1 1K 6 pmchooks 1 1K 1 sctp_vrf 1 1K 1 apmdev 1 1K 1 freework 1 1K 26 CAM I/O Scheduler 1 1K 1 vnet_data_free 1 1K 1 vnet 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 acpiintr 1 1K 1 pmc 1 1K 1 filecaps 1 1K 66 sctp_mcore 0 0K 0 sctp_socko 0 0K 0 sctp_iter 0 0K 3 sctp_mvrf 0 0K 0 sctp_timw 0 0K 0 sctp_cpal 0 0K 0 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_athm 0 0K 0 sctp_atky 0 0K 0 sctp_atcl 0 0K 0 sctp_a_it 0 0K 3 sctp_aadr 0 0K 0 sctp_stro 0 0K 0 sctp_stri 0 0K 0 sctp_map 0 0K 0 cdg data 0 0K 0 chd data 0 0K 0 dctcp data 0 0K 0 cubic data 0 0K 0 vegas data 0 0K 0 htcp data 0 0K 0 pf_table 0 0K 0 pf_rule 0 0K 0 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_temp 0 0K 0 tcp_do 0 0K 0 tcp_fsb 0 0K 0 mqdata 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 iavf 0 0K 0 ixl 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 axgbe 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xen_intr 0 0K 0 xenstore 0 0K 0 ciss_data 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 qpidrv 0 0K 0 BACKLIGHT 0 0K 0 xnb 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 sysmouse 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 vtfont 0 0K 0 vm_fictitious 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 amr 0 0K 0 pvscsi 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 UMAHash 0 0K 0 scsi_cd 0 0K 0 AHCI driver 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 18 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 4 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 newdirblk 0 0K 8 dirrem 0 0K 28 mkdir 0 0K 16 freefile 0 0K 26 freeblks 0 0K 25 freefrag 0 0K 7 allocindir 0 0K 0 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 agp 0 0K 0 nvme_da 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 ktls_ocf 0 0K 0 twsbuf 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpifw 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 3 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 twe_commands 0 0K 0 LRO 0 0K 0 newreno data 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 0 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 0 VN POLL 0 0K 0 twa_commands 0 0K 0 statfs 0 0K 197 namei_tracker 0 0K 0 export_host 0 0K 0 cl_savebuf 0 0K 6 tcp_log_dev 0 0K 0 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 1841 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 biobuf 0 0K 0 aios 0 0K 0 lio 0 0K 0 acl 0 0K 0 tempbuff 0 0K 0 mbuf_tag 0 0K 27 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 accf 0 0K 0 pts 0 0K 0 iov 0 0K 13523 ioctlops 0 0K 86 eventfd 0 0K 0 Witness 0 0K 0 stack 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 sbuf 0 0K 288 md_sectors 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 md_disk 0 0K 0 SWAP 0 0K 0 malodev 0 0K 0 LED 0 0K 0 sysctltmp 0 0K 618 sysctl 0 0K 3 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 ix_sriov 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 cache 0 0K 0 iirbuf 0 0K 0 kcovinfo 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 1 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8320 703 14642 0 254 36958208 0 malloc-384 384 34476 74 80966 0 30 13267200 0 malloc-1024 1024 4178 2 4598 0 16 4280320 0 malloc-8192 8192 328 1 488 0 1 2695168 0 pbuf 2624 0 989 0 0 2 2595136 0 mbuf 256 8577 768 22055 0 254 2392320 0 BUF TRIE 144 168 13300 480 0 62 1939392 0 UMA Slabs 0 112 11181 12 11181 0 126 1253616 0 malloc-384 384 2023 27 54212 0 30 787200 0 malloc-384 384 1714 36 5081 0 30 672000 0 FFS inode 1160 483 28 510 0 8 592760 0 malloc-512 512 984 24 1799 0 30 516096 0 malloc-8192 8192 48 5 3399 0 1 434176 0 malloc-65536 65536 4 2 181 0 1 393216 0 lkpimm 160 1 2324 1 0 62 372000 0 lkpicurr 160 2 2323 2 0 62 372000 0 RADIX NODE 144 2075 190 54209 0 62 326160 0 malloc-16384 16384 14 2 276 0 1 262144 0 malloc-2048 2048 115 13 1306 0 8 262144 0 VM OBJECT 264 852 93 33594 0 30 249480 0 VNODE 448 513 36 542 0 30 245952 0 THREAD 1808 116 16 2723 0 8 238656 0 malloc-384 384 570 20 17142 0 30 226560 0 DEVCTL 1024 0 216 116 0 0 221184 0 UMA Zones 768 242 2 242 0 16 187392 0 malloc-384 384 442 38 6049 0 30 184320 0 malloc-1024 1024 140 32 2959 0 16 176128 0 malloc-4096 4096 5 36 2609 0 2 167936 0 256 Bucket 2048 60 16 9856 0 8 155648 0 vmem btag 56 2338 98 2338 0 254 136416 0 FFS2 dinode 256 483 42 509 0 62 134400 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-16384 16384 8 0 8 0 1 131072 0 ksiginfo 112 34 1010 57 0 126 116928 0 malloc-8192 8192 12 2 24 0 1 114688 0 MAP ENTRY 96 772 404 66810 0 126 112896 0 S VFS Cache 104 970 83 1011 0 126 109512 0 malloc-32768 32768 2 1 147 0 1 98304 0 UMA Kegs 384 227 6 227 0 30 89472 0 malloc-1024 1024 37 47 101 0 16 86016 0 malloc-384 384 205 15 219 0 30 84480 0 malloc-1024 1024 63 17 82 0 16 81920 0 VMSPACE 2544 21 9 3373 0 4 76320 0 PROC 1328 43 8 3394 0 8 67728 0 filedesc0 1072 44 19 3395 0 8 67536 0 mbuf_cluster 2048 30 2 30 0 254 65536 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-32768 32768 2 0 2 0 1 65536 0 malloc-32768 32768 2 0 2 0 1 65536 0 malloc-16384 16384 3 1 10 0 1 65536 0 malloc-16384 16384 4 0 4 0 1 65536 0 malloc-8192 8192 8 0 9 0 1 65536 0 malloc-2048 2048 15 17 523 0 8 65536 0 malloc-1024 1024 56 4 58 0 16 61440 0 g_bio 408 0 150 4817 0 30 61200 0 malloc-384 384 112 38 3831 0 30 57600 0 malloc-8192 8192 5 2 643 0 1 57344 0 128 Bucket 1024 22 33 236 0 16 56320 0 32 Bucket 256 64 131 32919 0 62 49920 0 malloc-4096 4096 10 2 201 0 2 49152 0 malloc-4096 4096 5 6 77 0 2 45056 0 malloc-2048 2048 21 1 21 0 8 45056 0 clpbuf 2624 0 16 21 0 16 41984 0 DIRHASH 1024 34 6 34 0 16 40960 0 malloc-1024 1024 30 10 940 0 16 40960 0 malloc-512 512 47 33 115 0 30 40960 0 NAMEI 1024 0 36 12031 0 16 36864 0 pcpu-8 8 4220 388 4248 0 254 36864 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-16384 16384 2 0 2 0 1 32768 0 malloc-16384 16384 2 0 2 0 1 32768 0 malloc-4096 4096 5 3 15 0 2 32768 0 malloc-512 512 40 24 492 0 30 32768 0 malloc-512 512 21 43 337 0 30 32768 0 pcpu-64 64 487 25 487 0 254 32768 0 64 Bucket 512 39 25 2404 0 30 32768 0 malloc-512 512 10 46 265 0 30 28672 0 socket 944 22 6 1253 0 254 26432 0 malloc-8192 8192 1 2 198 0 1 24576 0 malloc-4096 4096 6 0 6 0 2 24576 0 malloc-1024 1024 12 12 60 0 16 24576 0 ertt_txseginfo 40 0 606 2817 0 254 24240 0 16 Bucket 144 40 128 1610 0 62 24192 0 ttyinq 160 135 15 300 0 62 24000 0 ttyoutq 256 72 18 160 0 62 23040 0 TURNSTILE 136 133 35 133 0 62 22848 0 2 Bucket 32 80 550 1205 0 254 20160 0 Mountpoints 2752 2 5 2 0 4 19264 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 4 Bucket 48 6 330 357 0 254 16128 0 vtnet_tx_hdr 24 0 668 3974 0 254 16032 0 tcpcb 1064 7 7 7 0 254 14896 0 SLEEPQUEUE 88 133 27 133 0 126 14080 0 malloc-4096 4096 3 0 3 0 2 12288 0 malloc-2048 2048 2 4 10 0 8 12288 0 malloc-1024 1024 8 4 10 0 16 12288 0 malloc-512 512 15 9 58 0 30 12288 0 malloc-512 512 21 3 924 0 30 12288 0 udp_inpcb 488 6 18 126 0 254 11712 0 kenv 258 15 30 1032 0 30 11610 0 malloc-384 384 23 7 27 0 30 11520 0 pipe 744 6 9 284 0 16 11160 0 malloc-8192 8192 1 0 1 0 1 8192 0 malloc-8192 8192 1 0 1 0 1 8192 0 malloc-4096 4096 0 2 16 0 2 8192 0 malloc-4096 4096 2 0 2 0 2 8192 0 malloc-2048 2048 3 1 3 0 8 8192 0 rtentry 176 13 33 17 0 62 8096 0 PGRP 88 19 73 31 0 126 8096 0 rl_entry 40 27 175 27 0 254 8080 0 sctp_laddr 48 0 168 4 0 254 8064 0 udpcb 32 6 246 126 0 254 8064 0 ertt 72 7 105 7 0 126 8064 0 PWD 32 10 242 100 0 254 8064 0 Files 80 67 33 6516 0 126 8000 0 8 Bucket 80 37 63 963 0 126 8000 0 ripcb 488 1 15 4 0 254 7808 0 tcp_inpcb 488 7 9 7 0 254 7808 0 routing nhops 256 10 20 17 0 62 7680 0 unpcb 256 7 23 1099 0 254 7680 0 mbuf_packet 256 0 30 93 0 254 7680 0 FPU_save_area 832 1 8 1 0 16 7488 0 cpuset 104 7 55 7 0 126 6448 0 epoch_record pcpu 256 4 12 4 0 62 4096 0 malloc-2048 2048 2 0 2 0 8 4096 0 malloc-2048 2048 2 0 2 0 8 4096 0 malloc-512 512 3 5 3 0 30 4096 0 pcpu-16 16 7 249 7 0 254 4096 0 syncache 168 0 24 5 0 254 4032 0 KNOTE 160 0 25 8 0 62 4000 0 UMA Slabs 1 176 8 14 8 0 62 3872 0 mqnode 416 3 6 3 0 30 3744 0 KMAP ENTRY 96 12 27 12 0 0 3744 0 vmem 1856 1 1 1 0 8 3712 0 SMR CPU 32 3 60 3 0 254 2016 0 SMR SHARED 24 3 60 3 0 254 1512 0 FFS1 dinode 128 0 0 0 0 126 0 0 da_ccb 544 0 0 0 0 16 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 sctp_stream_msg_out 112 0 0 0 0 254 0 0 sctp_readq 152 0 0 0 0 254 0 0 sctp_chunk 152 0 0 0 0 254 0 0 sctp_raddr 736 0 0 0 0 254 0 0 sctp_asoc 2288 0 0 0 0 254 0 0 sctp_ep 1280 0 0 0 0 254 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 62 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 304 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 48 0 0 0 0 254 0 0 tcp_rack_pcb 832 0 0 0 0 16 0 0 tcp_rack_map 112 0 0 0 0 126 0 0 tcp_bbr_pcb 832 0 0 0 0 16 0 0 tcp_bbr_map 128 0 0 0 0 126 0 0 udplite_inpcb 488 0 0 0 0 254 0 0 tcp_log_node 120 0 0 0 0 126 0 0 tcp_log_bucket 176 0 0 0 0 62 0 0 tcp_log 416 0 0 0 0 254 0 0 tcpreass 48 0 0 0 0 254 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 hostcache 64 0 0 0 0 254 0 0 tcptw 88 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 itimer 352 0 0 0 0 30 0 0 AIOLIO 272 0 0 0 0 30 0 0 AIOCB 552 0 0 0 0 16 0 0 AIOP 32 0 0 0 0 254 0 0 AIO 208 0 0 0 0 62 0 0 TMPFS node 224 0 0 0 0 62 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 NCLNODE 584 0 0 0 0 16 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 cryptop 280 0 0 0 0 30 0 0 linux_dma_object 24 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 ktls_session 128 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 domainset 40 0 0 0 0 254 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-2048 2048 0 0 0 0 8 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0