panic: pool_do_get: mbufpl free list modified: page 0xfffffd806f0a8000; item addr 0xfffffd806f0a8300; offset 0x0=0x0 != 0x3c51b1fad6aceb8e Stopped at db_enter+0x18: addq $0x8,%rsp TID PID UID PRFLAGS PFLAGS CPU COMMAND db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic() at panic+0x15c sys/kern/subr_prf.c:207 pool_do_get(ffffffff8263e270,2,ffff800020a1fae8) at pool_do_get+0x439 sys/kern/subr_pool.c:746 pool_get() at pool_get+0xeb sys/kern/subr_pool.c:581 m_clget(0,2,800) at m_clget+0x1c9 m_gethdr sys/kern/uipc_mbuf.c:283 [inline] m_clget(0,2,800) at m_clget+0x1c9 sys/kern/uipc_mbuf.c:400 vio_populate_rx_mbufs(ffff800000173000) at vio_populate_rx_mbufs+0xf9 vio_add_rx_mbuf sys/dev/pv/if_vio.c:908 [inline] vio_populate_rx_mbufs(ffff800000173000) at vio_populate_rx_mbufs+0xf9 sys/dev/pv/if_vio.c:951 vio_rx_intr(ffff800000173050) at vio_rx_intr+0x69 intr_handler(ffff800020a1fce0,ffff80000024db00) at intr_handler+0x8f sys/arch/amd64/amd64/intr.c:529 Xintr_ioapic_edge19_untramp(0,0,1388,0,0,ffff80000001dae0) at Xintr_ioapic_edge19_untramp+0x19f acpicpu_idle() at acpicpu_idle+0x331 sys/dev/acpi/acpicpu.c:1187 sched_idle(ffffffff8250aff0) at sched_idle+0x3f7 sys/kern/kern_sched.c:181 end trace frame: 0x0, count: 4 https://www.openbsd.org/ddb.html describes the minimum info required in bug reports. Insufficient info makes it difficult to find and fix bugs. ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic pool_do_get: mbufpl free list modified: page 0xfffffd806f0a8000; item addr 0xfffffd806f0a8300; offset 0x0=0x0 != 0x3c51b1fad6aceb8e ddb{0}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic() at panic+0x15c sys/kern/subr_prf.c:207 pool_do_get(ffffffff8263e270,2,ffff800020a1fae8) at pool_do_get+0x439 sys/kern/subr_pool.c:746 pool_get() at pool_get+0xeb sys/kern/subr_pool.c:581 m_clget(0,2,800) at m_clget+0x1c9 m_gethdr sys/kern/uipc_mbuf.c:283 [inline] m_clget(0,2,800) at m_clget+0x1c9 sys/kern/uipc_mbuf.c:400 vio_populate_rx_mbufs(ffff800000173000) at vio_populate_rx_mbufs+0xf9 vio_add_rx_mbuf sys/dev/pv/if_vio.c:908 [inline] vio_populate_rx_mbufs(ffff800000173000) at vio_populate_rx_mbufs+0xf9 sys/dev/pv/if_vio.c:951 vio_rx_intr(ffff800000173050) at vio_rx_intr+0x69 intr_handler(ffff800020a1fce0,ffff80000024db00) at intr_handler+0x8f sys/arch/amd64/amd64/intr.c:529 Xintr_ioapic_edge19_untramp(0,0,1388,0,0,ffff80000001dae0) at Xintr_ioapic_edge19_untramp+0x19f acpicpu_idle() at acpicpu_idle+0x331 sys/dev/acpi/acpicpu.c:1187 sched_idle(ffffffff8250aff0) at sched_idle+0x3f7 sys/kern/kern_sched.c:181 end trace frame: 0x0, count: -11 ddb{0}> show registers rdi 0 rsi 0x1 rbp 0xffff800020a1f930 rbx 0xffff800020a1f9e0 rdx 0xffffffff8250aff0 cpu_info_full_primary+0x1ff0 rcx 0xffffffff8250aff0 cpu_info_full_primary+0x1ff0 rax 0xffffffff8250aff0 cpu_info_full_primary+0x1ff0 r8 0xffffffff812fd86f kprintf+0x16f r9 0x1 r10 0x25 r11 0x2808596715e0214f r12 0x3000000008 r13 0xffff800020a1f940 r14 0x100 r15 0x1 rip 0xffffffff81c098c8 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff800020a1f920 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb{0}> show proc PROC (idle0) pid=259617 stat=onproc flags process=14000 proc=40000200 pri=0, usrpri=50, nice=20 forw=0x929fd840932ae0ba, list=0xffff800020a10ed0,0xffff800020a10010 process=0xffff800020a12000 user=0xffff800020a1a000, vmspace=0xffffffff826421b0 estcpu=0, cpticks=32564, pctcpu=0.0 user=0, sys=0, intr=1 ddb{0}> ps PID TID PPID UID S FLAGS WAIT COMMAND 59052 88550 31850 0 3 0x80 nanosleep syz-executor.1 59052 217026 31850 0 3 0x4000080 nanosleep syz-executor.1 61302 374682 54076 0 3 0x2 biowait syz-executor.0 31850 200696 54076 0 3 0x82 nanosleep syz-executor.1 2391 30835 0 0 3 0x14200 acct acct 7743 497256 0 0 3 0x14200 bored sosplice 54076 161386 15544 0 3 0x82 nanosleep syz-fuzzer 54076 232349 15544 0 3 0x4000082 nanosleep syz-fuzzer 54076 378014 15544 0 3 0x4000082 thrsleep syz-fuzzer 54076 73196 15544 0 3 0x4000082 thrsleep syz-fuzzer 54076 424038 15544 0 3 0x4000082 thrsleep syz-fuzzer 54076 101839 15544 0 3 0x4000082 thrsleep syz-fuzzer 54076 413898 15544 0 3 0x4000082 thrsleep syz-fuzzer 54076 29599 15544 0 3 0x4000082 thrsleep syz-fuzzer 54076 33186 15544 0 3 0x4000082 thrsleep syz-fuzzer 54076 261932 15544 0 3 0x4000082 kqread syz-fuzzer 15544 418443 60379 0 3 0x10008a pause ksh 60379 491206 97708 0 3 0x92 select sshd 8406 346672 1 0 3 0x100083 ttyin getty 97708 513486 1 0 3 0x80 select sshd 936 401878 7317 74 3 0x100092 bpf pflogd 7317 323918 1 0 3 0x80 netio pflogd 91246 386890 14048 73 3 0x100090 kqread syslogd 14048 271083 1 0 3 0x100082 netio syslogd 35197 175237 1 77 3 0x100090 poll dhclient 70491 87713 1 0 3 0x80 poll dhclient 68816 482833 0 0 3 0x14200 pgzero zerothread 56403 300952 0 0 3 0x14200 aiodoned aiodoned 91547 463040 0 0 3 0x14200 syncer update 83017 148780 0 0 3 0x14200 cleaner cleaner 29403 282563 0 0 3 0x14200 reaper reaper 76883 487398 0 0 3 0x14200 pgdaemon pagedaemon 40208 430189 0 0 3 0x14200 bored crynlk 19827 274561 0 0 3 0x14200 bored crypto 6568 184229 0 0 3 0x40014200 acpi0 acpi0 62331 440837 0 0 7 0x40014200 idle1 96781 444379 0 0 2 0x14200 softnet 75554 457579 0 0 3 0x14200 bored systqmp 53674 92529 0 0 3 0x14200 bored systq 47862 497218 0 0 3 0x40014200 bored softclock *13774 259617 0 0 7 0x40014200 idle0 96142 26931 0 0 3 0x14200 bored smr 1 57591 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{0}> show all locks CPU 0: exclusive mutex mbufpl r = 0 (0xffffffff8263e280) #0 witness_lock+0x52e sys/kern/subr_witness.c:1163 #1 mtx_enter_try+0x102 #2 mtx_enter+0x4b sys/kern/kern_lock.c:266 #3 pool_get+0xbf sys/kern/subr_pool.c:578 #4 m_clget+0x1c9 m_gethdr sys/kern/uipc_mbuf.c:283 [inline] #4 m_clget+0x1c9 sys/kern/uipc_mbuf.c:400 #5 vio_populate_rx_mbufs+0xf9 vio_add_rx_mbuf sys/dev/pv/if_vio.c:908 [inline] #5 vio_populate_rx_mbufs+0xf9 sys/dev/pv/if_vio.c:951 #6 vio_rx_intr+0x69 #7 intr_handler+0x8f sys/arch/amd64/amd64/intr.c:529 #8 Xintr_ioapic_edge19_untramp+0x19f #9 acpicpu_idle+0x331 sys/dev/acpi/acpicpu.c:1187 #10 sched_idle+0x3f7 sys/kern/kern_sched.c:181 #11 proc_trampoline+0x1c Process 61302 (syz-executor.0) thread 0xffff800020ab09f8 (374682) exclusive rrwlock inode r = 0 (0xfffffd8069f12c58) #0 witness_lock+0x52e sys/kern/subr_witness.c:1163 #1 rw_enter+0x447 sys/kern/kern_rwlock.c:306 #2 rrw_enter+0x4f sys/kern/kern_rwlock.c:435 #3 ufs_ihashins+0x45 sys/ufs/ufs/ufs_ihash.c:140 #4 ffs_vget+0x13e sys/ufs/ffs/ffs_vfsops.c:1352 #5 ffs_inode_alloc+0x1cf sys/ufs/ffs/ffs_alloc.c:392 #6 ufs_mkdir+0xf4 sys/ufs/ufs/ufs_vnops.c:1164 #7 VOP_MKDIR+0xc6 sys/kern/vfs_vops.c:450 #8 domkdirat+0x121 sys/kern/vfs_syscalls.c:2983 #9 syscall+0x4a4 mi_syscall sys/sys/syscall_mi.h:92 [inline] #9 syscall+0x4a4 sys/arch/amd64/amd64/trap.c:555 #10 Xsyscall+0x128 exclusive rrwlock inode r = 0 (0xfffffd8069f12818) #0 witness_lock+0x52e sys/kern/subr_witness.c:1163 #1 rw_enter+0x447 sys/kern/kern_rwlock.c:306 #2 rrw_enter+0x4f sys/kern/kern_rwlock.c:435 #3 VOP_LOCK+0xf0 sys/kern/vfs_vops.c:615 #4 vn_lock+0x81 sys/kern/vfs_vnops.c:562 #5 vfs_lookup+0xe6 sys/kern/vfs_lookup.c:418 #6 namei+0x62c sys/kern/vfs_lookup.c:248 #7 domkdirat+0x75 sys/kern/vfs_syscalls.c:2968 #8 syscall+0x4a4 mi_syscall sys/sys/syscall_mi.h:92 [inline] #8 syscall+0x4a4 sys/arch/amd64/amd64/trap.c:555 #9 Xsyscall+0x128 Process 13774 (idle0) thread 0xffff800020a104f0 (259617) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff82666ce0) #0 witness_lock+0x52e sys/kern/subr_witness.c:1163 #1 intr_handler+0x5e sys/arch/amd64/amd64/intr.c:525 #2 Xintr_ioapic_edge19_untramp+0x19f #3 acpicpu_idle+0x331 sys/dev/acpi/acpicpu.c:1187 #4 sched_idle+0x3f7 sys/kern/kern_sched.c:181 #5 proc_trampoline+0x1c exclusive mutex mbufpl r = 0 (0xffffffff8263e280) #0 witness_lock+0x52e sys/kern/subr_witness.c:1163 #1 mtx_enter_try+0x102 #2 mtx_enter+0x4b sys/kern/kern_lock.c:266 #3 pool_get+0xbf sys/kern/subr_pool.c:578 #4 m_clget+0x1c9 m_gethdr sys/kern/uipc_mbuf.c:283 [inline] #4 m_clget+0x1c9 sys/kern/uipc_mbuf.c:400 #5 vio_populate_rx_mbufs+0xf9 vio_add_rx_mbuf sys/dev/pv/if_vio.c:908 [inline] #5 vio_populate_rx_mbufs+0xf9 sys/dev/pv/if_vio.c:951 #6 vio_rx_intr+0x69 #7 intr_handler+0x8f sys/arch/amd64/amd64/intr.c:529 #8 Xintr_ioapic_edge19_untramp+0x19f #9 acpicpu_idle+0x331 sys/dev/acpi/acpicpu.c:1187 #10 sched_idle+0x3f7 sys/kern/kern_sched.c:181 #11 proc_trampoline+0x1c ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim Kern Lim devbuf 9594 6837K 7969K 78643K 24145 0 0 pcb 13 8K 8K 78643K 2751 0 0 rtable 110 12K 13K 78643K 2380 0 0 ifaddr 72 18K 20K 78643K 765 0 0 counters 39 33K 33K 78643K 39 0 0 ioctlops 0 0K 4K 78643K 1796 0 0 iov 0 0K 24K 78643K 728 0 0 mount 1 1K 1K 78643K 1 0 0 vnodes 1216 76K 77K 78643K 7190 0 0 UFS quota 1 32K 32K 78643K 1 0 0 UFS mount 5 36K 36K 78643K 5 0 0 shm 2 1K 9K 78643K 92 0 0 VM map 2 1K 1K 78643K 19 0 0 sem 12 0K 1K 78643K 1124 0 0 dirhash 12 2K 2K 78643K 12 0 0 ACPI 1808 196K 290K 78643K 12765 0 0 file desc 5 13K 25K 78643K 12436 0 0 sigio 0 0K 0K 78643K 75 0 0 proc 61 63K 95K 78643K 2348 0 0 subproc 32 2K 2K 78643K 493 0 0 NFS srvsock 1 0K 0K 78643K 1 0 0 NFS daemon 1 16K 16K 78643K 1 0 0 ip_moptions 0 0K 0K 78643K 415 0 0 in_multi 35 2K 2K 78643K 732 0 0 ether_multi 1 0K 0K 78643K 137 0 0 mrt 4 0K 0K 78643K 98 0 0 ISOFS mount 1 32K 32K 78643K 1 0 0 MSDOSFS mount 1 16K 16K 78643K 1 0 0 ttys 162 715K 715K 78643K 162 0 0 exec 0 0K 1K 78643K 974 0 0 pagedep 1 8K 8K 78643K 1 0 0 inodedep 1 32K 32K 78643K 1 0 0 newblk 1 0K 0K 78643K 1 0 0 VM swap 7 26K 26K 78643K 7 0 0 UVM amap 169 25K 34K 78643K 36092 0 0 UVM aobj 130 5K 5K 78643K 155 0 0 memdesc 1 4K 4K 78643K 1 0 0 crypto data 1 1K 1K 78643K 1 0 0 ip6_options 0 0K 1K 78643K 5846 0 0 NDP 15 0K 0K 78643K 239 0 0 temp 193 3556K 4195K 78643K 147176 0 0 kqueue 0 0K 0K 78643K 82 0 0 SYN cache 2 16K 16K 78643K 2 0 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle arp 64 36 0 31 1 0 1 1 0 8 0 plcache 128 20 0 0 1 0 1 1 0 8 0 rtpcb 80 259 0 257 1 0 1 1 0 8 0 rtentry 112 313 0 272 2 0 2 2 0 8 0 unpcb 120 2380 0 2359 7 6 1 2 0 8 0 syncache 264 4 0 4 1 1 0 1 0 8 0 tcpqe 32 208 0 208 17 17 0 1 0 8 0 tcpcb 544 5041 0 5037 1 0 1 1 0 8 0 inpcb 280 17904 0 17897 20 18 2 2 0 8 1 rttmr 72 21 0 21 6 6 0 1 0 8 0 nd6 48 56 0 52 1 0 1 1 0 8 0 pkpcb 40 24 0 24 7 7 0 1 0 8 0 ppxss 1128 102 0 102 21 21 0 1 0 8 0 pffrag 232 28 0 28 5 5 0 1 0 482 0 pffrnode 88 28 0 28 5 5 0 1 0 8 0 pffrent 40 68 0 68 6 6 0 1 0 8 0 pfosfp 40 846 0 423 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfstitem 24 584 0 563 3 2 1 3 0 8 0 pfstkey 112 584 0 563 11 9 2 11 0 8 0 pfstate 328 584 0 563 34 31 3 32 0 8 0 pfrule 1360 21 0 16 2 1 1 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 1423 0 1209 22 8 14 15 0 8 0 art_table 32 1424 0 1209 2 0 2 2 0 8 0 art_node 16 303 0 266 1 0 1 1 0 8 0 semupl 112 9 0 9 1 1 0 1 0 8 0 semapl 112 1102 0 1092 1 0 1 1 0 8 0 shmpl 112 153 0 25 4 0 4 4 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino1pl 128 19677 0 18251 47 0 47 47 0 8 0 ffsino 272 19677 0 18251 96 0 96 96 0 8 0 nchpl 144 36123 0 35643 64 40 24 61 0 8 1 uvmvnodes 72 6517 0 0 119 0 119 119 0 8 0 vnodes 208 6517 0 0 343 0 343 343 0 8 0 namei 1024 101640 0 101639 2 1 1 1 0 8 0 percpumem 16 30 0 0 1 0 1 1 0 8 0 vmpool 552 17 0 17 4 4 0 1 0 8 0 scsiplug 64 17 0 17 6 6 0 1 0 8 0 scxspl 192 106691 0 106690 26 24 2 7 0 8 1 plimitpl 152 514 0 506 1 0 1 1 0 8 0 sigapl 432 12556 0 12541 3 1 2 3 0 8 0 futexpl 56 181487 0 181487 1 0 1 1 0 8 1 knotepl 112 4790 0 4771 3 2 1 3 0 8 0 kqueuepl 104 10086 0 10084 2 1 1 2 0 8 0 pipepl 112 19856 0 19837 19 17 2 2 0 8 1 fdescpl 488 12557 0 12541 3 0 3 3 0 8 0 filepl 152 89825 0 89724 42 34 8 9 0 8 4 lockfpl 104 4760 0 4759 1 0 1 1 0 8 0 lockfspl 48 1112 0 1111 1 0 1 1 0 8 0 sessionpl 112 45 0 34 1 0 1 1 0 8 0 pgrppl 48 139 0 128 1 0 1 1 0 8 0 ucredpl 96 8767 0 8755 1 0 1 1 0 8 0 zombiepl 144 12545 0 12545 2 1 1 1 0 8 1 processpl 896 12578 0 12545 4 0 4 4 0 8 0 procpl 632 33107 0 33064 17 12 5 5 0 8 1 srpgc 64 52 0 52 21 20 1 1 0 8 1 sosppl 128 96 0 96 14 14 0 1 0 8 0 sockpl 384 20711 0 20681 38 33 5 8 0 8 1 mcl64k 65536 26 0 0 3 0 3 3 0 8 0 mcl16k 16384 12 0 0 2 0 2 2 0 8 0 mcl12k 12288 25 0 0 2 0 2 2 0 8 0 mcl9k 9216 21 0 0 2 0 2 2 0 8 0 mcl8k 8192 20 0 0 3 1 2 3 0 8 0 mcl4k 4096 49 0 0 5 2 3 3 0 8 0 mcl2k2 2112 15 0 0 1 0 1 1 0 8 0 mcl2k 2048 299 0 0 13 3 10 12 0 8 0 mtagpl 80 35 0 0 1 0 1 1 0 8 0 mbufpl 256 604 0 0 18 4 14 18 0 8 0 mbufpl: pool(0xffffffff8263e270:mbufpl): free list modified: page 0xfffffd806f0a8000; item ordinal 0; addr 0xfffffd806f0a8300 (p 0xfffffd806f4bb000); offset 0x0=0x0 mbufpl: pool(0xffffffff8263e270:mbufpl): page inconsistency: page 0xfffffd806f0a8000; item ordinal 1; addr 0xfffffd80bb7171d8 bufpl 256 43575 0 36527 441 0 441 441 0 8 0 anonpl 16 1803355 0 1797643 232 197 35 52 0 124 0 amapchunkpl 152 109792 0 109702 37 32 5 12 0 158 0 amappl16 192 63588 0 63313 225 209 16 30 0 8 1 amappl15 184 328 0 327 1 0 1 1 0 8 0 amappl14 176 5292 0 5286 2 1 1 1 0 8 0 amappl13 168 2329 0 2329 4 4 0 1 0 8 0 amappl12 160 1060 0 1054 3 2 1 1 0 8 0 amappl11 152 1070 0 1053 1 0 1 1 0 8 0 amappl10 144 1424 0 1417 1 0 1 1 0 8 0 amappl9 136 1783 0 1777 1 0 1 1 0 8 0 amappl8 128 1446 0 1348 5 1 4 4 0 8 0 amappl7 120 1519 0 1511 1 0 1 1 0 8 0 amappl6 112 1041 0 1020 1 0 1 1 0 8 0 amappl5 104 1494 0 1480 1 0 1 1 0 8 0 amappl4 96 12852 0 12810 3 1 2 2 0 8 0 amappl3 88 7718 0 7712 1 0 1 1 0 8 0 amappl2 80 98121 0 98040 4 2 2 3 0 8 0 amappl1 72 244011 0 243577 26 16 10 20 0 8 0 amappl 80 34204 0 34169 1 0 1 1 0 84 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma64 64 259 0 259 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 17 0 17 1 1 0 1 0 8 0 aobjpl 64 154 0 25 3 0 3 3 0 8 0 uaddrrnd 24 12574 0 12541 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 12574 0 12541 1 0 1 1 0 8 0 vmmpekpl 168 117217 0 117188 2 0 2 2 0 8 0 vmmpepl 168 1468335 0 1466830 239 161 78 83 0 357 1 vmsppl 368 12556 0 12541 2 0 2 2 0 8 0 pdppl 4096 25155 0 25116 6 0 6 6 0 8 1 pvpl 32 4098329 0 4089339 502 396 106 135 0 265 15 pmappl 232 12573 0 12558 7 6 1 2 0 8 0 extentpl 40 41 0 26 1 0 1 1 0 8 0 phpool 112 622 0 22 18 0 18 18 0 8 0