panic: pool_do_get: mbufpl free list modified: page 0xfffffd80535c8000; item addr 0xfffffd80535c8a00; offset 0x0=0x0 != 0x40a9b6baea7e3fcf Stopped at db_enter+0x18: addq $0x8,%rsp TID PID UID PRFLAGS PFLAGS CPU COMMAND *137666 4792 0 0x12 0 0 sshd db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic(ffffffff8247c7e9) at panic+0x164 sys/kern/subr_prf.c:218 pool_do_get(ffffffff82819ed0,2,ffff80001d6c8bb8) at pool_do_get+0x42a sys/kern/subr_pool.c:738 pool_get(ffffffff82819ed0,2) at pool_get+0xb5 sys/kern/subr_pool.c:581 m_gethdr(2,2) at m_gethdr+0x4c sys/kern/uipc_mbuf.c:283 tcp_output(ffff800000ac54d0) at tcp_output+0x147a tcp_usrreq(fffffd805da70198,9,fffffd80535c8900,0,0,ffff80001d6fc010) at tcp_usrreq+0xa54 sosend(fffffd805da70198,0,ffff80001d6c9038,0,0,80) at sosend+0x669 sys/kern/uipc_socket.c:555 dofilewritev(ffff80001d6fc010,4,ffff80001d6c9038,0,ffff80001d6c9120) at dofilewritev+0x1ab sys/kern/sys_generic.c:365 sys_write(ffff80001d6fc010,ffff80001d6c90d0,ffff80001d6c9120) at sys_write+0x83 sys/kern/sys_generic.c:285 syscall(ffff80001d6c91a0) at syscall+0x507 sys/arch/amd64/amd64/trap.c:570 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffe7b30, count: 3 https://www.openbsd.org/ddb.html describes the minimum info required in bug reports. Insufficient info makes it difficult to find and fix bugs. ddb> ddb> set $lines = 0 ddb> set $maxwidth = 0 ddb> show panic pool_do_get: mbufpl free list modified: page 0xfffffd80535c8000; item addr 0xfffffd80535c8a00; offset 0x0=0x0 != 0x40a9b6baea7e3fcf ddb> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic(ffffffff8247c7e9) at panic+0x164 sys/kern/subr_prf.c:218 pool_do_get(ffffffff82819ed0,2,ffff80001d6c8bb8) at pool_do_get+0x42a sys/kern/subr_pool.c:738 pool_get(ffffffff82819ed0,2) at pool_get+0xb5 sys/kern/subr_pool.c:581 m_gethdr(2,2) at m_gethdr+0x4c sys/kern/uipc_mbuf.c:283 tcp_output(ffff800000ac54d0) at tcp_output+0x147a tcp_usrreq(fffffd805da70198,9,fffffd80535c8900,0,0,ffff80001d6fc010) at tcp_usrreq+0xa54 sosend(fffffd805da70198,0,ffff80001d6c9038,0,0,80) at sosend+0x669 sys/kern/uipc_socket.c:555 dofilewritev(ffff80001d6fc010,4,ffff80001d6c9038,0,ffff80001d6c9120) at dofilewritev+0x1ab sys/kern/sys_generic.c:365 sys_write(ffff80001d6fc010,ffff80001d6c90d0,ffff80001d6c9120) at sys_write+0x83 sys/kern/sys_generic.c:285 syscall(ffff80001d6c91a0) at syscall+0x507 sys/arch/amd64/amd64/trap.c:570 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffe7b30, count: -12 ddb> show registers rdi 0 rsi 0x1 rbp 0xffff80001d6c8a20 rbx 0xffff80001d6c8ad0 rdx 0x8b rcx 0x2 rax 0x1 r8 0xffff80001d6c89e0 r9 0xffffffff81ceca3f kprintf+0x15f r10 0x1 r11 0x85b90e4224ec3815 r12 0x3000000008 r13 0xffff80001d6c8a30 r14 0x100 r15 0x1 rip 0xffffffff82063088 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff80001d6c8a10 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb> show proc PROC (sshd) pid=137666 stat=onproc flags process=12 proc=0 pri=24, usrpri=50, nice=20 forw=0xffffffffffffffff, list=0xffff80001d6fd8c0,0xffff80001d6fc298 process=0xffff80001d6ff288 user=0xffff80001d6c4000, vmspace=0xfffffd806bc0a550 estcpu=0, cpticks=1, pctcpu=0.0 user=0, sys=1, intr=0 ddb> ps PID TID PPID UID S FLAGS WAIT COMMAND 97220 274635 95680 0 2 0x2 syz-executor.1 83150 122203 95680 0 2 0x482 syz-executor.0 52221 125645 0 0 3 0x14280 nfsidl nfsio 5269 157968 0 0 3 0x14280 nfsidl nfsio 37750 69479 0 0 3 0x14280 nfsidl nfsio 45610 375287 0 0 3 0x14280 nfsidl nfsio 5293 422235 0 0 3 0x14280 nfsidl nfsio 59888 349680 0 0 3 0x14280 nfsidl nfsio 85909 381657 0 0 3 0x14280 nfsidl nfsio 13712 436632 0 0 3 0x14280 nfsidl nfsio 60206 127029 0 0 3 0x14280 nfsidl nfsio 42713 220676 0 0 3 0x14280 nfsidl nfsio 38333 186338 0 0 3 0x14280 nfsidl nfsio 89468 366677 0 0 3 0x14280 nfsidl nfsio 7326 372219 0 0 3 0x14280 nfsidl nfsio 13549 450062 0 0 3 0x14280 nfsidl nfsio 26823 497310 0 0 3 0x14280 nfsidl nfsio 13571 308219 0 0 3 0x14280 nfsidl nfsio 14408 10489 0 0 3 0x14280 nfsidl nfsio 25482 195003 0 0 3 0x14280 nfsidl nfsio 4251 312965 0 0 3 0x14280 nfsidl nfsio 51349 135641 0 0 3 0x14280 nfsidl nfsio 52007 278155 0 0 3 0x14200 bored sosplice 13095 422356 0 0 3 0x14200 acct acct 95680 381488 51282 0 3 0x82 thrsleep syz-fuzzer 95680 49393 51282 0 2 0x4000482 syz-fuzzer 95680 294128 51282 0 3 0x4000082 thrsleep syz-fuzzer 95680 155997 51282 0 3 0x4000082 thrsleep syz-fuzzer 95680 468274 51282 0 3 0x4000082 thrsleep syz-fuzzer 95680 1117 51282 0 2 0x4000002 syz-fuzzer 95680 213129 51282 0 3 0x4000082 thrsleep syz-fuzzer 51282 311650 4792 0 3 0x10008a pause ksh * 4792 137666 8004 0 7 0x12 sshd 99063 283362 1 0 3 0x100083 ttyopn getty 8004 226164 1 0 3 0x80 select sshd 31570 319668 63841 73 3 0x100090 kqread syslogd 63841 276098 1 0 3 0x100082 netio syslogd 18339 476929 1 77 3 0x100090 poll dhclient 15978 36621 1 0 3 0x80 poll dhclient 84568 264296 0 0 3 0x14200 bored smr 57625 80555 0 0 3 0x14200 pgzero zerothread 96694 70970 0 0 3 0x14200 aiodoned aiodoned 19700 297527 0 0 3 0x14200 syncer update 48395 355196 0 0 3 0x14200 cleaner cleaner 73064 106764 0 0 3 0x14200 reaper reaper 64823 408744 0 0 3 0x14200 pgdaemon pagedaemon 38975 497855 0 0 3 0x14200 bored crynlk 36680 357883 0 0 3 0x14200 bored crypto 62731 39550 0 0 3 0x40014200 acpi0 acpi0 87516 520907 0 0 3 0x14200 bored softnet 92392 144592 0 0 3 0x14200 bored systqmp 30807 98001 0 0 3 0x14200 bored systq 88008 443771 0 0 3 0x40014200 bored softclock 89604 421743 0 0 3 0x40014200 idle0 1 499888 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb> show all locks No such command ddb> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 9542 6502K 6815K 78643K 11772 0 pcb 13 8K 8K 78643K 138 0 rtable 105 15K 16K 78643K 639 0 ifaddr 107 20K 21K 78643K 261 0 sysctl 2 0K 0K 78643K 2 0 counters 22 16K 16K 78643K 41 0 ioctlops 0 0K 4K 78643K 214 0 iov 0 0K 16K 78643K 91 0 mount 1 1K 1K 78643K 1 0 vnodes 1213 76K 77K 78643K 1636 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 5K 78643K 20 0 VM map 2 0K 0K 78643K 2 0 sem 12 0K 0K 78643K 158 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1809 195K 288K 78643K 12938 0 file desc 4 9K 25K 78643K 1145 0 sigio 0 0K 0K 78643K 19 0 proc 51 38K 55K 78643K 540 0 subproc 32 2K 2K 78643K 85 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 63 0 in_multi 82 4K 4K 78643K 181 0 ether_multi 1 0K 0K 78643K 18 0 mrt 0 0K 0K 78643K 7 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 37 175K 175K 78643K 37 0 exec 0 0K 1K 78643K 275 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 7 26K 26K 78643K 7 0 UVM amap 148 88K 91K 78643K 3976 0 UVM aobj 44 4K 4K 78643K 52 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 0K 78643K 100 0 NDP 17 0K 0K 78643K 50 0 temp 154 3873K 3937K 78643K 16256 0 kqueue 3 4K 18K 78643K 34 0 SYN cache 2 16K 16K 78643K 2 0 ddb> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle arp 64 15 0 8 1 0 1 1 0 8 0 rtpcb 88 874 0 872 1 0 1 1 0 8 0 rtentry 112 106 0 75 2 0 2 2 0 8 0 unpcb 120 393 0 383 1 0 1 1 0 8 0 syncache 272 4 0 4 1 1 0 1 0 8 0 tcpcb 592 291 0 282 9 7 2 3 0 8 1 ipq 40 1 0 1 1 1 0 1 0 8 0 ipqe 40 2 0 2 1 1 0 1 0 8 0 inpcb 296 2339 0 2332 7 6 1 2 0 8 0 rttmr 72 2 0 2 2 2 0 1 0 8 0 ip6q 72 1 0 1 1 1 0 1 0 8 0 ip6af 40 2 0 2 1 1 0 1 0 8 0 nd6 48 21 0 18 1 0 1 1 0 8 0 ppxss 1136 4 0 4 4 4 0 1 0 8 0 pfrktable 1344 56 0 48 1 0 1 1 0 8 0 pftag 88 8 0 4 2 1 1 1 0 8 0 pfrule 1360 26 0 12 3 1 2 2 0 8 0 art_heap8 4096 2 0 0 2 0 2 2 0 8 0 art_heap4 256 570 0 424 22 12 10 14 0 8 0 art_table 32 572 0 424 2 0 2 2 0 8 0 art_node 16 104 0 79 1 0 1 1 0 8 0 sysvmsgpl 40 20 0 5 1 0 1 1 0 8 0 semupl 112 2 0 2 1 1 0 1 0 8 0 semapl 112 156 0 146 1 0 1 1 0 8 0 shmpl 112 49 0 8 2 0 2 2 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 2802 0 1410 88 0 88 88 0 8 0 ffsino 240 2802 0 1410 83 0 83 83 0 8 0 nchpl 144 4535 0 2947 60 0 60 60 0 8 0 uvmvnodes 72 3155 0 0 58 0 58 58 0 8 0 vnodes 208 3155 0 0 167 0 167 167 0 8 0 namei 1024 12145 0 12145 3 2 1 1 0 8 1 vcpupl 1984 7 0 0 1 0 1 1 0 8 0 vmpool 528 10 0 3 1 0 1 1 0 8 0 pfiaddrpl 120 20 0 14 1 0 1 1 0 8 0 scxspl 200 12884 0 12884 2 1 1 1 0 8 1 plimitpl 152 63 0 56 1 0 1 1 0 8 0 sigapl 424 1344 0 1295 6 0 6 6 0 8 0 futexpl 56 21030 0 21030 2 1 1 1 0 8 1 knotepl 112 142 0 122 1 0 1 1 0 8 0 kqueuepl 152 112 0 110 1 0 1 1 0 8 0 pipepl 272 187 0 176 1 0 1 1 0 8 0 fdescpl 432 1308 0 1295 2 0 2 2 0 8 0 filepl 120 8028 0 7931 4 0 4 4 0 8 1 lockfpl 104 130 0 129 1 0 1 1 0 8 0 lockfspl 48 49 0 48 1 0 1 1 0 8 0 sessionpl 120 20 0 10 1 0 1 1 0 8 0 pgrppl 48 32 0 22 1 0 1 1 0 8 0 ucredpl 96 593 0 586 1 0 1 1 0 8 0 zombiepl 144 1295 0 1294 2 1 1 1 0 8 0 processpl 944 1344 0 1294 8 1 7 7 0 8 0 procpl 632 3108 0 3052 13 7 6 6 0 8 1 sosppl 144 8 0 8 3 3 0 1 0 8 0 sockpl 400 3639 0 3620 14 11 3 4 0 8 0 mcl64k 65536 41 0 41 2 2 0 1 0 8 0 mcl16k 16384 2 0 2 2 2 0 1 0 8 0 mcl12k 12288 26 0 26 10 10 0 1 0 8 0 mcl9k 9216 14 0 14 6 6 0 1 0 8 0 mcl8k 8192 33 0 33 9 9 0 1 0 8 0 mcl4k 4096 66 0 66 9 8 1 1 0 8 1 mcl2k2 2112 7 0 7 4 4 0 1 0 8 0 mcl2k 2048 95833 0 95785 17 10 7 13 0 8 0 mtagpl 96 95 0 43 3 1 2 2 0 8 0 mbufpl 256 157406 0 157188 40 21 19 33 0 8 0 mbufpl: pool(0xffffffff82819ed0:mbufpl): free list modified: page 0xfffffd80535c8000; item ordinal 0; addr 0xfffffd80535c8a00 (p 0xfffffd806c3c4000); offset 0x0=0x0 pool(mbufpl): free list modified: page 0xfffffd80535c8000; item ordinal 0; addr 0xfffffd80535c8a00 (p 0xfffffd806c3c4000); offset 0x0=0x0 mbufpl: pool(0xffffffff82819ed0:mbufpl): page inconsistency: page 0xfffffd80535c8000; item ordinal 1; addr 0x9acc65be348b5612 bufpl 280 4964 0 122 346 0 346 346 0 8 0 anonpl 16 143789 0 128008 90 8 82 87 0 107 0 amapchunkpl 152 7071 0 6885 41 33 8 22 0 158 0 amappl16 192 5418 0 4352 71 17 54 66 0 8 0 amappl15 184 71 0 70 1 0 1 1 0 8 0 amappl14 176 10 0 7 1 0 1 1 0 8 0 amappl13 168 629 0 624 1 0 1 1 0 8 0 amappl12 160 591 0 589 1 0 1 1 0 8 0 amappl11 152 52 0 43 1 0 1 1 0 8 0 amappl10 144 466 0 462 1 0 1 1 0 8 0 amappl9 136 404 0 401 1 0 1 1 0 8 0 amappl8 128 425 0 387 3 1 2 2 0 8 0 amappl7 120 550 0 538 1 0 1 1 0 8 0 amappl6 112 25 0 21 1 0 1 1 0 8 0 amappl5 104 703 0 693 1 0 1 1 0 8 0 amappl4 96 976 0 947 1 0 1 1 0 8 0 amappl3 88 635 0 627 1 0 1 1 0 8 0 amappl2 80 9211 0 9154 2 0 2 2 0 8 0 amappl1 72 43177 0 42781 23 13 10 17 0 8 0 amappl 80 3398 0 3342 2 0 2 2 0 84 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 64 51 0 8 1 0 1 1 0 8 0 uaddrrnd 24 1318 0 1298 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 1318 0 1298 1 0 1 1 0 8 0 vmmpekpl 168 12108 0 12078 2 0 2 2 0 8 0 vmmpepl 168 170858 0 168791 217 110 107 138 0 357 7 vmsppl 272 1317 0 1298 3 1 2 2 0 8 0 pdppl 4096 2642 0 2603 8 2 6 6 0 8 0 pvpl 32 392766 0 373965 218 29 189 204 0 265 0 pmappl 200 1317 0 1298 2 0 2 2 0 8 0 extentpl 40 53 0 36 1 0 1 1 0 8 0 phpool 112 312 0 84 8 0 8 8 0 8 0 ddb> machine ddbcpu 0 No such command ddb> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic(ffffffff8247c7e9) at panic+0x164 sys/kern/subr_prf.c:218 pool_do_get(ffffffff82819ed0,2,ffff80001d6c8bb8) at pool_do_get+0x42a sys/kern/subr_pool.c:738 pool_get(ffffffff82819ed0,2) at pool_get+0xb5 sys/kern/subr_pool.c:581 m_gethdr(2,2) at m_gethdr+0x4c sys/kern/uipc_mbuf.c:283 tcp_output(ffff800000ac54d0) at tcp_output+0x147a tcp_usrreq(fffffd805da70198,9,fffffd80535c8900,0,0,ffff80001d6fc010) at tcp_usrreq+0xa54 sosend(fffffd805da70198,0,ffff80001d6c9038,0,0,80) at sosend+0x669 sys/kern/uipc_socket.c:555 dofilewritev(ffff80001d6fc010,4,ffff80001d6c9038,0,ffff80001d6c9120) at dofilewritev+0x1ab sys/kern/sys_generic.c:365 sys_write(ffff80001d6fc010,ffff80001d6c90d0,ffff80001d6c9120) at sys_write+0x83 sys/kern/sys_generic.c:285 syscall(ffff80001d6c91a0) at syscall+0x507 sys/arch/amd64/amd64/trap.c:570 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffe7b30, count: -12 ddb> machine ddbcpu 1 No such command ddb> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic(ffffffff8247c7e9) at panic+0x164 sys/kern/subr_prf.c:218 pool_do_get(ffffffff82819ed0,2,ffff80001d6c8bb8) at pool_do_get+0x42a sys/kern/subr_pool.c:738 pool_get(ffffffff82819ed0,2) at pool_get+0xb5 sys/kern/subr_pool.c:581 m_gethdr(2,2) at m_gethdr+0x4c sys/kern/uipc_mbuf.c:283 tcp_output(ffff800000ac54d0) at tcp_output+0x147a tcp_usrreq(fffffd805da70198,9,fffffd80535c8900,0,0,ffff80001d6fc010) at tcp_usrreq+0xa54 sosend(fffffd805da70198,0,ffff80001d6c9038,0,0,80) at sosend+0x669 sys/kern/uipc_socket.c:555 dofilewritev(ffff80001d6fc010,4,ffff80001d6c9038,0,ffff80001d6c9120) at dofilewritev+0x1ab sys/kern/sys_generic.c:365 sys_write(ffff80001d6fc010,ffff80001d6c90d0,ffff80001d6c9120) at sys_write+0x83 sys/kern/sys_generic.c:285 syscall(ffff80001d6c91a0) at syscall+0x507 sys/arch/amd64/amd64/trap.c:570 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffe7b30, count: -12