panic: pool_do_get: mbufpl free list modified: page 0xfffffd805b8ed000; item addr 0xfffffd805b8ed900; offset 0x0=0x0 != 0x299e47f517ea64ea Stopped at db_enter+0x18: addq $0x8,%rsp TID PID UID PRFLAGS PFLAGS CPU COMMAND * 56357 86754 0 0x12 0 0 sshd db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic(ffffffff82477326) at panic+0x164 sys/kern/subr_prf.c:218 pool_do_get(ffffffff82822648,2,ffff80001d723098) at pool_do_get+0x42a sys/kern/subr_pool.c:738 pool_get(ffffffff82822648,2) at pool_get+0xb5 sys/kern/subr_pool.c:581 m_gethdr(2,2) at m_gethdr+0x4c sys/kern/uipc_mbuf.c:283 tcp_output(ffff800000ac3320) at tcp_output+0x147a tcp_usrreq(fffffd805da687d8,9,fffffd805b8ed800,0,0,ffff80001d71c9d0) at tcp_usrreq+0xa54 sosend(fffffd805da687d8,0,ffff80001d723518,0,0,80) at sosend+0x669 sys/kern/uipc_socket.c:555 dofilewritev(ffff80001d71c9d0,4,ffff80001d723518,0,ffff80001d723600) at dofilewritev+0x1ab sys/kern/sys_generic.c:365 sys_write(ffff80001d71c9d0,ffff80001d7235b0,ffff80001d723600) at sys_write+0x83 sys/kern/sys_generic.c:285 syscall(ffff80001d723680) at syscall+0x507 sys/arch/amd64/amd64/trap.c:570 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffc1500, count: 3 https://www.openbsd.org/ddb.html describes the minimum info required in bug reports. Insufficient info makes it difficult to find and fix bugs. ddb> ddb> set $lines = 0 ddb> set $maxwidth = 0 ddb> show panic pool_do_get: mbufpl free list modified: page 0xfffffd805b8ed000; item addr 0xfffffd805b8ed900; offset 0x0=0x0 != 0x299e47f517ea64ea ddb> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic(ffffffff82477326) at panic+0x164 sys/kern/subr_prf.c:218 pool_do_get(ffffffff82822648,2,ffff80001d723098) at pool_do_get+0x42a sys/kern/subr_pool.c:738 pool_get(ffffffff82822648,2) at pool_get+0xb5 sys/kern/subr_pool.c:581 m_gethdr(2,2) at m_gethdr+0x4c sys/kern/uipc_mbuf.c:283 tcp_output(ffff800000ac3320) at tcp_output+0x147a tcp_usrreq(fffffd805da687d8,9,fffffd805b8ed800,0,0,ffff80001d71c9d0) at tcp_usrreq+0xa54 sosend(fffffd805da687d8,0,ffff80001d723518,0,0,80) at sosend+0x669 sys/kern/uipc_socket.c:555 dofilewritev(ffff80001d71c9d0,4,ffff80001d723518,0,ffff80001d723600) at dofilewritev+0x1ab sys/kern/sys_generic.c:365 sys_write(ffff80001d71c9d0,ffff80001d7235b0,ffff80001d723600) at sys_write+0x83 sys/kern/sys_generic.c:285 syscall(ffff80001d723680) at syscall+0x507 sys/arch/amd64/amd64/trap.c:570 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffc1500, count: -12 ddb> show registers rdi 0 rsi 0x1 rbp 0xffff80001d722f00 rbx 0xffff80001d722fb0 rdx 0x2 rcx 0 rax 0x1 r8 0xffffffff81fc279f kprintf+0x15f r9 0x1 r10 0x2 r11 0xc477d334e0497cc7 r12 0x3000000008 r13 0xffff80001d722f10 r14 0x100 r15 0x1 rip 0xffffffff8175b418 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff80001d722ef0 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb> show proc PROC (sshd) pid=56357 stat=onproc flags process=12 proc=0 pri=24, usrpri=50, nice=20 forw=0xffffffffffffffff, list=0xffff80001d71d600,0xffff80001d71c290 process=0xffff80001d700758 user=0xffff80001d71e000, vmspace=0xfffffd806bc0aaa0 estcpu=0, cpticks=1, pctcpu=0.0 user=0, sys=1, intr=0 ddb> ps PID TID PPID UID S FLAGS WAIT COMMAND 78991 523162 0 0 3 0x14200 bored sosplice 81821 21468 0 0 3 0x14280 nfsidl nfsio 24688 427944 0 0 3 0x14280 nfsidl nfsio 51109 337046 0 0 3 0x14280 nfsidl nfsio 16376 357408 0 0 3 0x14280 nfsidl nfsio 99210 158035 0 0 3 0x14280 nfsidl nfsio 84501 480030 0 0 3 0x14280 nfsidl nfsio 56407 402366 0 0 3 0x14280 nfsidl nfsio 32557 385632 0 0 3 0x14280 nfsidl nfsio 42021 395334 0 0 3 0x14280 nfsidl nfsio 1065 456001 0 0 3 0x14280 nfsidl nfsio 19101 313695 0 0 3 0x14280 nfsidl nfsio 18418 336625 0 0 3 0x14280 nfsidl nfsio 5991 459100 0 0 3 0x14280 nfsidl nfsio 10260 476702 0 0 3 0x14280 nfsidl nfsio 66603 165094 0 0 3 0x14280 nfsidl nfsio 45923 50661 0 0 3 0x14280 nfsidl nfsio 40907 251779 0 0 3 0x14280 nfsidl nfsio 64164 281514 0 0 3 0x14280 nfsidl nfsio 40986 212892 0 0 3 0x14280 nfsidl nfsio 28975 299093 0 0 3 0x14280 nfsidl nfsio 62644 25089 92344 0 3 0x82 piperd syz-executor.1 41226 240271 92344 0 3 0x82 piperd syz-executor.0 92344 122112 18798 0 3 0x82 thrsleep syz-fuzzer 92344 326008 18798 0 3 0x4000082 nanosleep syz-fuzzer 92344 127444 18798 0 3 0x4000082 thrsleep syz-fuzzer 92344 466851 18798 0 3 0x4000082 thrsleep syz-fuzzer 92344 399132 18798 0 2 0x4000002 syz-fuzzer 92344 80524 18798 0 3 0x4000082 thrsleep syz-fuzzer 92344 155514 18798 0 3 0x4000082 thrsleep syz-fuzzer 92344 88941 18798 0 3 0x4000082 thrsleep syz-fuzzer 18798 377742 86754 0 3 0x10008a pause ksh *86754 56357 44041 0 7 0x12 sshd 17899 62089 1 0 3 0x100083 ttyin getty 44041 410654 1 0 3 0x80 select sshd 23600 522294 89013 73 3 0x100090 kqread syslogd 89013 231386 1 0 3 0x100082 netio syslogd 78144 363845 1 77 3 0x100090 poll dhclient 56728 187023 1 0 3 0x80 poll dhclient 38886 4285 0 0 3 0x14200 bored smr 51723 39974 0 0 2 0x14200 zerothread 70371 350921 0 0 3 0x14200 aiodoned aiodoned 18868 257574 0 0 3 0x14200 syncer update 88472 447633 0 0 3 0x14200 cleaner cleaner 33276 50330 0 0 3 0x14200 reaper reaper 98766 442172 0 0 3 0x14200 pgdaemon pagedaemon 53248 484558 0 0 3 0x14200 bored crynlk 16267 506640 0 0 3 0x14200 bored crypto 48582 458706 0 0 3 0x40014200 acpi0 acpi0 49185 19065 0 0 3 0x14200 bored softnet 95772 268185 0 0 3 0x14200 bored systqmp 58774 518942 0 0 3 0x14200 bored systq 29587 259656 0 0 3 0x40014200 bored softclock 73798 496308 0 0 3 0x40014200 idle0 1 587 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb> show all locks No such command ddb> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 9469 6331K 6710K 78643K 10782 0 pcb 13 8K 8K 78643K 39 0 rtable 119 5K 9K 78643K 508 0 ifaddr 62 14K 15K 78643K 170 0 counters 21 16K 16K 78643K 27 0 ioctlops 0 0K 4K 78643K 609 0 iov 0 0K 16K 78643K 24 0 mount 1 1K 1K 78643K 1 0 vnodes 1218 77K 77K 78643K 1291 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 5K 78643K 4 0 VM map 2 0K 0K 78643K 2 0 sem 12 1K 1K 78643K 20 0 dirhash 9 1K 2K 78643K 12 0 ACPI 1809 195K 288K 78643K 12938 0 file desc 4 9K 25K 78643K 265 0 sigio 0 0K 0K 78643K 15 0 proc 50 38K 54K 78643K 374 0 subproc 32 2K 2K 78643K 34 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 22 0 in_multi 46 2K 2K 78643K 82 0 ether_multi 1 0K 0K 78643K 7 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 37 175K 175K 78643K 37 0 exec 0 0K 1K 78643K 203 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 7 26K 26K 78643K 7 0 UVM amap 125 23K 27K 78643K 1476 0 UVM aobj 7 2K 2K 78643K 7 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 0K 78643K 21 0 NDP 8 0K 0K 78643K 21 0 temp 80 3850K 3914K 78643K 6919 0 kqueue 3 4K 12K 78643K 13 0 SYN cache 2 16K 16K 78643K 2 0 ddb> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle arp 64 7 0 0 1 0 1 1 0 8 0 rtpcb 80 35 0 33 1 0 1 1 0 8 0 rtentry 112 57 0 12 2 0 2 2 0 8 0 unpcb 120 181 0 171 1 0 1 1 0 8 0 syncache 264 8 0 8 2 2 0 1 0 8 0 tcpqe 32 113 0 113 1 1 0 1 0 8 0 tcpcb 544 90 0 83 1 0 1 1 0 8 0 ipq 40 3 0 2 1 0 1 1 0 8 0 ipqe 40 92 0 91 1 0 1 1 0 8 0 inpcb 296 242 0 234 5 3 2 2 0 8 1 ip6q 72 1 0 1 1 1 0 1 0 8 0 ip6af 40 2 0 2 1 1 0 1 0 8 0 nd6 48 12 0 6 1 0 1 1 0 8 0 pkpcb 40 2 0 2 1 1 0 1 0 8 0 pfosfp 40 1 0 0 1 0 1 1 0 8 0 pfosfpen 112 2 0 0 1 0 1 1 0 8 0 pfrktable 1344 121 0 116 3 1 2 2 0 8 1 pftag 88 14 0 14 4 3 1 1 0 8 1 pfrule 1360 20 0 14 1 0 1 1 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 216 0 40 13 0 13 13 0 8 0 art_table 32 217 0 40 2 0 2 2 0 8 0 art_node 16 56 0 15 1 0 1 1 0 8 0 sysvmsgpl 40 2 0 0 1 0 1 1 0 8 0 semupl 112 5 0 5 2 2 0 1 0 8 0 semapl 112 12 0 2 1 0 1 1 0 8 0 shmpl 112 4 0 0 1 0 1 1 0 8 0 dirhash 1024 17 0 10 3 1 2 3 0 8 0 dino2pl 256 1718 0 320 88 0 88 88 0 8 0 ffsino 240 1718 0 320 83 0 83 83 0 8 0 nchpl 144 2293 0 703 60 0 60 60 0 8 0 uvmvnodes 72 1814 0 0 33 0 33 33 0 8 0 vnodes 208 1814 0 0 96 0 96 96 0 8 0 namei 1024 6440 0 6440 2 1 1 1 0 8 1 vmpool 528 12 0 12 3 2 1 1 0 8 1 pfiaddrpl 120 36 0 32 1 0 1 1 0 8 0 scsiplug 64 1 0 1 1 1 0 1 0 8 0 scxspl 192 6583 0 6583 2 1 1 1 0 8 1 plimitpl 152 26 0 19 1 0 1 1 0 8 0 sigapl 424 472 0 424 6 0 6 6 0 8 0 futexpl 56 5492 0 5492 2 1 1 1 0 8 1 knotepl 112 73 0 54 1 0 1 1 0 8 0 kqueuepl 144 36 0 33 1 0 1 1 0 8 0 pipepl 272 110 0 100 1 0 1 1 0 8 0 fdescpl 432 437 0 424 2 0 2 2 0 8 0 filepl 120 2586 0 2491 5 1 4 4 0 8 1 lockfpl 104 49 0 48 1 0 1 1 0 8 0 lockfspl 48 20 0 19 1 0 1 1 0 8 0 sessionpl 112 17 0 7 1 0 1 1 0 8 0 pgrppl 48 19 0 9 1 0 1 1 0 8 0 ucredpl 96 185 0 178 1 0 1 1 0 8 0 zombiepl 144 424 0 424 2 1 1 1 0 8 1 processpl 928 472 0 424 7 0 7 7 0 8 0 procpl 624 780 0 725 7 2 5 5 0 8 0 sosppl 128 3 0 3 1 0 1 1 0 8 1 sockpl 400 460 0 441 6 2 4 4 0 8 1 mcl64k 65536 533 0 533 69 7 62 65 0 8 62 mcl16k 16384 2 0 2 1 1 0 1 0 8 0 mcl12k 12288 10 0 10 6 6 0 1 0 8 0 mcl9k 9216 4 0 4 4 3 1 1 0 8 1 mcl8k 8192 11 0 11 5 4 1 1 0 8 1 mcl4k 4096 36 0 36 5 4 1 1 0 8 1 mcl2k2 2112 1 0 1 1 1 0 1 0 8 0 mcl2k 2048 95347 0 95298 16 9 7 15 0 8 0 mtagpl 96 23 0 2 2 1 1 1 0 8 0 mbufpl 256 152794 0 152638 47 24 23 42 0 8 8 mbufpl: pool(0xffffffff82822648:mbufpl): free list modified: page 0xfffffd805b8ed000; item ordinal 0; addr 0xfffffd805b8ed900 (p 0xfffffd8063fe0000); offset 0x0=0x0 pool(mbufpl): free list modified: page 0xfffffd805b8ed000; item ordinal 0; addr 0xfffffd805b8ed900 (p 0xfffffd8063fe0000); offset 0x0=0x0 mbufpl: pool(0xffffffff82822648:mbufpl): page inconsistency: page 0xfffffd805b8ed000; item ordinal 1; addr 0x8faf98c78a4af911 bufpl 280 4144 0 126 287 0 287 287 0 8 0 anonpl 16 67837 0 50662 121 35 86 86 0 107 16 amapchunkpl 152 1851 0 1724 9 3 6 7 0 158 0 amappl16 192 2791 0 1825 81 25 56 60 0 8 7 amappl15 184 1 0 0 1 0 1 1 0 8 0 amappl14 176 54 0 48 1 0 1 1 0 8 0 amappl13 168 232 0 228 1 0 1 1 0 8 0 amappl12 160 8 0 7 2 1 1 1 0 8 0 amappl11 152 71 0 61 1 0 1 1 0 8 0 amappl10 144 26 0 19 1 0 1 1 0 8 0 amappl9 136 382 0 381 1 0 1 1 0 8 0 amappl8 128 337 0 292 2 0 2 2 0 8 0 amappl7 120 107 0 94 1 0 1 1 0 8 0 amappl6 112 22 0 18 1 0 1 1 0 8 0 amappl5 104 337 0 326 1 0 1 1 0 8 0 amappl4 96 654 0 625 1 0 1 1 0 8 0 amappl3 88 142 0 136 1 0 1 1 0 8 0 amappl2 80 2677 0 2620 2 0 2 2 0 8 0 amappl1 72 18937 0 18537 23 13 10 17 0 8 0 amappl 80 991 0 953 1 0 1 1 0 84 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 64 6 0 0 1 0 1 1 0 8 0 uaddrrnd 24 449 0 436 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 449 0 436 1 0 1 1 0 8 0 vmmpekpl 168 6809 0 6778 2 0 2 2 0 8 0 vmmpepl 168 61996 0 60037 153 42 111 114 0 357 23 vmsppl 272 448 0 436 2 1 1 2 0 8 0 pdppl 4096 904 0 872 6 1 5 6 0 8 0 pvpl 32 192199 0 172017 266 66 200 202 0 265 36 pmappl 200 448 0 436 1 0 1 1 0 8 0 extentpl 40 53 0 36 1 0 1 1 0 8 0 phpool 112 359 0 59 10 0 10 10 0 8 0 ddb> machine ddbcpu 0 No such command ddb> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic(ffffffff82477326) at panic+0x164 sys/kern/subr_prf.c:218 pool_do_get(ffffffff82822648,2,ffff80001d723098) at pool_do_get+0x42a sys/kern/subr_pool.c:738 pool_get(ffffffff82822648,2) at pool_get+0xb5 sys/kern/subr_pool.c:581 m_gethdr(2,2) at m_gethdr+0x4c sys/kern/uipc_mbuf.c:283 tcp_output(ffff800000ac3320) at tcp_output+0x147a tcp_usrreq(fffffd805da687d8,9,fffffd805b8ed800,0,0,ffff80001d71c9d0) at tcp_usrreq+0xa54 sosend(fffffd805da687d8,0,ffff80001d723518,0,0,80) at sosend+0x669 sys/kern/uipc_socket.c:555 dofilewritev(ffff80001d71c9d0,4,ffff80001d723518,0,ffff80001d723600) at dofilewritev+0x1ab sys/kern/sys_generic.c:365 sys_write(ffff80001d71c9d0,ffff80001d7235b0,ffff80001d723600) at sys_write+0x83 sys/kern/sys_generic.c:285 syscall(ffff80001d723680) at syscall+0x507 sys/arch/amd64/amd64/trap.c:570 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffc1500, count: -12 ddb> machine ddbcpu 1 No such command ddb> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic(ffffffff82477326) at panic+0x164 sys/kern/subr_prf.c:218 pool_do_get(ffffffff82822648,2,ffff80001d723098) at pool_do_get+0x42a sys/kern/subr_pool.c:738 pool_get(ffffffff82822648,2) at pool_get+0xb5 sys/kern/subr_pool.c:581 m_gethdr(2,2) at m_gethdr+0x4c sys/kern/uipc_mbuf.c:283 tcp_output(ffff800000ac3320) at tcp_output+0x147a tcp_usrreq(fffffd805da687d8,9,fffffd805b8ed800,0,0,ffff80001d71c9d0) at tcp_usrreq+0xa54 sosend(fffffd805da687d8,0,ffff80001d723518,0,0,80) at sosend+0x669 sys/kern/uipc_socket.c:555 dofilewritev(ffff80001d71c9d0,4,ffff80001d723518,0,ffff80001d723600) at dofilewritev+0x1ab sys/kern/sys_generic.c:365 sys_write(ffff80001d71c9d0,ffff80001d7235b0,ffff80001d723600) at sys_write+0x83 sys/kern/sys_generic.c:285 syscall(ffff80001d723680) at syscall+0x507 sys/arch/amd64/amd64/trap.c:570 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffc1500, count: -12