panic: pool_do_get: mbufpl free list modified: page 0xfffffd805c36e000; item addr 0xfffffd805c36e400; offset 0x0=0x0 != 0xaac1fe27deb7315e Stopped at db_enter+0x18: addq $0x8,%rsp TID PID UID PRFLAGS PFLAGS CPU COMMAND *352933 44076 0 0x12 0 0 sshd db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic(ffffffff8246ccb8) at panic+0x15c sys/kern/subr_prf.c:207 pool_do_get(ffffffff82855e90,2,ffff80001d6cd588) at pool_do_get+0x42a sys/kern/subr_pool.c:738 pool_get(ffffffff82855e90,2) at pool_get+0xb5 sys/kern/subr_pool.c:581 m_gethdr(2,2) at m_gethdr+0x4c sys/kern/uipc_mbuf.c:283 tcp_output(ffff800000ac3100) at tcp_output+0x147a tcp_usrreq(fffffd805da74198,9,fffffd805c36e300,0,0,ffff80001d71d390) at tcp_usrreq+0xa54 sosend(fffffd805da74198,0,ffff80001d6cda08,0,0,80) at sosend+0x669 sys/kern/uipc_socket.c:555 dofilewritev(ffff80001d71d390,4,ffff80001d6cda08,0,ffff80001d6cdaf0) at dofilewritev+0x1ab sys/kern/sys_generic.c:365 sys_write(ffff80001d71d390,ffff80001d6cdaa0,ffff80001d6cdaf0) at sys_write+0x83 sys/kern/sys_generic.c:285 syscall(ffff80001d6cdb70) at syscall+0x507 sys/arch/amd64/amd64/trap.c:570 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffdd290, count: 3 https://www.openbsd.org/ddb.html describes the minimum info required in bug reports. Insufficient info makes it difficult to find and fix bugs. ddb> ddb> set $lines = 0 ddb> set $maxwidth = 0 ddb> show panic pool_do_get: mbufpl free list modified: page 0xfffffd805c36e000; item addr 0xfffffd805c36e400; offset 0x0=0x0 != 0xaac1fe27deb7315e ddb> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic(ffffffff8246ccb8) at panic+0x15c sys/kern/subr_prf.c:207 pool_do_get(ffffffff82855e90,2,ffff80001d6cd588) at pool_do_get+0x42a sys/kern/subr_pool.c:738 pool_get(ffffffff82855e90,2) at pool_get+0xb5 sys/kern/subr_pool.c:581 m_gethdr(2,2) at m_gethdr+0x4c sys/kern/uipc_mbuf.c:283 tcp_output(ffff800000ac3100) at tcp_output+0x147a tcp_usrreq(fffffd805da74198,9,fffffd805c36e300,0,0,ffff80001d71d390) at tcp_usrreq+0xa54 sosend(fffffd805da74198,0,ffff80001d6cda08,0,0,80) at sosend+0x669 sys/kern/uipc_socket.c:555 dofilewritev(ffff80001d71d390,4,ffff80001d6cda08,0,ffff80001d6cdaf0) at dofilewritev+0x1ab sys/kern/sys_generic.c:365 sys_write(ffff80001d71d390,ffff80001d6cdaa0,ffff80001d6cdaf0) at sys_write+0x83 sys/kern/sys_generic.c:285 syscall(ffff80001d6cdb70) at syscall+0x507 sys/arch/amd64/amd64/trap.c:570 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffdd290, count: -12 ddb> show registers rdi 0 rsi 0x1 rbp 0xffff80001d6cd3f0 rbx 0xffff80001d6cd4a0 rdx 0x2 rcx 0 rax 0x1 r8 0xffffffff8138bf7f kprintf+0x15f r9 0x1 r10 0x2 r11 0x8a713f57cfd2fc43 r12 0x3000000008 r13 0xffff80001d6cd400 r14 0x100 r15 0x1 rip 0xffffffff813ac878 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff80001d6cd3e0 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb> show proc PROC (sshd) pid=352933 stat=onproc flags process=12 proc=0 pri=24, usrpri=50, nice=20 forw=0xffffffffffffffff, list=0xffff80001d71c760,0xffff80001d71c9e0 process=0xffff80001d7003b8 user=0xffff80001d6c8000, vmspace=0xfffffd806bc09bb0 estcpu=0, cpticks=1, pctcpu=0.0 user=0, sys=1, intr=0 ddb> ps PID TID PPID UID S FLAGS WAIT COMMAND 47701 366564 0 0 3 0x14280 nfsidl nfsio 6161 450262 0 0 3 0x14280 nfsidl nfsio 54038 411446 0 0 3 0x14280 nfsidl nfsio 41369 168576 0 0 3 0x14280 nfsidl nfsio 24557 99486 0 0 3 0x14280 nfsidl nfsio 23986 141115 0 0 3 0x14280 nfsidl nfsio 33163 252692 0 0 3 0x14280 nfsidl nfsio 20406 381936 0 0 3 0x14280 nfsidl nfsio 74513 454422 0 0 3 0x14280 nfsidl nfsio 4152 330501 0 0 3 0x14280 nfsidl nfsio 38568 69921 0 0 3 0x14280 nfsidl nfsio 29689 459669 0 0 3 0x14280 nfsidl nfsio 41212 472687 0 0 3 0x14280 nfsidl nfsio 62545 54717 0 0 3 0x14280 nfsidl nfsio 27958 225840 0 0 3 0x14280 nfsidl nfsio 75888 449321 0 0 3 0x14280 nfsidl nfsio 94924 197474 0 0 3 0x14280 nfsidl nfsio 76145 491850 0 0 3 0x14280 nfsidl nfsio 59998 501116 0 0 3 0x14280 nfsidl nfsio 45243 517578 0 0 3 0x14280 nfsidl nfsio 1103 440341 1 0 3 0x100083 ttyin getty 83973 397398 0 0 3 0x14200 acct acct 19598 51905 0 0 3 0x14200 bored sosplice 67060 280440 53014 0 3 0x82 piperd syz-executor.0 43050 167879 53014 0 3 0x82 piperd syz-executor.1 53014 247375 69535 0 3 0x82 thrsleep syz-fuzzer 53014 340174 69535 0 3 0x4000082 nanosleep syz-fuzzer 53014 142789 69535 0 3 0x4000082 thrsleep syz-fuzzer 53014 77836 69535 0 3 0x4000082 thrsleep syz-fuzzer 53014 13471 69535 0 3 0x4000082 thrsleep syz-fuzzer 53014 487920 69535 0 2 0x4000002 syz-fuzzer 53014 246845 69535 0 3 0x4000082 thrsleep syz-fuzzer 53014 263656 69535 0 3 0x4000082 thrsleep syz-fuzzer 69535 4366 44076 0 3 0x10008a pause ksh *44076 352933 25892 0 7 0x12 sshd 25892 114840 1 0 3 0x80 select sshd 34204 94115 19218 73 3 0x100090 kqread syslogd 19218 376112 1 0 3 0x100082 netio syslogd 6196 162264 1 77 3 0x100090 poll dhclient 57868 507362 1 0 3 0x80 poll dhclient 98909 150011 0 0 3 0x14200 bored smr 86478 495260 0 0 2 0x14200 zerothread 64362 360296 0 0 3 0x14200 aiodoned aiodoned 36725 373618 0 0 3 0x14200 syncer update 23154 299612 0 0 3 0x14200 cleaner cleaner 86449 245661 0 0 3 0x14200 reaper reaper 64338 221885 0 0 3 0x14200 pgdaemon pagedaemon 59901 146316 0 0 3 0x14200 bored crynlk 65820 390068 0 0 3 0x14200 bored crypto 95506 317856 0 0 3 0x40014200 acpi0 acpi0 37203 472321 0 0 3 0x14200 bored softnet 53036 430740 0 0 3 0x14200 bored systqmp 58034 233986 0 0 3 0x14200 bored systq 74070 200824 0 0 3 0x40014200 bored softclock 51413 319700 0 0 3 0x40014200 idle0 1 494601 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb> show all locks No such command ddb> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 9491 6354K 6727K 78643K 11712 0 pcb 13 8K 8K 78643K 362 0 rtable 111 5K 9K 78643K 660 0 ifaddr 83 18K 18K 78643K 246 0 sysctl 2 0K 0K 78643K 2 0 counters 21 16K 16K 78643K 38 0 ioctlops 0 0K 4K 78643K 139 0 iov 0 0K 16K 78643K 119 0 mount 1 1K 1K 78643K 1 0 vnodes 1227 77K 77K 78643K 1684 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 5K 78643K 11 0 VM map 2 0K 0K 78643K 2 0 sem 12 1K 1K 78643K 35 0 dirhash 9 1K 2K 78643K 12 0 ACPI 1809 195K 288K 78643K 12938 0 file desc 4 9K 25K 78643K 1489 0 sigio 0 0K 0K 78643K 31 0 proc 50 38K 55K 78643K 409 0 subproc 32 2K 2K 78643K 36 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 90 0 in_multi 85 3K 3K 78643K 224 0 ether_multi 1 0K 0K 78643K 24 0 mrt 0 0K 0K 78643K 9 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 49 228K 228K 78643K 49 0 exec 0 0K 1K 78643K 256 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 7 26K 26K 78643K 7 0 UVM amap 133 39K 55K 78643K 4219 0 UVM aobj 32 2K 2K 78643K 56 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 0K 78643K 96 0 NDP 10 0K 0K 78643K 40 0 temp 126 3868K 3932K 78643K 15539 0 kqueue 3 4K 10K 78643K 63 0 SYN cache 2 16K 16K 78643K 2 0 ddb> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle arp 64 8 0 2 1 0 1 1 0 8 0 rtpcb 80 53 0 51 1 0 1 1 0 8 0 rtentry 112 95 0 54 2 0 2 2 0 8 0 unpcb 120 1370 0 1360 1 0 1 1 0 8 0 syncache 264 14 0 14 5 5 0 1 0 8 0 sackhl 24 1 0 1 1 1 0 1 0 8 0 tcpqe 32 191 0 191 4 4 0 1 0 8 0 tcpcb 544 242 0 237 1 0 1 1 0 8 0 inpcb 296 3797 0 3789 7 6 1 2 0 8 0 rttmr 72 1 0 1 1 1 0 1 0 8 0 ip6q 72 5 0 5 3 3 0 1 0 8 0 ip6af 40 8 0 8 3 3 0 1 0 8 0 nd6 48 25 0 20 1 0 1 1 0 8 0 pkpcb 40 10 0 10 4 4 0 1 0 8 0 ppxss 1136 6 0 6 5 5 0 1 0 8 0 pfstscr 40 4 0 2 1 0 1 1 0 8 0 pfrktable 1344 102 0 99 5 4 1 1 0 8 0 pftag 88 25 0 24 5 4 1 1 0 8 0 pfstitem 24 4 0 0 1 0 1 1 0 8 0 pfstkey 112 8 0 4 1 0 1 1 0 8 0 pfstate 328 4 0 2 1 0 1 1 0 8 0 pfrule 1360 26 0 22 2 1 1 1 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 394 0 203 13 0 13 13 0 8 0 art_table 32 395 0 203 2 0 2 2 0 8 0 art_node 16 92 0 55 1 0 1 1 0 8 0 sysvmsgpl 40 4 0 1 1 0 1 1 0 8 0 semupl 112 1 0 1 1 1 0 1 0 8 0 semapl 112 33 0 23 1 0 1 1 0 8 0 shmpl 112 53 0 24 1 0 1 1 0 8 0 dirhash 1024 17 0 10 3 1 2 3 0 8 0 dino2pl 256 3384 0 1986 88 0 88 88 0 8 0 ffsino 240 3384 0 1986 83 0 83 83 0 8 0 nchpl 144 5564 0 3975 60 0 60 60 0 8 0 uvmvnodes 72 3678 0 0 67 0 67 67 0 8 0 vnodes 208 3678 0 0 194 0 194 194 0 8 0 namei 1024 15468 0 15468 2 1 1 1 0 8 1 vcpupl 1984 2 0 0 1 0 1 1 0 8 0 vmpool 528 2 0 0 1 0 1 1 0 8 0 pfiaddrpl 120 28 0 26 4 3 1 1 0 8 0 scxspl 192 15751 0 15751 2 1 1 1 0 8 1 plimitpl 152 87 0 80 1 0 1 1 0 8 0 sigapl 424 1698 0 1649 6 0 6 6 0 8 0 futexpl 56 29242 0 29242 1 0 1 1 0 8 1 knotepl 112 133 0 114 1 0 1 1 0 8 0 kqueuepl 144 231 0 229 1 0 1 1 0 8 0 pipepl 272 180 0 170 1 0 1 1 0 8 0 fdescpl 432 1662 0 1649 2 0 2 2 0 8 0 filepl 120 11327 0 11232 7 3 4 4 0 8 1 lockfpl 104 171 0 170 1 0 1 1 0 8 0 lockfspl 48 60 0 59 1 0 1 1 0 8 0 sessionpl 112 18 0 8 1 0 1 1 0 8 0 pgrppl 48 26 0 16 1 0 1 1 0 8 0 ucredpl 96 669 0 662 1 0 1 1 0 8 0 zombiepl 144 1649 0 1649 3 2 1 1 0 8 1 processpl 928 1698 0 1649 7 0 7 7 0 8 0 procpl 624 3477 0 3421 6 1 5 6 0 8 0 sosppl 128 11 0 11 4 4 0 1 0 8 0 sockpl 400 5240 0 5220 15 12 3 4 0 8 0 mcl64k 65536 204 0 204 12 11 1 1 0 8 1 mcl16k 16384 9 0 9 6 6 0 1 0 8 0 mcl12k 12288 39 0 39 14 14 0 1 0 8 0 mcl9k 9216 13 0 13 7 7 0 1 0 8 0 mcl8k 8192 39 0 39 15 15 0 1 0 8 0 mcl4k 4096 89 0 89 11 10 1 1 0 8 1 mcl2k2 2112 7 0 7 4 4 0 1 0 8 0 mcl2k 2048 93056 0 93003 22 14 8 14 0 8 0 mtagpl 96 71 0 23 3 1 2 2 0 8 0 mbufpl 256 155875 0 155642 23 7 16 17 0 8 0 mbufpl: pool(0xffffffff82855e90:mbufpl): free list modified: page 0xfffffd805c36e000; item ordinal 0; addr 0xfffffd805c36e400 (p 0xfffffd805a261000); offset 0x0=0x0 pool(mbufpl): free list modified: page 0xfffffd805c36e000; item ordinal 0; addr 0xfffffd805c36e400 (p 0xfffffd805a261000); offset 0x0=0x0 mbufpl: pool(0xffffffff82855e90:mbufpl): page inconsistency: page 0xfffffd805c36e000; item ordinal 1; addr 0xe069707cfb5cef97 bufpl 280 5420 0 129 378 0 378 378 0 8 0 anonpl 16 160309 0 144545 110 40 70 83 0 107 0 amapchunkpl 152 7476 0 7321 36 29 7 21 0 158 0 amappl16 192 8181 0 7187 81 31 50 58 0 8 0 amappl15 184 1 0 0 1 0 1 1 0 8 0 amappl14 176 393 0 388 1 0 1 1 0 8 0 amappl13 168 1118 0 1114 1 0 1 1 0 8 0 amappl12 160 7 0 6 2 1 1 1 0 8 0 amappl11 152 51 0 40 1 0 1 1 0 8 0 amappl10 144 25 0 20 1 0 1 1 0 8 0 amappl9 136 379 0 377 1 0 1 1 0 8 0 amappl8 128 351 0 305 2 0 2 2 0 8 0 amappl7 120 107 0 95 1 0 1 1 0 8 0 amappl6 112 23 0 18 1 0 1 1 0 8 0 amappl5 104 1619 0 1609 1 0 1 1 0 8 0 amappl4 96 1512 0 1485 1 0 1 1 0 8 0 amappl3 88 520 0 515 1 0 1 1 0 8 0 amappl2 80 12489 0 12431 2 0 2 2 0 8 0 amappl1 72 45476 0 45078 23 14 9 17 0 8 0 amappl 80 3715 0 3671 3 1 2 2 0 84 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 64 55 0 24 1 0 1 1 0 8 0 uaddrrnd 24 1664 0 1649 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 1664 0 1649 1 0 1 1 0 8 0 vmmpekpl 168 13028 0 12994 2 0 2 2 0 8 0 vmmpepl 168 206874 0 204877 144 51 93 118 0 357 1 vmsppl 272 1663 0 1649 3 1 2 2 0 8 0 pdppl 4096 3334 0 3300 6 1 5 6 0 8 0 pvpl 32 438105 0 419285 268 102 166 195 0 265 0 pmappl 200 1663 0 1649 1 0 1 1 0 8 0 extentpl 40 53 0 36 1 0 1 1 0 8 0 phpool 112 315 0 83 7 0 7 7 0 8 0 ddb> machine ddbcpu 0 No such command ddb> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic(ffffffff8246ccb8) at panic+0x15c sys/kern/subr_prf.c:207 pool_do_get(ffffffff82855e90,2,ffff80001d6cd588) at pool_do_get+0x42a sys/kern/subr_pool.c:738 pool_get(ffffffff82855e90,2) at pool_get+0xb5 sys/kern/subr_pool.c:581 m_gethdr(2,2) at m_gethdr+0x4c sys/kern/uipc_mbuf.c:283 tcp_output(ffff800000ac3100) at tcp_output+0x147a tcp_usrreq(fffffd805da74198,9,fffffd805c36e300,0,0,ffff80001d71d390) at tcp_usrreq+0xa54 sosend(fffffd805da74198,0,ffff80001d6cda08,0,0,80) at sosend+0x669 sys/kern/uipc_socket.c:555 dofilewritev(ffff80001d71d390,4,ffff80001d6cda08,0,ffff80001d6cdaf0) at dofilewritev+0x1ab sys/kern/sys_generic.c:365 sys_write(ffff80001d71d390,ffff80001d6cdaa0,ffff80001d6cdaf0) at sys_write+0x83 sys/kern/sys_generic.c:285 syscall(ffff80001d6cdb70) at syscall+0x507 sys/arch/amd64/amd64/trap.c:570 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffdd290, count: -12 ddb> machine ddbcpu 1 No such command ddb> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic(ffffffff8246ccb8) at panic+0x15c sys/kern/subr_prf.c:207 pool_do_get(ffffffff82855e90,2,ffff80001d6cd588) at pool_do_get+0x42a sys/kern/subr_pool.c:738 pool_get(ffffffff82855e90,2) at pool_get+0xb5 sys/kern/subr_pool.c:581 m_gethdr(2,2) at m_gethdr+0x4c sys/kern/uipc_mbuf.c:283 tcp_output(ffff800000ac3100) at tcp_output+0x147a tcp_usrreq(fffffd805da74198,9,fffffd805c36e300,0,0,ffff80001d71d390) at tcp_usrreq+0xa54 sosend(fffffd805da74198,0,ffff80001d6cda08,0,0,80) at sosend+0x669 sys/kern/uipc_socket.c:555 dofilewritev(ffff80001d71d390,4,ffff80001d6cda08,0,ffff80001d6cdaf0) at dofilewritev+0x1ab sys/kern/sys_generic.c:365 sys_write(ffff80001d71d390,ffff80001d6cdaa0,ffff80001d6cdaf0) at sys_write+0x83 sys/kern/sys_generic.c:285 syscall(ffff80001d6cdb70) at syscall+0x507 sys/arch/amd64/amd64/trap.c:570 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffdd290, count: -12