panic: pool_do_get: mbufpl free list modified: page 0xfffffd8076ce3000; item addr 0xfffffd8076ce3300; offset 0x0=0x70003efff != 0xc26618410becf51e Stopped at db_enter+0x1c: addq $0x8,%rsp TID PID UID PRFLAGS PFLAGS CPU COMMAND *210746 8862 0 0x1a000002 0x4000000 0 syz-fuzzer db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 panic(ffffffff82859f69) at panic+0x165 sys/kern/subr_prf.c:198 pool_do_get(ffffffff82d63e00,2,ffff80002a6d57bc) at pool_do_get+0x434 pool_get(ffffffff82d63e00,2) at pool_get+0xba sys/kern/subr_pool.c:582 m_gethdr(2,2) at m_gethdr+0x67 sys/kern/uipc_mbuf.c:276 tcp_output(ffff800000dcd328) at tcp_output+0x15af sys/netinet/tcp_output.c:689 tcp_send(fffffd806e5fcdd8,fffffd805b6cb500,0,0) at tcp_send+0xfd sys/netinet/tcp_usrreq.c:841 sosend(fffffd806e5fcdd8,0,ffff80002a6d5be8,0,0,80) at sosend+0x7da dofilewritev(ffff80002a608538,6,ffff80002a6d5be8,0,ffff80002a6d5ca0) at dofilewritev+0x1a9 sys/kern/sys_generic.c:375 sys_write(ffff80002a608538,ffff80002a6d5d50,ffff80002a6d5ca0) at sys_write+0x87 sys/kern/sys_generic.c:295 syscall(ffff80002a6d5d50) at syscall+0x72a sys/arch/amd64/amd64/trap.c:577 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x298091560, count: 3 https://www.openbsd.org/ddb.html describes the minimum info required in bug reports. Insufficient info makes it difficult to find and fix bugs. ddb> ddb> set $lines = 0 ddb> set $maxwidth = 0 ddb> show panic *cpu0: pool_do_get: mbufpl free list modified: page 0xfffffd8076ce3000; item addr 0xfffffd8076ce3300; offset 0x0=0x70003efff != 0xc26618410becf51e ddb> trace db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 panic(ffffffff82859f69) at panic+0x165 sys/kern/subr_prf.c:198 pool_do_get(ffffffff82d63e00,2,ffff80002a6d57bc) at pool_do_get+0x434 pool_get(ffffffff82d63e00,2) at pool_get+0xba sys/kern/subr_pool.c:582 m_gethdr(2,2) at m_gethdr+0x67 sys/kern/uipc_mbuf.c:276 tcp_output(ffff800000dcd328) at tcp_output+0x15af sys/netinet/tcp_output.c:689 tcp_send(fffffd806e5fcdd8,fffffd805b6cb500,0,0) at tcp_send+0xfd sys/netinet/tcp_usrreq.c:841 sosend(fffffd806e5fcdd8,0,ffff80002a6d5be8,0,0,80) at sosend+0x7da dofilewritev(ffff80002a608538,6,ffff80002a6d5be8,0,ffff80002a6d5ca0) at dofilewritev+0x1a9 sys/kern/sys_generic.c:375 sys_write(ffff80002a608538,ffff80002a6d5d50,ffff80002a6d5ca0) at sys_write+0x87 sys/kern/sys_generic.c:295 syscall(ffff80002a6d5d50) at syscall+0x72a sys/arch/amd64/amd64/trap.c:577 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x298091560, count: -12 ddb> show registers rdi 0 rsi 0x1 rbp 0xffff80002a6d5630 rbx 0xfffffd8076ce3300 rdx 0 rcx 0 rax 0xffff80002a608538 r8 0x101010101010101 r9 0x8080808080808080 r10 0xc93ccf792a092682 r11 0x2d291c7524114f29 r12 0 r13 0xfffffd806a83f700 r14 0 r15 0x1 rip 0xffffffff81755a4c db_enter+0x1c cs 0x8 rflags 0x246 rsp 0xffff80002a6d5620 ss 0x10 db_enter+0x1c: addq $0x8,%rsp ddb> show proc PROC (syz-fuzzer) tid=210746 pid=8862 tcnt=15 stat=onproc flags process=1a000002 proc=4000000 runpri=24, usrpri=54, slppri=24, nice=20 wchan=0x0, wmesg=, ps_single=0x0 forw=0xffffffffffffffff, list=0xffff80002a609208,0xffff80002a608028 process=0xffff8000ffff6e20 user=0xffff80002a6d0000, vmspace=0xfffffd807f01c408 estcpu=4, cpticks=2, pctcpu=0.25, user=1, sys=1, intr=0 ddb> ps PID TID PPID UID S FLAGS WAIT COMMAND 90922 505737 28161 0 2 0x8000000 syz-executor.0 13775 316848 67727 0 2 0x8000000 syz-executor.4 13775 177872 67727 0 3 0xc000080 fsleep syz-executor.4 11116 143272 61878 0 2 0x8000000 syz-executor.7 11116 157521 61878 0 3 0xc000080 fsleep syz-executor.7 6183 34914 93300 0 2 0x8000000 syz-executor.6 6183 82419 93300 0 2 0xc000000 syz-executor.6 49367 397355 36632 0 2 0x8000480 syz-executor.1 49367 292166 36632 0 3 0xc000080 kqread syz-executor.1 49367 485367 36632 0 3 0xc000080 fsleep syz-executor.1 49367 503258 36632 0 3 0xc000080 fsleep syz-executor.1 86519 188635 0 0 3 0x14200 acct acct 13632 451222 8862 0 2 0x8000002 syz-executor.5 36632 77186 8862 0 2 0x8000482 syz-executor.1 46343 163922 8862 0 2 0x8000002 syz-executor.2 30115 350483 8862 0 2 0x8000002 syz-executor.3 28161 147212 8862 0 2 0x8000482 syz-executor.0 67727 455594 8862 0 2 0x8000482 syz-executor.4 93300 40872 8862 0 2 0x8000482 syz-executor.6 61878 263384 8862 0 2 0x8000482 syz-executor.7 95787 353191 1 0 3 0x18100083 ttyin getty 19779 226986 0 0 3 0x14280 nfsidl nfsio 76740 102677 0 0 3 0x14280 nfsidl nfsio 28084 505915 0 0 3 0x14280 nfsidl nfsio 11408 433122 0 0 3 0x14280 nfsidl nfsio 24630 71708 0 0 3 0x14280 nfsidl nfsio 51504 131850 0 0 3 0x14280 nfsidl nfsio 81397 485276 0 0 3 0x14280 nfsidl nfsio 4980 323272 0 0 3 0x14280 nfsidl nfsio 55920 371873 0 0 3 0x14280 nfsidl nfsio 80444 519802 0 0 3 0x14280 nfsidl nfsio 87467 277612 0 0 3 0x14280 nfsidl nfsio 24003 82607 0 0 3 0x14280 nfsidl nfsio 15283 181129 0 0 3 0x14280 nfsidl nfsio 8740 110459 0 0 3 0x14280 nfsidl nfsio 30386 75020 0 0 3 0x14280 nfsidl nfsio 27479 315965 0 0 3 0x14280 nfsidl nfsio 89812 133374 0 0 3 0x14280 nfsidl nfsio 49647 417359 0 0 3 0x14280 nfsidl nfsio 16108 375285 0 0 3 0x14280 nfsidl nfsio 84050 6182 0 0 3 0x14280 nfsidl nfsio 52920 142967 0 0 3 0x14200 bored sosplice 8862 217190 21231 0 3 0x1a000082 thrsleep syz-fuzzer 8862 274968 21231 0 2 0x1e000482 syz-fuzzer 8862 235508 21231 0 3 0x1e000082 wait syz-fuzzer 8862 235017 21231 0 3 0x1e000082 wait syz-fuzzer 8862 329661 21231 0 3 0x1e000082 thrsleep syz-fuzzer 8862 450904 21231 0 3 0x1e000082 wait syz-fuzzer 8862 355117 21231 0 3 0x1e000082 wait syz-fuzzer * 8862 210746 21231 0 7 0x1e000002 syz-fuzzer 8862 116399 21231 0 3 0x1e000082 wait syz-fuzzer 8862 444945 21231 0 3 0x1e000082 wait syz-fuzzer 8862 463499 21231 0 3 0x1e000082 thrsleep syz-fuzzer 8862 127468 21231 0 3 0x1e000082 wait syz-fuzzer 8862 106534 21231 0 3 0x1e000082 wait syz-fuzzer 8862 225444 21231 0 3 0x1e000082 thrsleep syz-fuzzer 8862 235835 21231 0 3 0x1e000082 thrsleep syz-fuzzer 21231 306031 52776 0 3 0x810008a sigsusp ksh 52776 295157 45887 0 3 0x1800009a kqread sshd 45887 211302 1 0 3 0x18000088 kqread sshd 45187 324430 29240 73 2 0x19100010 syslogd 29240 209090 1 0 3 0x18100082 sbwait syslogd 60961 169590 1 0 3 0x18100080 kqread resolvd 97720 165453 90327 77 3 0x18100092 kqread dhcpleased 56325 57451 90327 77 2 0x18100492 dhcpleased 90327 132778 1 0 3 0x18000080 kqread dhcpleased 87926 399777 0 0 2 0x14200 smr 96066 191780 0 0 2 0x14200 zerothread 80180 491867 0 0 3 0x14200 aiodoned aiodoned 17476 383153 0 0 3 0x14200 syncer update 63624 471864 0 0 3 0x14200 cleaner cleaner 67050 80553 0 0 3 0x14200 reaper reaper 27037 467934 0 0 3 0x14200 pgdaemon pagedaemon 51841 507575 0 0 3 0x14200 bored viomb 80448 206446 0 0 3 0x40014200 acpi0 acpi0 13819 516264 0 0 3 0x14200 bored softnet3 88913 238689 0 0 3 0x14200 bored softnet2 29790 383571 0 0 3 0x14200 bored softnet1 56183 345572 0 0 2 0x14200 softnet0 3403 1745 0 0 3 0x14200 bored systqmp 19192 502994 0 0 3 0x14200 bored systq 32520 105405 0 0 3 0x40014200 tmoslp softclock 93190 344777 0 0 3 0x40014200 idle0 1 157452 0 0 3 0x8080082 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb> show all locks No such command ddb> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10224 6883K 7189K 166960K 13757 0 pcb 19 13K 13K 166960K 208 0 rtable 238 8K 9K 166960K 1682 0 pf 33 9K 10K 166960K 143 0 ifaddr 47 12K 12K 166960K 220 0 ifgroup 58 2K 2K 166960K 262 0 sysctl 4 1K 1K 166960K 6 0 counters 32 17K 17K 166960K 82 0 ioctlops 0 0K 2K 166960K 152 0 iov 0 0K 18K 166960K 96 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1388 87K 87K 166960K 3110 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 1K 9K 166960K 32 0 VM map 2 1K 1K 166960K 2 0 sem 12 0K 1K 166960K 133 0 dirhash 12 2K 2K 166960K 33 0 ACPI 1697 195K 286K 166960K 12548 0 file desc 15 53K 89K 166960K 1937 0 sigio 0 0K 0K 166960K 34 0 proc 58 59K 116K 166960K 1725 0 subproc 104 6K 7K 166960K 599 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 1 0K 0K 166960K 196 0 in_multi 95 7K 7K 166960K 560 0 ether_multi 1 0K 0K 166960K 4 0 mrt 0 0K 0K 166960K 1 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 151 678K 678K 166960K 151 0 exec 0 0K 1K 166960K 1023 0 pfkey data 0 0K 0K 166960K 3 0 tdb 3 0K 0K 166960K 3 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 265 78K 98K 166960K 17053 0 UVM aobj 49 6K 6K 166960K 56 0 pinsyscall 35 70K 100K 166960K 4100 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 0K 166960K 100 0 NDP 13 0K 2K 166960K 156 0 temp 75 6812K 14748K 166960K 72703 0 kqueue 13 20K 28K 166960K 235 0 SYN cache 2 16K 16K 166960K 2 0 ddb> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle rtpcb 120 364 0 361 3 0 3 3 0 8 2 rtentry 112 577 0 471 4 0 4 4 0 8 0 unpcb 144 1039 0 1023 4 0 4 4 0 8 3 syncache 336 12 0 12 1 0 1 1 0 8 1 sackhl 24 1 0 1 1 0 1 1 0 8 1 tcpqe 32 6 0 6 1 0 1 1 0 8 1 tcpcb 808 556 0 549 5 0 5 5 0 8 4 arp 88 105 0 87 1 0 1 1 0 8 0 ipq 40 5 0 4 1 0 1 1 0 8 0 ipqe 40 8 0 7 1 0 1 1 0 8 0 inpcb 352 1866 0 1853 5 0 5 5 0 8 3 nd6 104 146 0 124 1 0 1 1 0 8 0 pkpcb 40 7 0 7 1 0 1 1 0 8 1 kcovpl 48 46 0 38 1 0 1 1 0 8 0 ppxss 1072 5 0 5 1 0 1 1 0 8 1 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 2255 0 1814 45 17 28 29 0 8 0 art_table 32 2256 0 1814 4 0 4 4 0 8 0 art_node 16 572 0 476 1 0 1 1 0 8 0 sysvmsgpl 40 20 0 13 1 0 1 1 0 8 0 semupl 112 2 0 2 1 0 1 1 0 8 1 semapl 112 129 0 119 1 0 1 1 0 8 0 shmpl 112 53 0 7 2 0 2 2 0 8 0 dirhash 1024 31 0 14 3 0 3 3 0 8 0 dino2pl 256 4105 0 2590 96 0 96 96 0 8 0 ffsino 240 4105 0 2590 90 0 90 90 0 8 0 nchpl 144 6687 0 6075 67 33 34 67 0 8 8 uvmvnodes 80 5416 0 0 111 0 111 111 0 8 0 vnodes 216 5416 0 0 301 0 301 301 0 8 0 namei 1024 25828 0 25828 3 0 3 3 0 8 3 vcpupl 3904 3 0 1 1 0 1 1 0 8 0 vmpool 664 8 0 6 1 0 1 1 0 8 0 kstatmem 264 126 0 100 2 0 2 2 0 8 0 scsiplug 72 2 0 2 1 0 1 1 0 8 1 scxspl 216 39489 0 39489 8 0 8 8 1 8 8 plimitpl 152 287 0 272 1 0 1 1 0 8 0 sigapl 424 2186 0 2122 9 0 9 9 0 8 0 futexpl 64 27263 0 27259 1 0 1 1 0 8 0 knotepl 120 10304 0 10217 11 0 11 11 0 8 7 kqueuepl 184 506 0 497 4 0 4 4 0 8 3 pipepl 288 434 0 406 3 0 3 3 0 8 0 fdescpl 432 2147 0 2121 4 0 4 4 0 8 0 filepl 120 12962 0 12715 14 0 14 14 0 8 5 lockfpl 104 369 0 367 1 0 1 1 0 8 0 lockfspl 48 171 0 169 1 0 1 1 0 8 0 sessionpl 144 65 0 49 1 0 1 1 0 8 0 pgrppl 48 95 0 79 1 0 1 1 0 8 0 ucredpl 104 2243 0 2233 1 0 1 1 0 8 0 zombiepl 144 2122 0 2122 1 0 1 1 0 8 1 processpl 1072 2186 0 2122 5 0 5 5 0 8 0 procpl 656 3726 0 3642 8 0 8 8 0 8 0 sosppl 168 34 0 34 1 0 1 1 0 8 1 sockpl 504 3324 0 3292 18 6 12 14 0 8 8 mcl64k 65536 17 0 17 1 0 1 1 0 8 1 mcl12k 12288 2 0 2 1 0 1 1 0 8 1 mcl9k 9216 1 0 1 1 0 1 1 0 8 1 mcl8k 8192 37 0 37 1 0 1 1 0 8 1 mcl4k 4096 10 0 10 1 0 1 1 0 8 1 mcl2k 2048 27522 0 27422 43 23 20 39 0 8 5 mtagpl 96 50 0 45 1 0 1 1 0 8 0 mbufpl 256 65710 0 65390 127 105 22 63 0 8 2 mbufpl: pool(0xffffffff82d63e00:mbufpl): free list modified: page 0xfffffd8076ce3000; item ordinal 0; addr 0xfffffd8076ce3300 (p 0xfffffd806a83f000); offset 0x0=0x70003efff pool(mbufpl): free list modified: page 0xfffffd8076ce3000; item ordinal 0; addr 0xfffffd8076ce3300 (p 0xfffffd806a83f000); offset 0x0=0x7 mbufpl: pool(0xffffffff82d63e00:mbufpl): page inconsistency: page 0xfffffd8076ce3000; item ordinal 1; addr 0xbd4470325b271bd4 bufpl 280 10725 0 2987 553 0 553 553 0 8 0 anonpl 24 365137 0 359193 68 0 68 68 0 188 22 amapchunkpl 152 55587 0 54956 39 0 39 39 0 158 10 amappl16 200 7953 0 7842 19 4 15 19 0 8 6 amappl15 192 11 0 11 1 0 1 1 0 8 1 amappl14 184 278 0 266 2 0 2 2 0 8 1 amappl13 176 33 0 33 1 0 1 1 0 8 1 amappl12 168 3344 0 3318 2 0 2 2 0 8 0 amappl11 160 56 0 44 1 0 1 1 0 8 0 amappl10 152 92 0 83 1 0 1 1 0 8 0 amappl9 144 168 0 168 1 0 1 1 0 8 1 amappl8 136 253 0 221 2 0 2 2 0 8 0 amappl7 128 61 0 45 1 0 1 1 0 8 0 amappl6 120 889 0 874 2 0 2 2 0 8 1 amappl5 112 319 0 306 1 0 1 1 0 8 0 amappl4 104 782 0 752 2 0 2 2 0 8 1 amappl3 96 10897 0 10829 3 0 3 3 0 8 0 amappl2 88 2656 0 2586 4 0 4 4 0 8 2 amappl1 80 17618 0 17127 22 3 19 22 0 8 6 amappl 88 16142 0 15962 5 0 5 5 0 92 0 dma4096 4096 1 0 1 1 0 1 1 0 8 1 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 0 1 1 0 8 1 dma128 128 253 0 253 1 0 1 1 0 8 1 dma64 64 6 0 6 1 0 1 1 0 8 1 dma32 32 7 0 7 1 0 1 1 0 8 1 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 55 0 7 1 0 1 1 0 8 0 uaddrrnd 24 2155 0 2127 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 2155 0 2127 1 0 1 1 0 8 0 vmmpekpl 168 19472 0 19397 4 0 4 4 0 8 0 vmmpepl 168 155384 0 153631 111 0 111 111 0 357 24 vmsppl 344 2154 0 2127 4 0 4 4 0 8 0 rwobjpl 24 47216 0 40720 40 0 40 40 0 8 0 pdppl 4096 4316 0 4256 191 125 66 78 0 8 6 pvpl 32 986650 0 974774 365 15 350 365 0 265 238 pmappl 216 2154 0 2127 3 0 3 3 0 8 0 extentpl 40 56 0 38 1 0 1 1 0 8 0 phpool 112 632 0 276 12 0 12 12 0 8 0 ddb> machine ddbcpu 0 No such command ddb> trace db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 panic(ffffffff82859f69) at panic+0x165 sys/kern/subr_prf.c:198 pool_do_get(ffffffff82d63e00,2,ffff80002a6d57bc) at pool_do_get+0x434 pool_get(ffffffff82d63e00,2) at pool_get+0xba sys/kern/subr_pool.c:582 m_gethdr(2,2) at m_gethdr+0x67 sys/kern/uipc_mbuf.c:276 tcp_output(ffff800000dcd328) at tcp_output+0x15af sys/netinet/tcp_output.c:689 tcp_send(fffffd806e5fcdd8,fffffd805b6cb500,0,0) at tcp_send+0xfd sys/netinet/tcp_usrreq.c:841 sosend(fffffd806e5fcdd8,0,ffff80002a6d5be8,0,0,80) at sosend+0x7da dofilewritev(ffff80002a608538,6,ffff80002a6d5be8,0,ffff80002a6d5ca0) at dofilewritev+0x1a9 sys/kern/sys_generic.c:375 sys_write(ffff80002a608538,ffff80002a6d5d50,ffff80002a6d5ca0) at sys_write+0x87 sys/kern/sys_generic.c:295 syscall(ffff80002a6d5d50) at syscall+0x72a sys/arch/amd64/amd64/trap.c:577 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x298091560, count: -12 ddb> machine ddbcpu 1 No such command ddb> trace db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 panic(ffffffff82859f69) at panic+0x165 sys/kern/subr_prf.c:198 pool_do_get(ffffffff82d63e00,2,ffff80002a6d57bc) at pool_do_get+0x434 pool_get(ffffffff82d63e00,2) at pool_get+0xba sys/kern/subr_pool.c:582 m_gethdr(2,2) at m_gethdr+0x67 sys/kern/uipc_mbuf.c:276 tcp_output(ffff800000dcd328) at tcp_output+0x15af sys/netinet/tcp_output.c:689 tcp_send(fffffd806e5fcdd8,fffffd805b6cb500,0,0) at tcp_send+0xfd sys/netinet/tcp_usrreq.c:841 sosend(fffffd806e5fcdd8,0,ffff80002a6d5be8,0,0,80) at sosend+0x7da dofilewritev(ffff80002a608538,6,ffff80002a6d5be8,0,ffff80002a6d5ca0) at dofilewritev+0x1a9 sys/kern/sys_generic.c:375 sys_write(ffff80002a608538,ffff80002a6d5d50,ffff80002a6d5ca0) at sys_write+0x87 sys/kern/sys_generic.c:295 syscall(ffff80002a6d5d50) at syscall+0x72a sys/arch/amd64/amd64/trap.c:577 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x298091560, count: -12