panic: pool_do_get: mbufpl free list modified: page 0xfffffd80533a7000; item addr 0xfffffd80533a7d00; offset 0x0=0x21cf0f1fbb580000 != 0x21cf0f1fbb58a50f Stopped at db_enter+0x18: addq $0x8,%rsp TID PID UID PRFLAGS PFLAGS CPU COMMAND * 81051 92585 0 0x12 0 0 sshd db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic(ffffffff82474430) at panic+0x15c sys/kern/subr_prf.c:207 pool_do_get(ffffffff827ef020,1,ffff80001d722ca8) at pool_do_get+0x42a sys/kern/subr_pool.c:738 pool_get(ffffffff827ef020,1) at pool_get+0xb5 sys/kern/subr_pool.c:581 m_get(1,1) at m_get+0x4c sys/kern/uipc_mbuf.c:250 m_getuio(ffff80001d722df8,0,4200,ffff80001d722f68) at m_getuio+0xbe sys/kern/uipc_socket.c:592 sosend(fffffd805da64648,0,ffff80001d722f68,0,0,80) at sosend+0x54e sys/kern/uipc_socket.c:542 dofilewritev(ffff80001d71d600,4,ffff80001d722f68,0,ffff80001d723050) at dofilewritev+0x1ab sys/kern/sys_generic.c:365 sys_write(ffff80001d71d600,ffff80001d723000,ffff80001d723050) at sys_write+0x83 sys/kern/sys_generic.c:285 syscall(ffff80001d7230d0) at syscall+0x507 sys/arch/amd64/amd64/trap.c:570 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffe6c10, count: 4 https://www.openbsd.org/ddb.html describes the minimum info required in bug reports. Insufficient info makes it difficult to find and fix bugs. ddb> ddb> set $lines = 0 ddb> set $maxwidth = 0 ddb> show panic pool_do_get: mbufpl free list modified: page 0xfffffd80533a7000; item addr 0xfffffd80533a7d00; offset 0x0=0x21cf0f1fbb580000 != 0x21cf0f1fbb58a50f ddb> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic(ffffffff82474430) at panic+0x15c sys/kern/subr_prf.c:207 pool_do_get(ffffffff827ef020,1,ffff80001d722ca8) at pool_do_get+0x42a sys/kern/subr_pool.c:738 pool_get(ffffffff827ef020,1) at pool_get+0xb5 sys/kern/subr_pool.c:581 m_get(1,1) at m_get+0x4c sys/kern/uipc_mbuf.c:250 m_getuio(ffff80001d722df8,0,4200,ffff80001d722f68) at m_getuio+0xbe sys/kern/uipc_socket.c:592 sosend(fffffd805da64648,0,ffff80001d722f68,0,0,80) at sosend+0x54e sys/kern/uipc_socket.c:542 dofilewritev(ffff80001d71d600,4,ffff80001d722f68,0,ffff80001d723050) at dofilewritev+0x1ab sys/kern/sys_generic.c:365 sys_write(ffff80001d71d600,ffff80001d723000,ffff80001d723050) at sys_write+0x83 sys/kern/sys_generic.c:285 syscall(ffff80001d7230d0) at syscall+0x507 sys/arch/amd64/amd64/trap.c:570 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffe6c10, count: -11 ddb> show registers rdi 0 rsi 0x1 rbp 0xffff80001d722b10 rbx 0xffff80001d722bc0 rdx 0x2 rcx 0 rax 0x1 r8 0xffffffff8228533f kprintf+0x15f r9 0x1 r10 0x2 r11 0x218453a8da81718d r12 0x3000000008 r13 0xffff80001d722b20 r14 0x100 r15 0x1 rip 0xffffffff81cfcf18 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff80001d722b00 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb> show proc PROC (sshd) pid=81051 stat=onproc flags process=12 proc=0 pri=24, usrpri=50, nice=20 forw=0xffffffffffffffff, list=0xffff80001d71dae0,0xffff80001d71d130 process=0xffff80001d707238 user=0xffff80001d71e000, vmspace=0xfffffd806bc09220 estcpu=0, cpticks=1, pctcpu=0.0 user=0, sys=1, intr=0 ddb> ps PID TID PPID UID S FLAGS WAIT COMMAND 51118 392165 197 0 2 0 syz-executor.1 51118 245150 197 0 2 0x4000000 syz-executor.1 17431 218772 0 0 3 0x14200 acct acct 59588 361352 0 0 3 0x14280 nfsidl nfsio 44606 327943 0 0 3 0x14280 nfsidl nfsio 54507 118322 0 0 3 0x14280 nfsidl nfsio 91839 162916 0 0 3 0x14280 nfsidl nfsio 61046 95097 0 0 3 0x14280 nfsidl nfsio 2274 144200 0 0 3 0x14280 nfsidl nfsio 72747 192851 0 0 3 0x14280 nfsidl nfsio 28151 316941 0 0 3 0x14280 nfsidl nfsio 34464 170009 0 0 3 0x14280 nfsidl nfsio 40100 228867 0 0 3 0x14280 nfsidl nfsio 14790 282462 0 0 3 0x14280 nfsidl nfsio 52645 252159 0 0 3 0x14280 nfsidl nfsio 48526 360860 0 0 3 0x14280 nfsidl nfsio 38445 18488 0 0 3 0x14280 nfsidl nfsio 77640 143963 0 0 3 0x14280 nfsidl nfsio 83700 108623 0 0 3 0x14280 nfsidl nfsio 11775 190297 0 0 3 0x14280 nfsidl nfsio 39386 102509 0 0 3 0x14280 nfsidl nfsio 33268 423449 0 0 3 0x14280 nfsidl nfsio 47465 504586 0 0 3 0x14280 nfsidl nfsio 55246 372779 0 0 3 0x14200 bored sosplice 29051 346761 91468 0 2 0x2 syz-executor.0 197 326430 91468 0 2 0x482 syz-executor.1 91468 309233 38831 0 3 0x82 thrsleep syz-fuzzer 91468 208630 38831 0 2 0x4000482 syz-fuzzer 91468 485276 38831 0 3 0x4000082 thrsleep syz-fuzzer 91468 256083 38831 0 3 0x4000082 thrsleep syz-fuzzer 91468 243868 38831 0 2 0x4000002 syz-fuzzer 91468 392469 38831 0 3 0x4000082 thrsleep syz-fuzzer 91468 434463 38831 0 3 0x4000082 thrsleep syz-fuzzer 91468 165343 38831 0 3 0x4000082 thrsleep syz-fuzzer 38831 449035 92585 0 3 0x10008a pause ksh *92585 81051 96491 0 7 0x12 sshd 40178 432064 1 0 3 0x100083 ttyin getty 96491 170458 1 0 3 0x80 select sshd 18050 204512 4233 73 3 0x100090 kqread syslogd 4233 23222 1 0 3 0x100082 netio syslogd 30718 106152 1 77 3 0x100090 poll dhclient 20229 210874 1 0 3 0x80 poll dhclient 59546 333149 0 0 3 0x14200 bored smr 87241 515393 0 0 2 0x14200 zerothread 97205 466708 0 0 3 0x14200 aiodoned aiodoned 94989 259409 0 0 3 0x14200 syncer update 96828 402530 0 0 3 0x14200 cleaner cleaner 54888 276321 0 0 3 0x14200 reaper reaper 26516 49739 0 0 3 0x14200 pgdaemon pagedaemon 46431 447914 0 0 3 0x14200 bored crynlk 32785 166954 0 0 3 0x14200 bored crypto 50048 185960 0 0 3 0x40014200 acpi0 acpi0 493 512440 0 0 3 0x14200 bored softnet 83403 184506 0 0 3 0x14200 bored systqmp 81740 164215 0 0 3 0x14200 bored systq 13869 369224 0 0 3 0x40014200 bored softclock 8074 310435 0 0 3 0x40014200 idle0 1 31494 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb> show all locks No such command ddb> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 9488 6347K 6730K 78643K 11589 0 pcb 13 8K 8K 78643K 323 0 rtable 132 11K 11K 78643K 590 0 ifaddr 78 17K 18K 78643K 205 0 sysctl 2 0K 0K 78643K 4 0 counters 21 16K 16K 78643K 31 0 ioctlops 0 0K 4K 78643K 217 0 iov 0 0K 16K 78643K 246 0 mount 1 1K 1K 78643K 1 0 vnodes 1218 77K 77K 78643K 1542 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 5K 78643K 5 0 VM map 2 0K 0K 78643K 2 0 sem 12 0K 1K 78643K 108 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1809 195K 288K 78643K 12938 0 file desc 5 13K 25K 78643K 1165 0 sigio 0 0K 0K 78643K 4 0 proc 50 38K 54K 78643K 435 0 subproc 32 2K 2K 78643K 34 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 361 0 in_multi 55 2K 3K 78643K 205 0 ether_multi 1 0K 0K 78643K 15 0 mrt 1 0K 0K 78643K 5 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 61 281K 281K 78643K 61 0 exec 0 0K 1K 78643K 251 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 7 26K 26K 78643K 7 0 UVM amap 138 121K 141K 78643K 3437 0 UVM aobj 18 4K 4K 78643K 27 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 0K 78643K 70 0 NDP 10 0K 0K 78643K 37 0 temp 126 3870K 3934K 78643K 16077 0 kqueue 3 4K 14K 78643K 52 0 SYN cache 2 16K 16K 78643K 2 0 ddb> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle arp 64 7 0 1 1 0 1 1 0 8 0 rtpcb 80 61 0 59 1 0 1 1 0 8 0 rtentry 112 74 0 33 2 0 2 2 0 8 0 unpcb 120 461 0 453 1 0 1 1 0 8 0 syncache 264 8 0 8 3 3 0 1 0 8 0 tcpqe 32 465 0 465 2 2 0 1 0 8 0 tcpcb 544 234 0 228 1 0 1 1 0 8 0 ipq 40 2 0 2 1 1 0 1 0 8 0 ipqe 40 8 0 8 1 1 0 1 0 8 0 inpcb 296 1263 0 1254 8 6 2 2 0 8 1 rttmr 72 1 0 1 1 1 0 1 0 8 0 nd6 48 20 0 16 1 0 1 1 0 8 0 pkpcb 40 6 0 6 1 1 0 1 0 8 0 pfrktable 1344 115 0 99 4 2 2 2 0 8 0 pftag 88 19 0 14 1 0 1 1 0 8 0 pfqueue 264 2 0 0 1 0 1 1 0 8 0 pfrule 1360 36 0 21 2 0 2 2 0 8 0 art_heap8 4096 2 0 0 2 0 2 2 0 8 0 art_heap4 256 257 0 75 15 2 13 15 0 8 0 art_table 32 259 0 75 2 0 2 2 0 8 0 art_node 16 73 0 36 1 0 1 1 0 8 0 sysvmsgpl 40 23 0 14 1 0 1 1 0 8 0 semupl 112 4 0 4 1 1 0 1 0 8 0 semapl 112 98 0 88 1 0 1 1 0 8 0 shmpl 112 24 0 9 1 0 1 1 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 2768 0 1373 88 0 88 88 0 8 0 ffsino 240 2768 0 1373 83 0 83 83 0 8 0 nchpl 144 4522 0 2917 60 0 60 60 0 8 0 uvmvnodes 72 3040 0 0 56 0 56 56 0 8 0 vnodes 208 3040 0 0 160 0 160 160 0 8 0 namei 1024 16336 0 16336 2 1 1 1 0 8 1 vcpupl 1984 14 0 1 2 0 2 2 0 8 0 vmpool 528 24 0 11 1 0 1 1 0 8 0 pfiaddrpl 120 37 0 26 1 0 1 1 0 8 0 scsiplug 64 2 0 2 2 2 0 1 0 8 0 scxspl 192 13268 0 13268 1 0 1 1 0 8 1 plimitpl 152 483 0 476 1 0 1 1 0 8 0 sigapl 424 1373 0 1323 6 0 6 6 0 8 0 futexpl 56 21330 0 21330 2 1 1 1 0 8 1 knotepl 112 128 0 109 1 0 1 1 0 8 0 kqueuepl 144 133 0 129 1 0 1 1 0 8 0 pipepl 272 177 0 167 1 0 1 1 0 8 0 fdescpl 432 1337 0 1323 2 0 2 2 0 8 0 filepl 120 9936 0 9841 4 0 4 4 0 8 1 lockfpl 104 156 0 155 1 0 1 1 0 8 0 lockfspl 48 58 0 57 1 0 1 1 0 8 0 sessionpl 112 17 0 7 1 0 1 1 0 8 0 pgrppl 48 25 0 15 1 0 1 1 0 8 0 ucredpl 96 4145 0 4138 1 0 1 1 0 8 0 zombiepl 144 1323 0 1323 1 0 1 1 0 8 1 processpl 928 1373 0 1323 7 0 7 7 0 8 0 procpl 624 2688 0 2630 5 0 5 5 0 8 0 sosppl 128 7 0 7 2 2 0 1 0 8 0 sockpl 400 1804 0 1785 7 3 4 4 0 8 1 mcl64k 65536 34 0 34 7 6 1 1 0 8 1 mcl16k 16384 5 0 5 4 4 0 1 0 8 0 mcl12k 12288 60 0 60 8 7 1 1 0 8 1 mcl9k 9216 304 0 304 3 2 1 1 0 8 1 mcl8k 8192 48 0 48 5 5 0 1 0 8 0 mcl4k 4096 109 0 109 6 5 1 1 0 8 1 mcl2k2 2112 6 0 6 3 3 0 1 0 8 0 mcl2k 2048 94268 0 94206 22 13 9 18 0 8 0 mtagpl 96 86 0 15 3 1 2 2 0 8 0 mbufpl 256 159398 0 159079 43 21 22 33 0 8 0 mbufpl: pool(0xffffffff827ef020:mbufpl): free list modified: page 0xfffffd80533a7000; item ordinal 0; addr 0xfffffd80533a7d00 (p 0xfffffd805a00f000); offset 0x0=0x21cf0f1fbb580000 bufpl 280 5145 0 126 359 0 359 359 0 8 0 anonpl 16 139408 0 122421 103 23 80 85 0 107 0 amapchunkpl 152 6567 0 6432 47 40 7 20 0 158 1 amappl16 192 6234 0 5146 70 15 55 60 0 8 0 amappl15 184 1 0 0 1 0 1 1 0 8 0 amappl14 176 562 0 556 1 0 1 1 0 8 0 amappl13 168 26 0 23 1 0 1 1 0 8 0 amappl12 160 610 0 605 1 0 1 1 0 8 0 amappl11 152 593 0 582 1 0 1 1 0 8 0 amappl10 144 20 0 15 1 0 1 1 0 8 0 amappl9 136 350 0 349 1 0 1 1 0 8 0 amappl8 128 340 0 299 2 0 2 2 0 8 0 amappl7 120 105 0 93 1 0 1 1 0 8 0 amappl6 112 561 0 554 1 0 1 1 0 8 0 amappl5 104 1389 0 1377 1 0 1 1 0 8 0 amappl4 96 964 0 935 1 0 1 1 0 8 0 amappl3 88 474 0 467 1 0 1 1 0 8 0 amappl2 80 9905 0 9838 2 0 2 2 0 8 0 amappl1 72 38577 0 38162 23 14 9 17 0 8 0 amappl 80 2933 0 2883 2 0 2 2 0 84 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 64 26 0 9 1 0 1 1 0 8 0 uaddrrnd 24 1361 0 1334 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 1361 0 1334 1 0 1 1 0 8 0 vmmpekpl 168 11903 0 11869 2 0 2 2 0 8 0 vmmpepl 168 166750 0 164481 139 33 106 118 0 357 7 vmsppl 272 1360 0 1334 2 0 2 2 0 8 0 pdppl 4096 2728 0 2681 8 1 7 7 0 8 0 pvpl 32 523260 0 503954 222 38 184 196 0 265 6 pmappl 200 1360 0 1334 3 1 2 2 0 8 0 extentpl 40 53 0 36 1 0 1 1 0 8 0 phpool 112 308 0 63 8 0 8 8 0 8 0 ddb> machine ddbcpu 0 No such command ddb> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic(ffffffff82474430) at panic+0x15c sys/kern/subr_prf.c:207 pool_do_get(ffffffff827ef020,1,ffff80001d722ca8) at pool_do_get+0x42a sys/kern/subr_pool.c:738 pool_get(ffffffff827ef020,1) at pool_get+0xb5 sys/kern/subr_pool.c:581 m_get(1,1) at m_get+0x4c sys/kern/uipc_mbuf.c:250 m_getuio(ffff80001d722df8,0,4200,ffff80001d722f68) at m_getuio+0xbe sys/kern/uipc_socket.c:592 sosend(fffffd805da64648,0,ffff80001d722f68,0,0,80) at sosend+0x54e sys/kern/uipc_socket.c:542 dofilewritev(ffff80001d71d600,4,ffff80001d722f68,0,ffff80001d723050) at dofilewritev+0x1ab sys/kern/sys_generic.c:365 sys_write(ffff80001d71d600,ffff80001d723000,ffff80001d723050) at sys_write+0x83 sys/kern/sys_generic.c:285 syscall(ffff80001d7230d0) at syscall+0x507 sys/arch/amd64/amd64/trap.c:570 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffe6c10, count: -11 ddb> machine ddbcpu 1 No such command ddb> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic(ffffffff82474430) at panic+0x15c sys/kern/subr_prf.c:207 pool_do_get(ffffffff827ef020,1,ffff80001d722ca8) at pool_do_get+0x42a sys/kern/subr_pool.c:738 pool_get(ffffffff827ef020,1) at pool_get+0xb5 sys/kern/subr_pool.c:581 m_get(1,1) at m_get+0x4c sys/kern/uipc_mbuf.c:250 m_getuio(ffff80001d722df8,0,4200,ffff80001d722f68) at m_getuio+0xbe sys/kern/uipc_socket.c:592 sosend(fffffd805da64648,0,ffff80001d722f68,0,0,80) at sosend+0x54e sys/kern/uipc_socket.c:542 dofilewritev(ffff80001d71d600,4,ffff80001d722f68,0,ffff80001d723050) at dofilewritev+0x1ab sys/kern/sys_generic.c:365 sys_write(ffff80001d71d600,ffff80001d723000,ffff80001d723050) at sys_write+0x83 sys/kern/sys_generic.c:285 syscall(ffff80001d7230d0) at syscall+0x507 sys/arch/amd64/amd64/trap.c:570 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffe6c10, count: -11