panic: pool_do_get: mbufpl free list modified: page 0xfffffd805bb4c000; item addr 0xfffffd805bb4c800; offset 0x0=0x0 != 0xd74cece66d0a6c2e Stopped at db_enter+0x18: addq $0x8,%rsp TID PID UID PRFLAGS PFLAGS CPU COMMAND *488536 3955 0 0 0 0 dhclient db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic(ffffffff82473739) at panic+0x15c sys/kern/subr_prf.c:207 pool_do_get(ffffffff82862e28,2,ffff80001d6e4828) at pool_do_get+0x42a sys/kern/subr_pool.c:738 pool_get(ffffffff82862e28,2) at pool_get+0xb5 sys/kern/subr_pool.c:581 m_get(2,3) at m_get+0x4c sys/kern/uipc_mbuf.c:250 sbappendaddr(fffffd8065b71000,fffffd8065b71088,ffffffff82524d2c,fffffd805bb4c900,0) at sbappendaddr+0x2a0 sys/kern/uipc_socket2.c:804 rtm_sendup(fffffd8065b71000,fffffd805bb4c900,0) at rtm_sendup+0xef sys/net/rtsock.c:594 route_input(fffffd805bb4c900,fffffd8065b71000,2) at route_input+0x44b sys/net/rtsock.c:572 route_output(fffffd805bb4c900,fffffd8065b71000,0,0) at route_output+0x876 sys/net/rtsock.c:868 route_usrreq(fffffd8065b71000,9,fffffd805bb4c900,0,0,ffff80001d6a9008) at route_usrreq+0x36f sys/net/rtsock.c:275 sosend(fffffd8065b71000,0,ffff80001d6e4e10,0,0,0) at sosend+0x669 sys/kern/uipc_socket.c:555 dofilewritev(ffff80001d6a9008,6,ffff80001d6e4e10,0,ffff80001d6e4f00) at dofilewritev+0x1ab sys/kern/sys_generic.c:365 sys_writev(ffff80001d6a9008,ffff80001d6e4eb0,ffff80001d6e4f00) at sys_writev+0xa7 sys/kern/sys_generic.c:312 syscall(ffff80001d6e4f80) at syscall+0x507 sys/arch/amd64/amd64/trap.c:570 end trace frame: 0xffff80001d6e5000, count: 0 https://www.openbsd.org/ddb.html describes the minimum info required in bug reports. Insufficient info makes it difficult to find and fix bugs. ddb> ddb> set $lines = 0 ddb> set $maxwidth = 0 ddb> show panic pool_do_get: mbufpl free list modified: page 0xfffffd805bb4c000; item addr 0xfffffd805bb4c800; offset 0x0=0x0 != 0xd74cece66d0a6c2e ddb> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic(ffffffff82473739) at panic+0x15c sys/kern/subr_prf.c:207 pool_do_get(ffffffff82862e28,2,ffff80001d6e4828) at pool_do_get+0x42a sys/kern/subr_pool.c:738 pool_get(ffffffff82862e28,2) at pool_get+0xb5 sys/kern/subr_pool.c:581 m_get(2,3) at m_get+0x4c sys/kern/uipc_mbuf.c:250 sbappendaddr(fffffd8065b71000,fffffd8065b71088,ffffffff82524d2c,fffffd805bb4c900,0) at sbappendaddr+0x2a0 sys/kern/uipc_socket2.c:804 rtm_sendup(fffffd8065b71000,fffffd805bb4c900,0) at rtm_sendup+0xef sys/net/rtsock.c:594 route_input(fffffd805bb4c900,fffffd8065b71000,2) at route_input+0x44b sys/net/rtsock.c:572 route_output(fffffd805bb4c900,fffffd8065b71000,0,0) at route_output+0x876 sys/net/rtsock.c:868 route_usrreq(fffffd8065b71000,9,fffffd805bb4c900,0,0,ffff80001d6a9008) at route_usrreq+0x36f sys/net/rtsock.c:275 sosend(fffffd8065b71000,0,ffff80001d6e4e10,0,0,0) at sosend+0x669 sys/kern/uipc_socket.c:555 dofilewritev(ffff80001d6a9008,6,ffff80001d6e4e10,0,ffff80001d6e4f00) at dofilewritev+0x1ab sys/kern/sys_generic.c:365 sys_writev(ffff80001d6a9008,ffff80001d6e4eb0,ffff80001d6e4f00) at sys_writev+0xa7 sys/kern/sys_generic.c:312 syscall(ffff80001d6e4f80) at syscall+0x507 sys/arch/amd64/amd64/trap.c:570 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffdc500, count: -15 ddb> show registers rdi 0 rsi 0x1 rbp 0xffff80001d6e4690 rbx 0xffff80001d6e4740 rdx 0x2 rcx 0 rax 0x1 r8 0xffffffff8163080f kprintf+0x15f r9 0x1 r10 0x2 r11 0xd691613b8b39c1cf r12 0x3000000008 r13 0xffff80001d6e46a0 r14 0x100 r15 0x1 rip 0xffffffff8114a6e8 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff80001d6e4680 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb> show proc PROC (dhclient) pid=488536 stat=onproc flags process=0 proc=0 pri=24, usrpri=50, nice=20 forw=0xffffffffffffffff, list=0xffff80001d6aa388,0xffff80001d6a99d8 process=0xffff80001d6c4010 user=0xffff80001d6e0000, vmspace=0xfffffd806bc09770 estcpu=0, cpticks=1, pctcpu=0.0 user=0, sys=1, intr=0 ddb> ps PID TID PPID UID S FLAGS WAIT COMMAND 17537 432478 46059 0 2 0 syz-executor.0 17537 432470 46059 0 2 0x4000000 syz-executor.0 44764 456050 0 0 3 0x14200 bored sosplice 29907 260416 0 0 3 0x14280 nfsidl nfsio 44993 284254 0 0 3 0x14280 nfsidl nfsio 91364 445973 0 0 3 0x14280 nfsidl nfsio 26075 289878 0 0 3 0x14280 nfsidl nfsio 38458 434786 0 0 3 0x14280 nfsidl nfsio 1893 300810 0 0 3 0x14280 nfsidl nfsio 45967 238750 0 0 3 0x14280 nfsidl nfsio 5076 511385 0 0 3 0x14280 nfsidl nfsio 85729 448396 0 0 3 0x14280 nfsidl nfsio 99907 131100 0 0 3 0x14280 nfsidl nfsio 42326 280702 0 0 3 0x14280 nfsidl nfsio 33 69748 0 0 3 0x14280 nfsidl nfsio 75850 492458 0 0 3 0x14280 nfsidl nfsio 46721 336130 0 0 3 0x14280 nfsidl nfsio 75593 258098 0 0 3 0x14280 nfsidl nfsio 20689 354072 0 0 3 0x14280 nfsidl nfsio 38419 133787 0 0 3 0x14280 nfsidl nfsio 19775 246720 0 0 3 0x14280 nfsidl nfsio 37154 320356 0 0 3 0x14280 nfsidl nfsio 99716 109855 0 0 3 0x14280 nfsidl nfsio 46059 332502 95064 0 3 0x82 nanosleep syz-executor.0 27258 384286 95064 0 3 0x82 nanosleep syz-executor.1 95064 170443 56033 0 3 0x82 thrsleep syz-fuzzer 95064 156010 56033 0 3 0x4000082 thrsleep syz-fuzzer 95064 236213 56033 0 3 0x4000082 kqread syz-fuzzer 95064 477016 56033 0 3 0x4000082 thrsleep syz-fuzzer 95064 62332 56033 0 3 0x4000082 thrsleep syz-fuzzer 95064 173769 56033 0 3 0x4000082 thrsleep syz-fuzzer 95064 57773 56033 0 3 0x4000082 thrsleep syz-fuzzer 56033 128271 84902 0 3 0x10008a pause ksh 84902 349755 59196 0 3 0x92 select sshd 76837 206340 1 0 3 0x100083 ttyin getty 59196 210777 1 0 3 0x80 select sshd 88514 481003 77059 73 3 0x100090 kqread syslogd 77059 90130 1 0 3 0x100082 netio syslogd 35327 323225 1 77 2 0x100010 dhclient * 3955 488536 1 0 7 0 dhclient 43397 2649 0 0 3 0x14200 bored smr 40303 472004 0 0 2 0x14200 zerothread 95722 85130 0 0 3 0x14200 aiodoned aiodoned 52134 133165 0 0 3 0x14200 syncer update 40336 300210 0 0 3 0x14200 cleaner cleaner 18163 171242 0 0 3 0x14200 reaper reaper 59246 269955 0 0 3 0x14200 pgdaemon pagedaemon 62125 406046 0 0 3 0x14200 bored crynlk 36467 138255 0 0 3 0x14200 bored crypto 4870 347060 0 0 3 0x40014200 acpi0 acpi0 31141 243919 0 0 2 0x14200 softnet 34158 86764 0 0 3 0x14200 bored systqmp 40328 113968 0 0 3 0x14200 bored systq 45328 468488 0 0 3 0x40014200 bored softclock 86035 193849 0 0 3 0x40014200 idle0 1 74833 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb> show all locks No such command ddb> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 9461 6344K 6583K 78643K 10720 0 pcb 13 8K 8K 78643K 33 0 rtable 110 3K 6K 78643K 260 0 ifaddr 51 11K 11K 78643K 76 0 counters 22 16K 16K 78643K 23 0 ioctlops 0 0K 4K 78643K 73 0 iov 0 0K 12K 78643K 12 0 mount 1 1K 1K 78643K 1 0 vnodes 1218 77K 77K 78643K 1265 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 5K 78643K 3 0 VM map 2 0K 0K 78643K 2 0 sem 12 0K 1K 78643K 32 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1809 195K 288K 78643K 12938 0 file desc 5 13K 25K 78643K 121 0 proc 49 38K 54K 78643K 374 0 subproc 32 2K 2K 78643K 34 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 3 0 in_multi 40 2K 2K 78643K 45 0 ether_multi 1 0K 0K 78643K 3 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 49 228K 228K 78643K 49 0 exec 0 0K 1K 78643K 207 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 7 26K 26K 78643K 7 0 UVM amap 121 23K 23K 78643K 1118 0 UVM aobj 5 2K 3K 78643K 7 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 0K 78643K 8 0 NDP 7 0K 0K 78643K 12 0 temp 80 3854K 3917K 78643K 3040 0 kqueue 3 4K 8K 78643K 5 0 SYN cache 2 16K 16K 78643K 2 0 ddb> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle arp 64 6 0 0 1 0 1 1 0 8 0 rtpcb 80 21 0 19 1 0 1 1 0 8 0 rtentry 112 46 0 1 2 0 2 2 0 8 0 unpcb 120 111 0 103 1 0 1 1 0 8 0 syncache 264 4 0 4 1 1 0 1 0 8 0 tcpqe 32 383 0 383 1 1 0 1 0 8 0 tcpcb 544 20 0 16 1 0 1 1 0 8 0 inpcb 296 114 0 107 3 1 2 2 0 8 1 nd6 48 6 0 0 1 0 1 1 0 8 0 ppxss 1136 1 0 1 1 0 1 1 0 8 1 pfrktable 1344 28 0 28 1 0 1 1 0 8 1 pftag 88 8 0 8 1 0 1 1 0 8 1 pfrule 1360 4 0 4 1 0 1 1 0 8 1 art_heap8 4096 2 0 0 2 0 2 2 0 8 0 art_heap4 256 194 0 0 13 0 13 13 0 8 0 art_table 32 196 0 0 2 0 2 2 0 8 0 art_node 16 45 0 4 1 0 1 1 0 8 0 sysvmsgpl 40 6 0 2 1 0 1 1 0 8 0 semupl 112 1 0 1 1 1 0 1 0 8 0 semapl 112 28 0 18 1 0 1 1 0 8 0 shmpl 112 4 0 2 1 0 1 1 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 1547 0 148 88 0 88 88 0 8 0 ffsino 240 1547 0 148 83 0 83 83 0 8 0 nchpl 144 1898 0 293 60 0 60 60 0 8 0 uvmvnodes 72 1620 0 0 30 0 30 30 0 8 0 vnodes 208 1620 0 0 86 0 86 86 0 8 0 namei 1024 4908 0 4908 4 3 1 1 0 8 1 pfiaddrpl 120 8 0 8 1 0 1 1 0 8 1 scxspl 192 5431 0 5431 1 0 1 1 0 8 1 plimitpl 152 15 0 8 1 0 1 1 0 8 0 sigapl 424 328 0 279 6 0 6 6 0 8 0 futexpl 56 1769 0 1769 3 2 1 1 0 8 1 knotepl 112 65 0 46 1 0 1 1 0 8 0 kqueuepl 144 18 0 16 1 0 1 1 0 8 0 pipepl 272 85 0 74 1 0 1 1 0 8 0 fdescpl 432 293 0 279 2 0 2 2 0 8 0 filepl 120 1612 0 1508 5 1 4 4 0 8 0 lockfpl 104 25 0 22 1 0 1 1 0 8 0 lockfspl 48 10 0 7 1 0 1 1 0 8 0 sessionpl 112 17 0 7 1 0 1 1 0 8 0 pgrppl 48 17 0 7 1 0 1 1 0 8 0 ucredpl 96 137 0 128 1 0 1 1 0 8 0 zombiepl 144 279 0 278 3 2 1 1 0 8 0 processpl 928 328 0 278 8 1 7 7 0 8 0 procpl 624 439 0 382 5 0 5 5 0 8 0 sockpl 400 246 0 229 5 2 3 3 0 8 1 mcl64k 65536 3 0 3 1 0 1 1 0 8 1 mcl12k 12288 4 0 4 2 1 1 1 0 8 1 mcl9k 9216 1 0 1 1 1 0 1 0 8 0 mcl8k 8192 1 0 1 1 0 1 1 0 8 1 mcl4k 4096 19 0 19 4 3 1 1 0 8 1 mcl2k 2048 91430 0 91367 23 13 10 17 0 8 1 mtagpl 96 6 0 2 2 1 1 1 0 8 0 mbufpl 256 144600 0 144475 15 2 13 13 0 8 1 mbufpl: pool(0xffffffff82862e28:mbufpl): free list modified: page 0xfffffd805bb4c000; item ordinal 0; addr 0xfffffd805bb4c800 (p 0xfffffd805dc36000); offset 0x0=0x0 pool(mbufpl): free list modified: page 0xfffffd805bb4c000; item ordinal 0; addr 0xfffffd805bb4c800 (p 0xfffffd805dc36000); offset 0x0=0x0 mbufpl: pool(0xffffffff82862e28:mbufpl): page inconsistency: page 0xfffffd805bb4c000; item ordinal 1; addr 0xc19095ffc274079c bufpl 280 3406 0 127 235 0 235 235 0 8 0 anonpl 16 42919 0 27243 66 3 63 63 0 107 0 amapchunkpl 152 1315 0 1183 8 1 7 8 0 158 0 amappl16 192 1328 0 458 44 0 44 44 0 8 0 amappl15 184 1 0 0 1 0 1 1 0 8 0 amappl14 176 27 0 22 1 0 1 1 0 8 0 amappl13 168 72 0 68 1 0 1 1 0 8 0 amappl12 160 99 0 96 2 1 1 1 0 8 0 amappl11 152 53 0 43 1 0 1 1 0 8 0 amappl10 144 21 0 15 1 0 1 1 0 8 0 amappl9 136 374 0 372 1 0 1 1 0 8 0 amappl8 128 330 0 289 2 0 2 2 0 8 0 amappl7 120 107 0 95 1 0 1 1 0 8 0 amappl6 112 27 0 21 1 0 1 1 0 8 0 amappl5 104 224 0 212 1 0 1 1 0 8 0 amappl4 96 469 0 439 1 0 1 1 0 8 0 amappl3 88 154 0 146 1 0 1 1 0 8 0 amappl2 80 1524 0 1457 2 0 2 2 0 8 0 amappl1 72 15537 0 15112 23 14 9 17 0 8 0 amappl 80 649 0 608 1 0 1 1 0 84 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 64 6 0 2 1 0 1 1 0 8 0 uaddrrnd 24 293 0 279 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 293 0 279 1 0 1 1 0 8 0 vmmpekpl 168 6068 0 6039 2 0 2 2 0 8 0 vmmpepl 168 43245 0 41265 168 75 93 151 0 357 6 vmsppl 272 292 0 279 2 1 1 2 0 8 0 pdppl 4096 592 0 558 6 1 5 6 0 8 0 pvpl 32 144491 0 125747 156 4 152 152 0 265 0 pmappl 200 292 0 279 1 0 1 1 0 8 0 extentpl 40 53 0 36 1 0 1 1 0 8 0 phpool 112 260 0 27 7 0 7 7 0 8 0 ddb> machine ddbcpu 0 No such command ddb> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic(ffffffff82473739) at panic+0x15c sys/kern/subr_prf.c:207 pool_do_get(ffffffff82862e28,2,ffff80001d6e4828) at pool_do_get+0x42a sys/kern/subr_pool.c:738 pool_get(ffffffff82862e28,2) at pool_get+0xb5 sys/kern/subr_pool.c:581 m_get(2,3) at m_get+0x4c sys/kern/uipc_mbuf.c:250 sbappendaddr(fffffd8065b71000,fffffd8065b71088,ffffffff82524d2c,fffffd805bb4c900,0) at sbappendaddr+0x2a0 sys/kern/uipc_socket2.c:804 rtm_sendup(fffffd8065b71000,fffffd805bb4c900,0) at rtm_sendup+0xef sys/net/rtsock.c:594 route_input(fffffd805bb4c900,fffffd8065b71000,2) at route_input+0x44b sys/net/rtsock.c:572 route_output(fffffd805bb4c900,fffffd8065b71000,0,0) at route_output+0x876 sys/net/rtsock.c:868 route_usrreq(fffffd8065b71000,9,fffffd805bb4c900,0,0,ffff80001d6a9008) at route_usrreq+0x36f sys/net/rtsock.c:275 sosend(fffffd8065b71000,0,ffff80001d6e4e10,0,0,0) at sosend+0x669 sys/kern/uipc_socket.c:555 dofilewritev(ffff80001d6a9008,6,ffff80001d6e4e10,0,ffff80001d6e4f00) at dofilewritev+0x1ab sys/kern/sys_generic.c:365 sys_writev(ffff80001d6a9008,ffff80001d6e4eb0,ffff80001d6e4f00) at sys_writev+0xa7 sys/kern/sys_generic.c:312 syscall(ffff80001d6e4f80) at syscall+0x507 sys/arch/amd64/amd64/trap.c:570 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffdc500, count: -15 ddb> machine ddbcpu 1 No such command ddb> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic(ffffffff82473739) at panic+0x15c sys/kern/subr_prf.c:207 pool_do_get(ffffffff82862e28,2,ffff80001d6e4828) at pool_do_get+0x42a sys/kern/subr_pool.c:738 pool_get(ffffffff82862e28,2) at pool_get+0xb5 sys/kern/subr_pool.c:581 m_get(2,3) at m_get+0x4c sys/kern/uipc_mbuf.c:250 sbappendaddr(fffffd8065b71000,fffffd8065b71088,ffffffff82524d2c,fffffd805bb4c900,0) at sbappendaddr+0x2a0 sys/kern/uipc_socket2.c:804 rtm_sendup(fffffd8065b71000,fffffd805bb4c900,0) at rtm_sendup+0xef sys/net/rtsock.c:594 route_input(fffffd805bb4c900,fffffd8065b71000,2) at route_input+0x44b sys/net/rtsock.c:572 route_output(fffffd805bb4c900,fffffd8065b71000,0,0) at route_output+0x876 sys/net/rtsock.c:868 route_usrreq(fffffd8065b71000,9,fffffd805bb4c900,0,0,ffff80001d6a9008) at route_usrreq+0x36f sys/net/rtsock.c:275 sosend(fffffd8065b71000,0,ffff80001d6e4e10,0,0,0) at sosend+0x669 sys/kern/uipc_socket.c:555 dofilewritev(ffff80001d6a9008,6,ffff80001d6e4e10,0,ffff80001d6e4f00) at dofilewritev+0x1ab sys/kern/sys_generic.c:365 sys_writev(ffff80001d6a9008,ffff80001d6e4eb0,ffff80001d6e4f00) at sys_writev+0xa7 sys/kern/sys_generic.c:312 syscall(ffff80001d6e4f80) at syscall+0x507 sys/arch/amd64/amd64/trap.c:570 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffdc500, count: -15