panic: pool_do_get: mcl4k free list modified: page 0xfffffd806d3bf000; item addr 0xfffffd806d3bf000; offset 0x0=0x11e57ed0 != 0x62dc609dd4cff30e Stopped at db_enter+0x25: addq $0x8,%rsp TID PID UID PRFLAGS PFLAGS CPU COMMAND *158768 11311 0 0 0 0 syz-executor db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438 panic(ffffffff83448e88) at panic+0x1cf sys/kern/subr_prf.c:198 pool_do_get(ffffffff83955a10,2,ffff80002cccb8e8) at pool_do_get+0x574 sys/kern/subr_pool.c:747 pool_get(ffffffff83955a10,2) at pool_get+0x11a sys/kern/subr_pool.c:-1 m_clget(0,2,1000) at m_clget+0x354 sys/kern/uipc_mbuf.c:-1 vio_populate_rx_mbufs(ffff8000002a2000,ffff80000002fc00) at vio_populate_rx_mbufs+0x197 vio_add_rx_mbuf sys/dev/pv/if_vio.c:-1 [inline] vio_populate_rx_mbufs(ffff8000002a2000,ffff80000002fc00) at vio_populate_rx_mbufs+0x197 sys/dev/pv/if_vio.c:1485 vio_rx_intr(ffff8000002a3000) at vio_rx_intr+0xc9 sys/dev/pv/if_vio.c:1645 intr_handler(ffff80002cccbac0,ffff8000002a1600) at intr_handler+0xcb sys/arch/amd64/amd64/intr.c:-1 Xintr_ioapic_edge25_untramp() at Xintr_ioapic_edge25_untramp+0x18f end of kernel end trace frame: 0x747745cf35e0, count: 6 https://www.openbsd.org/ddb.html describes the minimum info required in bug reports. Insufficient info makes it difficult to find and fix bugs. ddb> ddb> set $lines = 0 ddb> set $maxwidth = 0 ddb> show panic *cpu0: pool_do_get: mcl4k free list modified: page 0xfffffd806d3bf000; item addr 0xfffffd806d3bf000; offset 0x0=0x11e57ed0 != 0x62dc609dd4cff30e ddb> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438 panic(ffffffff83448e88) at panic+0x1cf sys/kern/subr_prf.c:198 pool_do_get(ffffffff83955a10,2,ffff80002cccb8e8) at pool_do_get+0x574 sys/kern/subr_pool.c:747 pool_get(ffffffff83955a10,2) at pool_get+0x11a sys/kern/subr_pool.c:-1 m_clget(0,2,1000) at m_clget+0x354 sys/kern/uipc_mbuf.c:-1 vio_populate_rx_mbufs(ffff8000002a2000,ffff80000002fc00) at vio_populate_rx_mbufs+0x197 vio_add_rx_mbuf sys/dev/pv/if_vio.c:-1 [inline] vio_populate_rx_mbufs(ffff8000002a2000,ffff80000002fc00) at vio_populate_rx_mbufs+0x197 sys/dev/pv/if_vio.c:1485 vio_rx_intr(ffff8000002a3000) at vio_rx_intr+0xc9 sys/dev/pv/if_vio.c:1645 intr_handler(ffff80002cccbac0,ffff8000002a1600) at intr_handler+0xcb sys/arch/amd64/amd64/intr.c:-1 Xintr_ioapic_edge25_untramp() at Xintr_ioapic_edge25_untramp+0x18f end of kernel end trace frame: 0x747745cf35e0, count: -9 ddb> show registers rdi 0 rsi 0x1 rbp 0xffff80002cccb750 rbx 0xfffffd806d3bf000 rdx 0 rcx 0 rax 0xffff80002f0d0038 r8 0x101010101010101 r9 0x8080808080808080 r10 0xa7ac1b07b9b942c3 r11 0x3f684f08e71c45ee r12 0 r13 0xe9dbda04c2dc17ae r14 0 r15 0x1 rip 0xffffffff82289575 db_enter+0x25 cs 0x8 rflags 0x246 rsp 0xffff80002cccb740 ss 0 db_enter+0x25: addq $0x8,%rsp ddb> show proc PROC (syz-executor) tid=158768 pid=11311 tcnt=2 stat=onproc flags process=0 proc=0 runpri=50, usrpri=50, slppri=17, nice=20 wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0 forw=0xffffffffffffffff, list=0xffff80002f0d1a28,0xffff80002f0d1270 process=0xffff8000fffefaa0 user=0xffff80002ccc6000, vmspace=0xfffffd807c59c748 estcpu=36, cpticks=2, pctcpu=0.0, user=1, sys=0, intr=1 ddb> ps PID TID PPID UID S FLAGS WAIT COMMAND *11311 158768 59335 0 7 0 syz-executor 11311 40984 59335 0 2 0x4000000 syz-executor 8479 344792 36168 0 2 0 syz-executor 8479 488349 36168 0 3 0x4000080 fsleep syz-executor 8479 38671 36168 0 2 0x4000000 syz-executor 49871 351583 65627 0 2 0 syz-executor 49871 318109 65627 0 3 0x4000080 fsleep syz-executor 49871 329211 65627 0 3 0x4000080 fsleep syz-executor 80012 43573 83062 0 2 0x1 syz-executor 80012 172966 83062 0 3 0x4000080 fsleep syz-executor 80012 418732 83062 0 3 0x4000080 fsleep syz-executor 42348 47792 98346 0 3 0x80 nanoslp syz-executor 42348 413886 98346 0 3 0x4000080 fsleep syz-executor 42348 144537 98346 0 3 0x4000080 fsleep syz-executor 46540 178655 74528 0 2 0x1 syz-executor 46540 223709 74528 0 3 0x4000080 fsleep syz-executor 54545 203299 65835 0 3 0x80 nanoslp syz-executor 54545 390870 65835 0 3 0x4000080 sbwait syz-executor 54545 281924 65835 0 3 0x4000080 fsleep syz-executor 54545 204406 65835 0 3 0x4000080 fsleep syz-executor 3883 387353 0 0 3 0x14280 nfsidl nfsio 18100 376510 0 0 3 0x14280 nfsidl nfsio 33896 412769 0 0 3 0x14280 nfsidl nfsio 84645 416041 0 0 3 0x14280 nfsidl nfsio 86601 124810 0 0 3 0x14280 nfsidl nfsio 60343 154141 0 0 3 0x14280 nfsidl nfsio 89104 88148 0 0 3 0x14280 nfsidl nfsio 9522 262685 0 0 3 0x14280 nfsidl nfsio 8857 460036 0 0 3 0x14280 nfsidl nfsio 187 244597 0 0 3 0x14280 nfsidl nfsio 79149 424333 0 0 3 0x14280 nfsidl nfsio 13152 129721 0 0 3 0x14280 nfsidl nfsio 14210 197477 0 0 3 0x14280 nfsidl nfsio 80969 290045 0 0 3 0x14280 nfsidl nfsio 53988 387555 0 0 3 0x14280 nfsidl nfsio 67629 262189 0 0 3 0x14280 nfsidl nfsio 39929 295899 0 0 3 0x14280 nfsidl nfsio 6429 172287 0 0 3 0x14280 nfsidl nfsio 37946 461791 0 0 3 0x14280 nfsidl nfsio 34505 487025 0 0 3 0x14280 nfsidl nfsio 50371 362841 3745 0 3 0x82 sbwait sshd-session 98346 393910 9308 0 3 0x82 nanoslp syz-executor 65627 249180 9308 0 3 0x82 nanoslp syz-executor 36168 293402 9308 0 3 0x82 nanoslp syz-executor 83062 274083 9308 0 3 0x82 nanoslp syz-executor 65835 438953 9308 0 3 0x82 nanoslp syz-executor 59335 461881 9308 0 3 0x82 nanoslp syz-executor 7848 149096 9308 0 2 0x3 syz-executor 74528 350088 9308 0 3 0x82 nanoslp syz-executor 9308 102177 12414 0 2 0x3 syz-executor 12414 58045 3734 0 3 0x10008a sigsusp ksh 3734 128167 54191 0 3 0x98 kqread sshd-session 54191 136930 3745 0 3 0x92 kqread sshd-session 65427 393864 1 0 3 0x100083 ttyopn getty 3745 289409 1 0 3 0x88 kqread sshd 41933 426454 42568 73 3 0x1100090 kqread syslogd 42568 171255 1 0 3 0x100082 sbwait syslogd 37767 316593 1 0 3 0x100080 kqread resolvd 42912 382519 88017 77 3 0x100092 kqread dhcpleased 59336 143029 88017 77 3 0x100092 kqread dhcpleased 88017 132906 1 0 3 0x80 kqread dhcpleased 554 222701 0 0 3 0x14200 bored smr 61828 204950 0 0 2 0x14200 zerothread 68518 193053 0 0 3 0x14200 aiodoned aiodoned 2845 404852 0 0 3 0x14200 syncer update 60390 52310 0 0 3 0x14200 cleaner cleaner 41107 236025 0 0 3 0x14200 reaper reaper 27791 357511 0 0 3 0x14200 pgdaemon pagedaemon 63922 474139 0 0 3 0x14200 bored viomb 85078 514845 0 0 3 0x40014200 acpi0 acpi0 25276 375899 0 0 2 0x14200 softnet0 90574 209281 0 0 3 0x14200 bored systqmp 26851 461407 0 0 3 0x14200 bored systq 12224 195359 0 0 3 0x40014200 tmoslp softclock 83578 331300 0 0 3 0x40014200 idle0 1 226883 0 0 3 0x82 wait init 0 0 -1 0 3 0x10010200 scheduler swapper ddb> show all locks No such command ddb> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 11066 12111K 12519K 166960K 13913 0 pcb 17 14K 15K 166960K 294 0 rtable 243 10K 10K 166960K 805 0 pf 36 14K 16K 166960K 102 0 ifaddr 41 7K 8K 166960K 87 0 ifgroup 54 2K 2K 166960K 134 0 sysctl 4 1K 9K 166960K 13 0 counters 34 17K 18K 166960K 212 0 ioctlops 0 0K 4K 166960K 488 0 iov 0 0K 24K 166960K 85 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1378 87K 87K 166960K 2240 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 1K 9K 166960K 19 0 VM map 2 1K 1K 166960K 2 0 sem 12 0K 1K 166960K 50 0 dirhash 12 2K 2K 166960K 30 0 ACPI 1692 195K 286K 166960K 12470 0 file desc 17 61K 97K 166960K 930 0 sigio 0 0K 0K 166960K 18 0 proc 60 59K 124K 166960K 562 0 subproc 72 4K 4K 166960K 72 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 111 0 in_multi 89 6K 7K 166960K 154 0 ether_multi 1 0K 0K 166960K 10 0 mrt 1 0K 0K 166960K 24 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 235 1049K 1049K 166960K 235 0 exec 0 0K 1K 166960K 509 0 fusefs mount 1 32K 32K 166960K 1 0 pfkey data 0 0K 0K 166960K 1 0 tdb 3 0K 0K 166960K 3 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 235 161K 181K 166960K 10105 0 UVM aobj 59 12K 12K 166960K 63 0 pinsyscall 40 80K 96K 166960K 2086 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 2K 166960K 48 0 NDP 12 0K 2K 166960K 56 0 temp 80 9076K 9334K 166960K 31784 0 kqueue 13 20K 28K 166960K 142 0 SYN cache 2 16K 16K 166960K 2 0 ddb> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle rtpcb 120 123 0 120 1 0 1 1 0 8 0 rtentry 136 236 0 138 4 0 4 4 0 8 0 unpcb 144 1127 0 1104 7 1 6 6 0 8 5 syncache 336 16 0 16 2 1 1 1 0 8 1 tcpqe 32 10 0 10 2 1 1 1 0 8 1 tcpcb 736 422 0 407 11 3 8 8 0 8 6 arp 96 53 0 35 1 0 1 1 0 8 0 ipq 40 7 0 1 1 0 1 1 0 8 0 ipqe 40 9 0 3 1 0 1 1 0 8 0 inpcb 328 1089 0 1070 12 4 8 8 0 8 6 ip6q 72 7 0 2 1 0 1 1 0 8 0 ip6af 40 11 0 6 1 0 1 1 0 8 0 nd6 112 29 0 6 1 0 1 1 0 8 0 pkpcb 40 4 0 4 2 1 1 1 0 8 1 kcovpl 48 8 0 0 1 0 1 1 0 8 0 mppekey 1024 1 0 1 1 1 0 1 0 8 0 ppxss 1072 170 0 170 1 0 1 1 0 8 1 pppxif 1384 71 0 71 2 1 1 1 0 8 1 pfrktable 1344 2 0 2 1 1 0 1 0 8 0 pfqueue 320 1 0 1 1 1 0 1 0 8 0 pfstitem 24 5 0 0 1 0 1 1 0 8 0 pfstkey 128 7 0 4 1 0 1 1 0 8 0 pfstate 384 5 0 2 1 0 1 1 0 8 0 pfrule 1360 2 0 2 1 1 0 1 0 8 0 rttmr 136 1 0 1 1 0 1 1 0 8 1 art_heap8 4096 3 0 0 3 0 3 3 0 8 0 art_heap4 256 743 0 326 39 10 29 39 0 8 0 art_table 40 746 0 326 7 1 6 7 0 8 0 art_node 32 234 0 145 2 0 2 2 0 8 0 sysvmsgpl 40 9 0 8 1 0 1 1 0 8 0 semupl 112 2 0 2 2 1 1 1 0 8 1 semapl 112 45 0 35 1 0 1 1 0 8 0 shmpl 112 54 0 3 2 0 2 2 0 8 0 dirhash 1024 30 0 13 3 0 3 3 0 8 0 dino2pl 256 3202 0 1756 92 0 92 92 0 8 0 ffsino 256 3202 0 1756 92 0 92 92 0 8 0 nchpl 144 4500 0 2808 64 0 64 64 0 8 0 rtmask 32 11 0 11 2 1 1 1 0 8 1 vnodes 216 3764 0 0 210 0 210 210 0 8 0 namei 1024 16462 0 16462 2 1 1 2 0 8 1 vcpupl 3904 6 0 1 1 0 1 1 0 8 0 vmpool 808 6 0 1 1 0 1 1 0 8 0 kstatmem 264 80 0 56 3 1 2 3 0 8 0 acpiwqpl 32 1 0 1 1 0 1 1 1 8 1 scsiplug 72 3 0 3 1 1 0 1 0 8 0 scxspl 216 16791 0 16791 9 7 2 8 1 8 2 plimitpl 152 390 0 374 1 0 1 1 0 8 0 sigapl 424 1257 0 1194 8 0 8 8 0 8 0 knotepl 120 46406 0 46358 21 9 12 15 0 8 8 kqueuepl 184 323 0 314 4 3 1 4 0 8 0 pipepl 304 353 0 326 12 5 7 8 0 8 4 fdescpl 448 1223 0 1193 5 1 4 5 0 8 0 filepl 120 9276 0 9047 19 4 15 15 0 8 7 lockfpl 104 344 0 342 1 0 1 1 0 8 0 lockfspl 48 131 0 129 1 0 1 1 0 8 0 sessionpl 144 34 0 25 1 0 1 1 0 8 0 pgrppl 48 52 0 35 1 0 1 1 0 8 0 ucredpl 104 1286 0 1275 1 0 1 1 0 8 0 zombiepl 144 1323 0 1323 1 0 1 1 0 8 1 processpl 1152 1257 0 1194 5 0 5 5 0 8 0 procpl 664 2575 0 2499 8 0 8 8 0 8 1 sosppl 176 6 0 6 2 1 1 1 0 8 1 sockpl 552 2439 0 2394 16 4 12 13 0 8 8 mcl64k 65536 40 0 39 2 1 1 1 0 8 0 mcl16k 16384 3 0 3 2 1 1 1 0 8 1 mcl12k 12288 2 0 2 1 1 0 1 0 8 0 mcl8k 8192 15 0 15 2 1 1 1 0 8 1 mcl4k 4096 3543 0 3485 16 8 8 15 0 8 0 mcl4k: pool(0xffffffff83955a10:mcl4k): free list modified: page 0xfffffd806d3bf000; item ordinal 0; addr 0xfffffd806d3bf000 (p 0xfffffd8078ae8000); offset 0x0=0x11e57ed0 pool(mcl4k): free list modified: page 0xfffffd806d3bf000; item ordinal 0; addr 0xfffffd806d3bf000 (p 0xfffffd8078ae8000); offset 0x0=0x0 mcl4k: pool(0xffffffff83955a10:mcl4k): page inconsistency: page 0xfffffd806d3bf000; item ordinal 1; addr 0xe9dbda04c2dc17ae mcl2k 2048 1213 0 1205 4 2 2 3 0 8 1 mtagpl 96 20 0 9 1 0 1 1 0 8 0 mbufpl 256 17841 0 17614 149 129 20 88 0 8 1 bufpl 280 5265 0 111 369 0 369 369 0 8 0 anonpl 24 196792 0 191373 60 25 35 58 0 187 0 amapchunkpl 152 34242 0 33617 38 11 27 32 0 158 1 amappl16 200 3215 0 3183 24 20 4 20 0 8 0 amappl15 192 19 0 19 1 1 0 1 0 8 0 amappl14 184 426 0 425 1 0 1 1 0 8 0 amappl13 176 137 0 126 1 0 1 1 0 8 0 amappl12 168 1472 0 1443 2 0 2 2 0 8 0 amappl11 160 10 0 10 1 1 0 1 0 8 0 amappl10 152 59 0 49 1 0 1 1 0 8 0 amappl9 144 267 0 265 1 0 1 1 0 8 0 amappl8 136 94 0 93 1 0 1 1 0 8 0 amappl7 128 127 0 125 1 0 1 1 0 8 0 amappl6 120 257 0 245 1 0 1 1 0 8 0 amappl5 112 98 0 90 1 0 1 1 0 8 0 amappl4 104 548 0 510 2 0 2 2 0 8 0 amappl3 96 5928 0 5835 3 0 3 3 0 8 0 amappl2 88 1371 0 1304 2 0 2 2 0 8 0 amappl1 80 16018 0 15385 15 0 15 15 0 8 0 amappl 88 9205 0 9039 5 0 5 5 0 92 0 uvmvnodes 80 121 0 0 3 0 3 3 0 8 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 7 0 7 1 1 0 1 0 8 0 dma128 128 254 0 254 1 1 0 1 0 8 0 dma64 64 7 0 7 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 19 0 18 1 0 1 1 0 8 0 aobjpl 72 62 0 4 2 0 2 2 0 8 0 uaddrrnd 24 1223 0 1193 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 1223 0 1193 1 0 1 1 0 8 0 vmmpekpl 168 12972 0 12942 2 0 2 2 0 8 0 vmmpepl 168 87996 0 86037 103 9 94 103 0 357 5 vmsppl 368 1222 0 1193 4 1 3 4 0 8 0 rwobjpl 40 26333 0 25192 15 1 14 15 0 8 0 pdppl 4096 2464 0 2393 118 45 73 82 0 8 2 pvpl 32 553622 0 541503 146 33 113 136 0 265 10 pmappl 216 1228 0 1194 3 0 3 3 0 8 0 extentpl 40 45 0 27 1 0 1 1 0 8 0 phpool 112 552 0 208 13 2 11 13 0 8 0 ddb> machine ddbcpu 0 No such command ddb> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438 panic(ffffffff83448e88) at panic+0x1cf sys/kern/subr_prf.c:198 pool_do_get(ffffffff83955a10,2,ffff80002cccb8e8) at pool_do_get+0x574 sys/kern/subr_pool.c:747 pool_get(ffffffff83955a10,2) at pool_get+0x11a sys/kern/subr_pool.c:-1 m_clget(0,2,1000) at m_clget+0x354 sys/kern/uipc_mbuf.c:-1 vio_populate_rx_mbufs(ffff8000002a2000,ffff80000002fc00) at vio_populate_rx_mbufs+0x197 vio_add_rx_mbuf sys/dev/pv/if_vio.c:-1 [inline] vio_populate_rx_mbufs(ffff8000002a2000,ffff80000002fc00) at vio_populate_rx_mbufs+0x197 sys/dev/pv/if_vio.c:1485 vio_rx_intr(ffff8000002a3000) at vio_rx_intr+0xc9 sys/dev/pv/if_vio.c:1645 intr_handler(ffff80002cccbac0,ffff8000002a1600) at intr_handler+0xcb sys/arch/amd64/amd64/intr.c:-1 Xintr_ioapic_edge25_untramp() at Xintr_ioapic_edge25_untramp+0x18f end of kernel end trace frame: 0x747745cf35e0, count: -9 ddb> machine ddbcpu 1 No such command ddb> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438 panic(ffffffff83448e88) at panic+0x1cf sys/kern/subr_prf.c:198 pool_do_get(ffffffff83955a10,2,ffff80002cccb8e8) at pool_do_get+0x574 sys/kern/subr_pool.c:747 pool_get(ffffffff83955a10,2) at pool_get+0x11a sys/kern/subr_pool.c:-1 m_clget(0,2,1000) at m_clget+0x354 sys/kern/uipc_mbuf.c:-1 vio_populate_rx_mbufs(ffff8000002a2000,ffff80000002fc00) at vio_populate_rx_mbufs+0x197 vio_add_rx_mbuf sys/dev/pv/if_vio.c:-1 [inline] vio_populate_rx_mbufs(ffff8000002a2000,ffff80000002fc00) at vio_populate_rx_mbufs+0x197 sys/dev/pv/if_vio.c:1485 vio_rx_intr(ffff8000002a3000) at vio_rx_intr+0xc9 sys/dev/pv/if_vio.c:1645 intr_handler(ffff80002cccbac0,ffff8000002a1600) at intr_handler+0xcb sys/arch/amd64/amd64/intr.c:-1 Xintr_ioapic_edge25_untramp() at Xintr_ioapic_edge25_untramp+0x18f end of kernel end trace frame: 0x747745cf35e0, count: -9