panic: pool_cache_item_magic_check: mbufpl cpu free list modified: item addr 0xfffffd807d2aa700+24 0x26cff2e01dde9153!=0x26cff2e072d46f53 Stopped at db_enter+0x18: addq $0x8,%rsp TID PID UID PRFLAGS PFLAGS CPU COMMAND 163403 79767 0 0x2 0x480 0 syz-executor.1 *138808 28607 0 0x12 0 1 sshd db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic() at panic+0x15c sys/kern/subr_prf.c:207 pool_cache_get(ffffffff82644168) at pool_cache_get+0x323 pool_cache_item_magic_check sys/kern/subr_pool.c:1789 [inline] pool_cache_get(ffffffff82644168) at pool_cache_get+0x323 sys/kern/subr_pool.c:1892 pool_get() at pool_get+0x91 sys/kern/subr_pool.c:572 m_gethdr(1,1) at m_gethdr+0x4c sys/kern/uipc_mbuf.c:283 m_getuio(ffff800020aeaf18,0,4780,ffff800020aeb098) at m_getuio+0xe4 sys/kern/uipc_socket.c:556 sosend(fffffd806e977908,0,ffff800020aeb098,0,0,80) at sosend+0x510 sys/kern/uipc_socket.c:511 dofilewritev(ffff800020ac18c0,4,ffff800020aeb098,0,ffff800020aeb180) at dofilewritev+0x1b7 sys/kern/sys_generic.c:364 sys_write(ffff800020ac18c0,ffff800020aeb130,ffff800020aeb180) at sys_write+0x83 sys/kern/sys_generic.c:284 syscall(ffff800020aeb200) at syscall+0x4a4 mi_syscall sys/sys/syscall_mi.h:92 [inline] syscall(ffff800020aeb200) at syscall+0x4a4 sys/arch/amd64/amd64/trap.c:555 Xsyscall(0,4,182d28a1616b,4,4,182fa937d3c0) at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffe3830, count: 4 https://www.openbsd.org/ddb.html describes the minimum info required in bug reports. Insufficient info makes it difficult to find and fix bugs. ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic pool_cache_item_magic_check: mbufpl cpu free list modified: item addr 0xfffffd807d2aa700+24 0x26cff2e01dde9153!=0x26cff2e072d46f53 ddb{1}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic() at panic+0x15c sys/kern/subr_prf.c:207 pool_cache_get(ffffffff82644168) at pool_cache_get+0x323 pool_cache_item_magic_check sys/kern/subr_pool.c:1789 [inline] pool_cache_get(ffffffff82644168) at pool_cache_get+0x323 sys/kern/subr_pool.c:1892 pool_get() at pool_get+0x91 sys/kern/subr_pool.c:572 m_gethdr(1,1) at m_gethdr+0x4c sys/kern/uipc_mbuf.c:283 m_getuio(ffff800020aeaf18,0,4780,ffff800020aeb098) at m_getuio+0xe4 sys/kern/uipc_socket.c:556 sosend(fffffd806e977908,0,ffff800020aeb098,0,0,80) at sosend+0x510 sys/kern/uipc_socket.c:511 dofilewritev(ffff800020ac18c0,4,ffff800020aeb098,0,ffff800020aeb180) at dofilewritev+0x1b7 sys/kern/sys_generic.c:364 sys_write(ffff800020ac18c0,ffff800020aeb130,ffff800020aeb180) at sys_write+0x83 sys/kern/sys_generic.c:284 syscall(ffff800020aeb200) at syscall+0x4a4 mi_syscall sys/sys/syscall_mi.h:92 [inline] syscall(ffff800020aeb200) at syscall+0x4a4 sys/arch/amd64/amd64/trap.c:555 Xsyscall(0,4,182d28a1616b,4,4,182fa937d3c0) at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffe3830, count: -11 ddb{1}> show registers rdi 0 rsi 0x1 rbp 0xffff800020aeac50 rbx 0xffff800020aead00 rdx 0xffff800020ac18c0 rcx 0 rax 0 r8 0xffffffff8130092f kprintf+0x16f r9 0x1 r10 0x25 r11 0xf6eca0e95c4095c7 r12 0x3000000008 r13 0xffff800020aeac60 r14 0x100 r15 0x1 rip 0xffffffff81923338 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff800020aeac40 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb{1}> show proc PROC (sshd) pid=138808 stat=onproc flags process=12 proc=0 pri=50, usrpri=50, nice=20 forw=0xffffffffffffffff, list=0xffff800020ac1158,0xffff800020ac0298 process=0xffff800020a8bc10 user=0xffff800020ae6000, vmspace=0xfffffd806e7d45c8 estcpu=0, cpticks=3, pctcpu=0.0 user=0, sys=2, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 83772 197806 79767 0 2 0 syz-executor.1 83772 509404 79767 0 3 0x4000080 fifor syz-executor.1 83772 520309 79767 0 2 0x4000000 syz-executor.1 52764 142993 32955 0 2 0x2 syz-executor.0 79767 163403 32955 0 7 0x482 syz-executor.1 97547 213772 0 0 3 0x14200 acct acct 37422 522144 0 0 3 0x14200 bored sosplice 7071 381570 1 0 3 0x100083 ttyin getty 32955 407698 97176 0 3 0x82 thrsleep syz-fuzzer 32955 460490 97176 0 3 0x4000082 nanosleep syz-fuzzer 32955 454938 97176 0 3 0x4000082 kqread syz-fuzzer 32955 470699 97176 0 3 0x4000082 thrsleep syz-fuzzer 32955 423024 97176 0 3 0x4000082 nanosleep syz-fuzzer 32955 186782 97176 0 3 0x4000082 thrsleep syz-fuzzer 32955 71122 97176 0 3 0x4000082 thrsleep syz-fuzzer 32955 284881 97176 0 3 0x4000082 thrsleep syz-fuzzer 32955 79408 97176 0 3 0x4000082 thrsleep syz-fuzzer 32955 433407 97176 0 3 0x4000082 thrsleep syz-fuzzer 97176 156312 28607 0 3 0x10008a pause ksh *28607 138808 78681 0 7 0x12 sshd 78681 209856 1 0 3 0x80 select sshd 64748 154130 17778 74 3 0x100092 bpf pflogd 17778 361426 1 0 3 0x80 netio pflogd 98748 142340 15944 73 3 0x100090 kqread syslogd 15944 433844 1 0 3 0x100082 netio syslogd 19508 190375 1 77 3 0x100090 poll dhclient 1983 196962 1 0 3 0x80 poll dhclient 89065 426744 0 0 2 0x14200 zerothread 65972 42239 0 0 3 0x14200 aiodoned aiodoned 96941 378143 0 0 3 0x14200 syncer update 68284 460685 0 0 3 0x14200 cleaner cleaner 92507 146666 0 0 3 0x14200 reaper reaper 20435 516122 0 0 3 0x14200 pgdaemon pagedaemon 61955 34565 0 0 3 0x14200 bored crynlk 10695 211574 0 0 3 0x14200 bored crypto 97279 84386 0 0 3 0x40014200 acpi0 acpi0 37226 291933 0 0 3 0x40014200 idle1 43099 497655 0 0 3 0x14200 bored softnet 95406 257719 0 0 3 0x14200 bored systqmp 4086 437698 0 0 3 0x14200 bored systq 17844 1212 0 0 3 0x40014200 bored softclock 49263 428045 0 0 3 0x40014200 idle0 9276 469723 0 0 3 0x14200 bored smr 1 235564 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{1}> show all locks Process 52764 (syz-executor.0) thread 0xffff800020abe290 (142993) shared rwlock vmmaplk r = 0 (0xfffffd807f00b5d8) #0 witness_lock+0x52e sys/kern/subr_witness.c:1163 #1 uvmfault_lookup+0xd9 sys/uvm/uvm_fault.c:1448 #2 uvm_fault+0xd85 sys/uvm/uvm_fault.c:524 #3 pageflttrap+0x20b sys/arch/amd64/amd64/trap.c:199 #4 usertrap+0x21a sys/arch/amd64/amd64/trap.c:369 #5 recall_trap+0x8 exclusive kernel_lock &kernel_lock r = 0 (0xffffffff826668e0) #0 witness_lock+0x52e sys/kern/subr_witness.c:1163 #1 pageflttrap+0x6f sys/arch/amd64/amd64/trap.c:162 #2 usertrap+0x21a sys/arch/amd64/amd64/trap.c:369 #3 recall_trap+0x8 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim Kern Lim devbuf 9542 7065K 8037K 78643K 12457 0 0 pcb 13 8K 8K 78643K 172 0 0 rtable 94 8K 8K 78643K 622 0 0 ifaddr 63 13K 15K 78643K 162 0 0 counters 39 33K 33K 78643K 39 0 0 ioctlops 0 0K 4K 78643K 1503 0 0 iov 0 0K 24K 78643K 144 0 0 mount 1 1K 1K 78643K 1 0 0 vnodes 1220 77K 77K 78643K 1992 0 0 UFS quota 1 32K 32K 78643K 1 0 0 UFS mount 5 36K 36K 78643K 5 0 0 shm 2 1K 5K 78643K 10 0 0 VM map 2 1K 1K 78643K 11 0 0 sem 12 0K 1K 78643K 160 0 0 dirhash 12 2K 2K 78643K 12 0 0 ACPI 1808 196K 290K 78643K 12765 0 0 file desc 5 13K 25K 78643K 717 0 0 sigio 0 0K 0K 78643K 9 0 0 proc 60 63K 95K 78643K 722 0 0 subproc 32 2K 2K 78643K 119 0 0 NFS srvsock 1 0K 0K 78643K 1 0 0 NFS daemon 1 16K 16K 78643K 1 0 0 ip_moptions 0 0K 0K 78643K 84 0 0 in_multi 23 1K 2K 78643K 115 0 0 ether_multi 1 0K 0K 78643K 7 0 0 mrt 0 0K 0K 78643K 6 0 0 ISOFS mount 1 32K 32K 78643K 1 0 0 MSDOSFS mount 1 16K 16K 78643K 1 0 0 ttys 84 371K 371K 78643K 84 0 0 exec 0 0K 1K 78643K 384 0 0 pagedep 1 8K 8K 78643K 1 0 0 inodedep 1 32K 32K 78643K 1 0 0 newblk 1 0K 0K 78643K 1 0 0 VM swap 7 26K 26K 78643K 7 0 0 UVM amap 113 22K 23K 78643K 3440 0 0 UVM aobj 48 2K 2K 78643K 50 0 0 memdesc 1 4K 4K 78643K 1 0 0 crypto data 1 1K 1K 78643K 1 0 0 ip6_options 0 0K 0K 78643K 129 0 0 NDP 16 0K 0K 78643K 51 0 0 temp 187 3562K 4199K 78643K 10886 0 0 kqueue 0 0K 0K 78643K 12 0 0 SYN cache 2 16K 16K 78643K 2 0 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle arp 64 23 0 19 1 0 1 1 0 8 0 plcache 128 20 0 0 1 0 1 1 0 8 0 rtpcb 80 65 0 63 1 0 1 1 0 8 0 rtentry 112 117 0 83 2 0 2 2 0 8 0 unpcb 120 314 0 302 1 0 1 1 0 8 0 syncache 264 6 0 6 2 1 1 1 0 8 1 tcpqe 32 5889 0 5889 1 1 0 1 0 8 0 tcpcb 544 249 0 245 2 1 1 2 0 8 0 inpcb 280 757 0 750 5 3 2 2 0 8 1 rttmr 72 2 0 2 1 1 0 1 0 8 0 nd6 48 15 0 13 1 0 1 1 0 8 0 pkpcb 40 4 0 4 1 1 0 1 0 8 0 ppxss 1128 10 0 10 5 4 1 1 0 8 1 pffrag 232 11 0 11 2 1 1 1 0 482 1 pffrnode 88 11 0 11 2 1 1 1 0 8 1 pffrent 40 151 0 151 2 1 1 1 0 8 1 pfosfp 40 846 0 423 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfstitem 24 55 0 13 1 0 1 1 0 8 0 pfstkey 112 55 0 13 2 0 2 2 0 8 0 pfstate 328 55 0 13 4 0 4 4 0 8 0 pfrule 1360 21 0 16 2 1 1 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 499 0 327 16 2 14 15 0 8 3 art_table 32 500 0 327 2 0 2 2 0 8 0 art_node 16 116 0 85 1 0 1 1 0 8 0 sysvmsgpl 40 61 0 23 1 0 1 1 0 8 0 semapl 112 158 0 148 1 0 1 1 0 8 0 shmpl 112 48 0 2 2 0 2 2 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino1pl 128 2674 0 1269 46 0 46 46 0 8 0 ffsino 272 2674 0 1269 95 0 95 95 0 8 0 nchpl 144 3927 0 2324 61 0 61 61 0 8 0 uvmvnodes 72 3213 0 0 59 0 59 59 0 8 0 vnodes 208 3213 0 0 170 0 170 170 0 8 0 namei 1024 14475 0 14475 2 1 1 1 0 8 1 percpumem 16 30 0 0 1 0 1 1 0 8 0 vmpool 552 9 0 9 3 3 0 1 0 8 0 scxspl 192 12480 0 12480 15 14 1 7 0 8 1 plimitpl 152 84 0 76 1 0 1 1 0 8 0 sigapl 432 904 0 889 3 1 2 3 0 8 0 futexpl 56 14844 0 14844 2 1 1 1 0 8 1 knotepl 112 214 0 194 1 0 1 1 0 8 0 kqueuepl 104 174 0 171 1 0 1 1 0 8 0 pipepl 112 578 0 559 4 3 1 2 0 8 0 fdescpl 488 905 0 889 3 0 3 3 0 8 0 filepl 152 8040 0 7936 9 4 5 6 0 8 0 lockfpl 104 367 0 366 1 0 1 1 0 8 0 lockfspl 48 126 0 125 1 0 1 1 0 8 0 sessionpl 112 25 0 14 1 0 1 1 0 8 0 pgrppl 48 34 0 23 1 0 1 1 0 8 0 ucredpl 96 2417 0 2408 1 0 1 1 0 8 0 zombiepl 144 889 0 889 3 2 1 1 0 8 1 processpl 896 922 0 889 4 0 4 4 0 8 0 procpl 632 2403 0 2359 6 1 5 5 0 8 1 srpgc 64 10 0 10 4 3 1 1 0 8 1 sosppl 128 4 0 4 2 1 1 1 0 8 1 sockpl 384 1155 0 1134 8 5 3 4 0 8 0 mcl64k 65536 515 0 0 65 0 65 65 0 8 1 mcl16k 16384 4 0 0 1 0 1 1 0 8 0 mcl12k 12288 12 0 0 2 0 2 2 0 8 0 mcl9k 9216 4 0 0 1 0 1 1 0 8 0 mcl8k 8192 11 0 0 2 0 2 2 0 8 0 mcl4k 4096 17 0 0 3 0 3 3 0 8 0 mcl2k2 2112 3 0 0 1 0 1 1 0 8 0 mcl2k 2048 137 0 0 15 0 15 15 0 8 0 mtagpl 80 54 0 0 2 0 2 2 0 8 0 mbufpl 256 684 0 0 41 1 40 40 0 8 0 bufpl 256 8303 0 1325 437 0 437 437 0 8 0 anonpl 16 118787 0 104891 95 11 84 86 0 124 15 amapchunkpl 152 5777 0 5642 11 3 8 10 0 158 0 amappl16 192 5077 0 4103 84 28 56 60 0 8 7 amappl15 184 85 0 85 1 1 0 1 0 8 0 amappl14 176 154 0 150 2 1 1 1 0 8 0 amappl13 168 4 0 3 1 0 1 1 0 8 0 amappl12 160 18 0 16 1 0 1 1 0 8 0 amappl11 152 353 0 338 1 0 1 1 0 8 0 amappl10 144 128 0 126 1 0 1 1 0 8 0 amappl9 136 766 0 760 1 0 1 1 0 8 0 amappl8 128 332 0 304 2 0 2 2 0 8 0 amappl7 120 175 0 167 1 0 1 1 0 8 0 amappl6 112 349 0 339 1 0 1 1 0 8 0 amappl5 104 181 0 167 1 0 1 1 0 8 0 amappl4 96 1181 0 1146 2 1 1 2 0 8 0 amappl3 88 215 0 207 1 0 1 1 0 8 0 amappl2 80 5971 0 5900 4 2 2 3 0 8 0 amappl1 72 29947 0 29495 25 15 10 20 0 8 0 amappl 80 2751 0 2705 2 0 2 2 0 84 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 17 0 17 1 1 0 1 0 8 0 aobjpl 64 49 0 2 1 0 1 1 0 8 0 uaddrrnd 24 914 0 889 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 914 0 889 1 0 1 1 0 8 0 vmmpekpl 168 11348 0 11313 2 0 2 2 0 8 0 vmmpepl 168 122539 0 120414 177 49 128 128 0 357 31 vmsppl 368 904 0 889 2 0 2 2 0 8 0 pdppl 4096 1835 0 1796 7 1 6 6 0 8 0 pvpl 32 355178 0 337582 229 31 198 201 0 265 32 pmappl 232 913 0 898 5 4 1 2 0 8 0 extentpl 40 41 0 26 1 0 1 1 0 8 0 phpool 112 696 0 9 20 0 20 20 0 8 0