panic: pool_cache_item_magic_check: mbufpl cpu free list modified: item addr 0xfffffd806559d000+16 0x0!=0x4ee92198c7025715 Stopped at db_enter+0x18: addq $0x8,%rsp TID PID UID PRFLAGS PFLAGS CPU COMMAND *157385 1832 0 0x2 0 0K ndp db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic() at panic+0x15c sys/kern/subr_prf.c:207 pool_cache_get(ffffffff82633050) at pool_cache_get+0x323 pool_cache_item_magic_check sys/kern/subr_pool.c:1789 [inline] pool_cache_get(ffffffff82633050) at pool_cache_get+0x323 sys/kern/subr_pool.c:1892 pool_get() at pool_get+0x91 sys/kern/subr_pool.c:572 m_get(1,4) at m_get+0x4c sys/kern/uipc_mbuf.c:250 sys_setsockopt(ffff800020ace298,ffff800022d73320,ffff800022d73370) at sys_setsockopt+0x12c sys/kern/uipc_syscalls.c:949 syscall(ffff800022d733f0) at syscall+0x4a4 mi_syscall sys/sys/syscall_mi.h:92 [inline] syscall(ffff800022d733f0) at syscall+0x4a4 sys/arch/amd64/amd64/trap.c:555 Xsyscall(6,69,7f7ffffea47f,69,7f7ffffea3c8,2) at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffea2a0, count: 7 https://www.openbsd.org/ddb.html describes the minimum info required in bug reports. Insufficient info makes it difficult to find and fix bugs. ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic pool_cache_item_magic_check: mbufpl cpu free list modified: item addr 0xfffffd806559d000+16 0x0!=0x4ee92198c7025715 ddb{0}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic() at panic+0x15c sys/kern/subr_prf.c:207 pool_cache_get(ffffffff82633050) at pool_cache_get+0x323 pool_cache_item_magic_check sys/kern/subr_pool.c:1789 [inline] pool_cache_get(ffffffff82633050) at pool_cache_get+0x323 sys/kern/subr_pool.c:1892 pool_get() at pool_get+0x91 sys/kern/subr_pool.c:572 m_get(1,4) at m_get+0x4c sys/kern/uipc_mbuf.c:250 sys_setsockopt(ffff800020ace298,ffff800022d73320,ffff800022d73370) at sys_setsockopt+0x12c sys/kern/uipc_syscalls.c:949 syscall(ffff800022d733f0) at syscall+0x4a4 mi_syscall sys/sys/syscall_mi.h:92 [inline] syscall(ffff800022d733f0) at syscall+0x4a4 sys/arch/amd64/amd64/trap.c:555 Xsyscall(6,69,7f7ffffea47f,69,7f7ffffea3c8,2) at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffea2a0, count: -8 ddb{0}> show registers rdi 0 rsi 0x1 rbp 0xffff800022d73090 rbx 0xffff800022d73140 rdx 0xffff800020ace298 rcx 0 rax 0 r8 0xffffffff8150e41f kprintf+0x16f r9 0x1 r10 0x25 r11 0xd2e6da4aeda5b396 r12 0x3000000008 r13 0xffff800022d730a0 r14 0x100 r15 0x1 rip 0xffffffff814f2878 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff800022d73080 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb{0}> show proc PROC (ndp) pid=157385 stat=onproc flags process=2 proc=0 pri=71, usrpri=71, nice=20 forw=0xffffffffffffffff, list=0xffff800020acec78,0xffffffff8264e048 process=0xffff800020a8b890 user=0xffff800022d6e000, vmspace=0xfffffd807f00b450 estcpu=36, cpticks=1, pctcpu=0.0 user=0, sys=1, intr=0 ddb{0}> ps PID TID PPID UID S FLAGS WAIT COMMAND * 1832 157385 57038 0 7 0x2 ndp 57038 483561 18660 0 3 0x10008a pause sh 18660 121336 71501 0 3 0x82 wait syz-executor.1 69669 189588 71501 0 3 0x82 piperd syz-executor.0 88209 207247 1 0 3 0x100083 ttyopn getty 29021 42847 0 0 3 0x14200 bored sosplice 71501 7027 90225 0 3 0x82 thrsleep syz-fuzzer 71501 360400 90225 0 3 0x4000082 thrsleep syz-fuzzer 71501 130573 90225 0 3 0x4000082 kqread syz-fuzzer 71501 132367 90225 0 3 0x4000082 thrsleep syz-fuzzer 71501 102070 90225 0 3 0x4000082 thrsleep syz-fuzzer 71501 503748 90225 0 3 0x4000082 thrsleep syz-fuzzer 71501 422580 90225 0 3 0x4000082 thrsleep syz-fuzzer 71501 278706 90225 0 3 0x4000082 thrsleep syz-fuzzer 71501 314565 90225 0 3 0x4000082 thrsleep syz-fuzzer 71501 314439 90225 0 3 0x4000082 thrsleep syz-fuzzer 90225 70325 64246 0 3 0x10008a pause ksh 64246 168257 46207 0 3 0x92 select sshd 46207 145864 1 0 3 0x80 select sshd 14180 297919 99594 74 3 0x100092 bpf pflogd 99594 80644 1 0 3 0x80 netio pflogd 63564 463304 91952 73 3 0x100090 kqread syslogd 91952 193050 1 0 3 0x100082 netio syslogd 80226 91062 1 77 3 0x100090 poll dhclient 79759 495204 1 0 3 0x80 poll dhclient 2408 169581 0 0 3 0x14200 pgzero zerothread 97653 23817 0 0 3 0x14200 aiodoned aiodoned 37418 87840 0 0 3 0x14200 syncer update 5398 236087 0 0 3 0x14200 cleaner cleaner 49914 13263 0 0 3 0x14200 reaper reaper 22146 463624 0 0 3 0x14200 pgdaemon pagedaemon 38316 262240 0 0 3 0x14200 bored crynlk 72444 127211 0 0 3 0x14200 bored crypto 3593 228843 0 0 3 0x40014200 acpi0 acpi0 9432 328529 0 0 7 0x40014200 idle1 52905 208871 0 0 3 0x14200 bored softnet 72890 266483 0 0 3 0x14200 bored systqmp 73914 16628 0 0 3 0x14200 bored systq 23071 263274 0 0 3 0x40014200 bored softclock 49575 330222 0 0 3 0x40014200 idle0 95514 353093 0 0 3 0x14200 bored smr 1 332643 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{0}> show all locks Process 1832 (ndp) thread 0xffff800020ace298 (157385) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff826567d8) #0 witness_lock+0x52e sys/kern/subr_witness.c:1163 #1 syscall+0x400 mi_syscall sys/sys/syscall_mi.h:83 [inline] #1 syscall+0x400 sys/arch/amd64/amd64/trap.c:555 #2 Xsyscall+0x128 ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim Kern Lim devbuf 9579 6444K 11461K 78643K 24102 0 0 pcb 13 10K 12K 78643K 329 0 0 rtable 108 9K 9K 78643K 1289 0 0 ifaddr 83 17K 18K 78643K 383 0 0 counters 39 33K 33K 78643K 39 0 0 ioctlops 0 0K 4K 78643K 1565 0 0 iov 0 0K 24K 78643K 325 0 0 mount 1 1K 1K 78643K 1 0 0 vnodes 1313 82K 83K 78643K 4826 0 0 UFS quota 1 32K 32K 78643K 1 0 0 UFS mount 5 36K 36K 78643K 5 0 0 shm 2 1K 9K 78643K 25 0 0 VM map 9 4K 4K 78643K 10 0 0 sem 12 0K 0K 78643K 1349 0 0 dirhash 12 2K 2K 78643K 12 0 0 ACPI 1808 196K 290K 78643K 12765 0 0 file desc 6 17K 25K 78643K 1663 0 0 sigio 0 0K 0K 78643K 23 0 0 proc 62 63K 95K 78643K 1089 0 0 subproc 32 2K 2K 78643K 238 0 0 NFS srvsock 1 0K 0K 78643K 1 0 0 NFS daemon 1 16K 16K 78643K 1 0 0 ip_moptions 0 0K 0K 78643K 110 0 0 in_multi 24 1K 2K 78643K 252 0 0 ether_multi 1 0K 0K 78643K 17 0 0 mrt 0 0K 0K 78643K 13 0 0 ISOFS mount 1 32K 32K 78643K 1 0 0 MSDOSFS mount 1 16K 16K 78643K 1 0 0 ttys 78 344K 344K 78643K 78 0 0 exec 0 0K 1K 78643K 498 0 0 pagedep 1 8K 8K 78643K 1 0 0 inodedep 1 32K 32K 78643K 1 0 0 newblk 1 0K 0K 78643K 1 0 0 VM swap 7 26K 26K 78643K 7 0 0 UVM amap 129 88K 89K 78643K 6975 0 0 UVM aobj 105 3K 4K 78643K 115 0 0 memdesc 1 4K 4K 78643K 1 0 0 crypto data 1 1K 1K 78643K 1 0 0 ip6_options 0 0K 0K 78643K 319 0 0 NDP 22 0K 0K 78643K 123 0 0 temp 229 3557K 3633K 78643K 62224 0 0 kqueue 0 0K 0K 78643K 19 0 0 SYN cache 2 16K 16K 78643K 2 0 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle arp 64 37 0 32 1 0 1 1 0 8 0 plcache 128 20 0 0 1 0 1 1 0 8 0 rtpcb 80 146 0 143 1 0 1 1 0 8 0 rtentry 112 237 0 199 2 0 2 2 0 8 0 unpcb 120 790 0 780 1 0 1 1 0 8 0 syncache 264 12 0 12 6 5 1 1 0 8 1 tcpqe 32 3 0 3 3 3 0 1 0 8 0 tcpcb 544 883 0 879 13 12 1 13 0 8 0 inpcb 280 2522 0 2515 15 13 2 9 0 8 1 rttmr 72 3 0 2 3 2 1 1 0 8 0 ip6q 72 1 0 1 1 1 0 1 0 8 0 ip6af 40 1 0 1 1 1 0 1 0 8 0 nd6 48 37 0 35 2 1 1 1 0 8 0 pkpcb 40 6 0 6 1 1 0 1 0 8 0 ppxss 1128 44 0 44 7 7 0 1 0 8 0 pffrag 232 52 0 52 7 6 1 1 0 482 1 pffrnode 88 52 0 52 7 6 1 1 0 8 1 pffrent 40 1246 0 1246 7 6 1 1 0 8 1 pfosfp 40 846 0 423 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfstitem 24 151 0 97 1 0 1 1 0 8 0 pfstkey 112 151 0 97 2 0 2 2 0 8 0 pfstate 328 151 0 97 5 0 5 5 0 8 0 pfrule 1360 21 0 16 2 1 1 2 0 8 0 art_heap8 4096 8 0 6 6 3 3 3 0 8 1 art_heap4 256 1090 0 886 23 8 15 16 0 8 2 art_table 32 1098 0 892 3 1 2 3 0 8 0 art_node 16 236 0 201 1 0 1 1 0 8 0 sysvmsgpl 40 25 0 15 1 0 1 1 0 8 0 semapl 112 1347 0 1337 1 0 1 1 0 8 0 shmpl 112 113 0 10 3 0 3 3 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino1pl 128 4113 0 2712 46 0 46 46 0 8 0 ffsino 272 4113 0 2712 97 2 95 95 0 8 0 nchpl 144 7063 0 5462 61 0 61 61 0 8 0 uvmvnodes 72 5926 0 0 108 0 108 108 0 8 0 vnodes 208 5926 0 0 312 0 312 312 0 8 0 namei 1024 27315 0 27315 1 0 1 1 0 8 1 percpumem 16 30 0 0 1 0 1 1 0 8 0 vcpupl 1984 7 0 0 1 0 1 1 0 8 0 vmpool 552 8 0 1 2 1 1 1 0 8 0 scsiplug 64 2 0 2 1 1 0 1 0 8 0 scxspl 192 22709 0 22709 17 14 3 7 0 8 3 plimitpl 152 170 0 162 1 0 1 1 0 8 0 sigapl 432 1829 0 1813 3 1 2 3 0 8 0 futexpl 56 47207 0 47207 1 0 1 1 0 8 1 knotepl 112 580 0 559 2 1 1 2 0 8 0 kqueuepl 104 589 0 587 4 3 1 4 0 8 0 pipepl 112 1250 0 1231 7 5 2 2 0 8 1 fdescpl 488 1830 0 1813 3 0 3 3 0 8 0 filepl 152 18359 0 18258 33 26 7 14 0 8 3 lockfpl 104 611 0 610 1 0 1 1 0 8 0 lockfspl 48 203 0 202 1 0 1 1 0 8 0 sessionpl 112 32 0 21 1 0 1 1 0 8 0 pgrppl 48 49 0 38 1 0 1 1 0 8 0 ucredpl 96 1978 0 1969 1 0 1 1 0 8 0 zombiepl 144 1815 0 1815 1 0 1 1 0 8 1 processpl 896 1848 0 1815 4 0 4 4 0 8 0 procpl 632 5564 0 5522 5 0 5 5 0 8 0 srpgc 64 16 0 16 7 6 1 1 0 8 1 sosppl 128 26 0 26 8 8 0 1 0 8 0 sockpl 384 3488 0 3468 22 18 4 14 0 8 1 mcl64k 65536 323 0 0 39 6 33 34 0 8 1 mcl16k 16384 17 0 0 3 0 3 3 0 8 0 mcl12k 12288 17 0 0 2 0 2 2 0 8 0 mcl9k 9216 10 0 0 1 0 1 1 0 8 0 mcl8k 8192 7 0 0 1 0 1 1 0 8 0 mcl4k 4096 16 0 0 2 0 2 2 0 8 0 mcl2k2 2112 3 0 0 1 0 1 1 0 8 0 mcl2k 2048 168 0 0 19 0 19 19 0 8 0 mtagpl 80 34 0 0 1 0 1 1 0 8 0 mbufpl 256 500 0 0 29 0 29 29 0 8 0 bufpl 256 12248 0 5200 441 0 441 441 0 8 0 anonpl 16 259111 0 244980 127 50 77 92 0 124 0 amapchunkpl 152 13786 0 13652 29 18 11 12 0 158 5 amappl16 192 10698 0 9661 101 47 54 64 0 8 0 amappl15 184 334 0 333 3 2 1 1 0 8 0 amappl14 176 142 0 137 1 0 1 1 0 8 0 amappl13 168 278 0 277 3 2 1 1 0 8 0 amappl12 160 79 0 75 2 1 1 1 0 8 0 amappl11 152 225 0 207 1 0 1 1 0 8 0 amappl10 144 304 0 299 1 0 1 1 0 8 0 amappl9 136 1038 0 1031 1 0 1 1 0 8 0 amappl8 128 572 0 539 2 0 2 2 0 8 0 amappl7 120 384 0 376 1 0 1 1 0 8 0 amappl6 112 224 0 212 1 0 1 1 0 8 0 amappl5 104 328 0 311 1 0 1 1 0 8 0 amappl4 96 2398 0 2367 1 0 1 1 0 8 0 amappl3 88 276 0 269 1 0 1 1 0 8 0 amappl2 80 13250 0 13179 3 1 2 3 0 8 0 amappl1 72 51240 0 50755 25 15 10 20 0 8 0 amappl 80 6063 0 6019 3 1 2 2 0 84 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 17 0 17 1 1 0 1 0 8 0 aobjpl 64 114 0 10 2 0 2 2 0 8 0 uaddrrnd 24 1838 0 1813 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 1838 0 1813 1 0 1 1 0 8 0 vmmpekpl 168 19352 0 19317 2 0 2 2 0 8 0 vmmpepl 168 242199 0 239992 212 70 142 142 0 357 39 vmsppl 368 1829 0 1813 2 0 2 2 0 8 0 pdppl 4096 3683 0 3635 7 0 7 7 0 8 0 pvpl 32 670909 0 653650 272 81 191 215 0 265 14 pmappl 232 1837 0 1814 4 2 2 2 0 8 0 extentpl 40 41 0 26 1 0 1 1 0 8 0 phpool 112 690 0 34 19 0 19 19 0 8 0