panic: pool_cache_item_magic_check: mbufpl cpu free list modified: item addr 0xfffffd806d583100+16 0x0!=0x4a2d883d7730b6ba Stopped at db_enter+0x18: addq $0x8,%rsp TID PID UID PRFLAGS PFLAGS CPU COMMAND 21355 67437 0 0x2 0x80 0 syz-executor.1 *370224 60402 0 0x12 0 1 sshd db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic() at panic+0x15c sys/kern/subr_prf.c:207 pool_cache_get(ffffffff826333a0) at pool_cache_get+0x323 pool_cache_item_magic_check sys/kern/subr_pool.c:1789 [inline] pool_cache_get(ffffffff826333a0) at pool_cache_get+0x323 sys/kern/subr_pool.c:1892 pool_get() at pool_get+0x91 sys/kern/subr_pool.c:572 m_copym(fffffd806d583f00,0,580,2) at m_copym+0x174 m_get sys/kern/uipc_mbuf.c:250 [inline] m_copym(fffffd806d583f00,0,580,2) at m_copym+0x174 sys/kern/uipc_mbuf.c:667 tcp_output(ffff800000a7acc0) at tcp_output+0x151c sys/netinet/tcp_output.c:673 tcp_usrreq(fffffd806e97dc08,9,fffffd806d583f00,0,0,ffff800020ac18c0) at tcp_usrreq+0xa45 sosend(fffffd806e97dc08,0,ffff800020aebb38,0,0,80) at sosend+0x645 sys/kern/uipc_socket.c:524 dofilewritev(ffff800020ac18c0,4,ffff800020aebb38,0,ffff800020aebc20) at dofilewritev+0x1b7 sys/kern/sys_generic.c:364 sys_write(ffff800020ac18c0,ffff800020aebbd0,ffff800020aebc20) at sys_write+0x83 sys/kern/sys_generic.c:284 syscall(ffff800020aebca0) at syscall+0x4a4 mi_syscall sys/sys/syscall_mi.h:92 [inline] syscall(ffff800020aebca0) at syscall+0x4a4 sys/arch/amd64/amd64/trap.c:555 Xsyscall(6,4,1e2dea91616b,4,4,1e30bd22dd80) at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffc1ef0, count: 3 https://www.openbsd.org/ddb.html describes the minimum info required in bug reports. Insufficient info makes it difficult to find and fix bugs. ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic pool_cache_item_magic_check: mbufpl cpu free list modified: item addr 0xfffffd806d583100+16 0x0!=0x4a2d883d7730b6ba ddb{1}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic() at panic+0x15c sys/kern/subr_prf.c:207 pool_cache_get(ffffffff826333a0) at pool_cache_get+0x323 pool_cache_item_magic_check sys/kern/subr_pool.c:1789 [inline] pool_cache_get(ffffffff826333a0) at pool_cache_get+0x323 sys/kern/subr_pool.c:1892 pool_get() at pool_get+0x91 sys/kern/subr_pool.c:572 m_copym(fffffd806d583f00,0,580,2) at m_copym+0x174 m_get sys/kern/uipc_mbuf.c:250 [inline] m_copym(fffffd806d583f00,0,580,2) at m_copym+0x174 sys/kern/uipc_mbuf.c:667 tcp_output(ffff800000a7acc0) at tcp_output+0x151c sys/netinet/tcp_output.c:673 tcp_usrreq(fffffd806e97dc08,9,fffffd806d583f00,0,0,ffff800020ac18c0) at tcp_usrreq+0xa45 sosend(fffffd806e97dc08,0,ffff800020aebb38,0,0,80) at sosend+0x645 sys/kern/uipc_socket.c:524 dofilewritev(ffff800020ac18c0,4,ffff800020aebb38,0,ffff800020aebc20) at dofilewritev+0x1b7 sys/kern/sys_generic.c:364 sys_write(ffff800020ac18c0,ffff800020aebbd0,ffff800020aebc20) at sys_write+0x83 sys/kern/sys_generic.c:284 syscall(ffff800020aebca0) at syscall+0x4a4 mi_syscall sys/sys/syscall_mi.h:92 [inline] syscall(ffff800020aebca0) at syscall+0x4a4 sys/arch/amd64/amd64/trap.c:555 Xsyscall(6,4,1e2dea91616b,4,4,1e30bd22dd80) at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffc1ef0, count: -12 ddb{1}> show registers rdi 0 rsi 0x1 rbp 0xffff800020aeb4e0 rbx 0xffff800020aeb590 rdx 0xffff800020ac18c0 rcx 0 rax 0 r8 0xffffffff81988e9f kprintf+0x16f r9 0x1 r10 0x25 r11 0x2beed262405bb1a6 r12 0x3000000008 r13 0xffff800020aeb4f0 r14 0x100 r15 0x1 rip 0xffffffff820d9a08 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff800020aeb4d0 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb{1}> show proc PROC (sshd) pid=370224 stat=onproc flags process=12 proc=0 pri=50, usrpri=50, nice=20 forw=0xffffffffffffffff, list=0xffff800020ac0ee0,0xffff800020ac0510 process=0xffff800020a8bc10 user=0xffff800020ae6000, vmspace=0xfffffd806e7db178 estcpu=0, cpticks=1, pctcpu=0.0 user=0, sys=1, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 56878 172203 81033 0 2 0x2 syz-executor.0 67437 21355 81033 0 7 0x82 syz-executor.1 77630 352521 0 0 3 0x14200 acct acct 91176 55648 1 0 3 0x100083 ttyin getty 7737 474698 0 0 3 0x14200 bored sosplice 81033 67623 61896 0 2 0x2 syz-fuzzer 81033 52952 61896 0 3 0x4000082 nanosleep syz-fuzzer 81033 222239 61896 0 3 0x4000082 thrsleep syz-fuzzer 81033 436248 61896 0 3 0x4000082 thrsleep syz-fuzzer 81033 419611 61896 0 3 0x4000082 thrsleep syz-fuzzer 81033 485772 61896 0 3 0x4000082 thrsleep syz-fuzzer 81033 377184 61896 0 3 0x4000082 thrsleep syz-fuzzer 81033 205813 61896 0 3 0x4000082 nanosleep syz-fuzzer 81033 455328 61896 0 3 0x4000082 thrsleep syz-fuzzer 81033 206945 61896 0 3 0x4000082 thrsleep syz-fuzzer 61896 156264 60402 0 3 0x10008a pause ksh *60402 370224 69688 0 7 0x12 sshd 69688 379501 1 0 3 0x80 select sshd 98500 335433 37253 74 3 0x100092 bpf pflogd 37253 492011 1 0 3 0x80 netio pflogd 31859 20268 98734 73 3 0x100090 kqread syslogd 98734 498013 1 0 3 0x100082 netio syslogd 11847 243150 1 77 3 0x100090 poll dhclient 47389 516432 1 0 3 0x80 poll dhclient 39821 277586 0 0 3 0x14200 pgzero zerothread 97134 161940 0 0 3 0x14200 aiodoned aiodoned 444 168770 0 0 3 0x14200 syncer update 82100 28409 0 0 3 0x14200 cleaner cleaner 16065 97469 0 0 3 0x14200 reaper reaper 59931 244528 0 0 3 0x14200 pgdaemon pagedaemon 11968 288557 0 0 3 0x14200 bored crynlk 85069 411053 0 0 3 0x14200 bored crypto 14456 523095 0 0 3 0x40014200 acpi0 acpi0 58904 73659 0 0 3 0x40014200 idle1 96442 37853 0 0 3 0x14200 bored softnet 33088 275346 0 0 3 0x14200 bored systqmp 30196 409289 0 0 3 0x14200 bored systq 82761 310714 0 0 3 0x40014200 bored softclock 21935 83269 0 0 3 0x40014200 idle0 88126 380417 0 0 3 0x14200 bored smr 1 42742 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{1}> show all locks Process 56878 (syz-executor.0) thread 0xffff800020abf3d8 (172203) exclusive rrwlock inode r = 0 (0xfffffd806f79fd60) #0 witness_lock+0x52e sys/kern/subr_witness.c:1163 #1 rw_enter+0x447 sys/kern/kern_rwlock.c:306 #2 rrw_enter+0x4f sys/kern/kern_rwlock.c:435 #3 VOP_LOCK+0xf0 sys/kern/vfs_vops.c:615 #4 vn_lock+0x81 sys/kern/vfs_vnops.c:562 #5 vget+0x1c3 sys/kern/vfs_subr.c:672 #6 ufs_ihashget+0x141 sys/ufs/ufs/ufs_ihash.c:119 #7 ffs_vget+0x74 sys/ufs/ffs/ffs_vfsops.c:1323 #8 ufs_lookup+0x14b4 sys/ufs/ufs/ufs_lookup.c:487 #9 VOP_LOOKUP+0x5b sys/kern/vfs_vops.c:91 #10 vfs_lookup+0x7a6 sys/kern/vfs_lookup.c:568 #11 namei+0x63c sys/kern/vfs_lookup.c:249 #12 dounlinkat+0x99 sys/kern/vfs_syscalls.c:1785 #13 syscall+0x4a4 mi_syscall sys/sys/syscall_mi.h:92 [inline] #13 syscall+0x4a4 sys/arch/amd64/amd64/trap.c:555 #14 Xsyscall+0x128 exclusive rrwlock inode r = 0 (0xfffffd806fa5a1b0) #0 witness_lock+0x52e sys/kern/subr_witness.c:1163 #1 rw_enter+0x447 sys/kern/kern_rwlock.c:306 #2 rrw_enter+0x4f sys/kern/kern_rwlock.c:435 #3 VOP_LOCK+0xf0 sys/kern/vfs_vops.c:615 #4 vn_lock+0x81 sys/kern/vfs_vnops.c:562 #5 vfs_lookup+0xe6 sys/kern/vfs_lookup.c:419 #6 namei+0x63c sys/kern/vfs_lookup.c:249 #7 dounlinkat+0x99 sys/kern/vfs_syscalls.c:1785 #8 syscall+0x4a4 mi_syscall sys/sys/syscall_mi.h:92 [inline] #8 syscall+0x4a4 sys/arch/amd64/amd64/trap.c:555 #9 Xsyscall+0x128 Process 81033 (syz-fuzzer) thread 0xffff800020ac0c68 (67623) exclusive kernel_lock &kernel_lock r = 1 (0xffffffff826404e0) #0 witness_lock+0x52e sys/kern/subr_witness.c:1163 #1 intr_handler+0x5e sys/arch/amd64/amd64/intr.c:525 #2 Xintr_ioapic_edge17_untramp+0x19f Process 60402 (sshd) thread 0xffff800020ac18c0 (370224) exclusive rwlock netlock r = 0 (0xffffffff824dba68) #0 witness_lock+0x52e sys/kern/subr_witness.c:1163 #1 solock+0x5a sys/kern/uipc_socket2.c:282 #2 sosend+0x51b sys/kern/uipc_socket.c:512 #3 dofilewritev+0x1b7 sys/kern/sys_generic.c:364 #4 sys_write+0x83 sys/kern/sys_generic.c:284 #5 syscall+0x4a4 mi_syscall sys/sys/syscall_mi.h:92 [inline] #5 syscall+0x4a4 sys/arch/amd64/amd64/trap.c:555 #6 Xsyscall+0x128 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim Kern Lim devbuf 9610 7083K 7795K 78643K 17381 0 0 pcb 13 10K 12K 78643K 738 0 0 rtable 102 8K 9K 78643K 1849 0 0 ifaddr 101 22K 24K 78643K 646 0 0 counters 39 33K 33K 78643K 39 0 0 ioctlops 0 0K 4K 78643K 1665 0 0 iov 0 0K 18K 78643K 564 0 0 mount 1 1K 1K 78643K 1 0 0 vnodes 1210 76K 77K 78643K 3981 0 0 UFS quota 1 32K 32K 78643K 1 0 0 UFS mount 5 36K 36K 78643K 5 0 0 shm 2 1K 5K 78643K 34 0 0 VM map 13 6K 7K 78643K 26 0 0 sem 12 0K 0K 78643K 1002 0 0 dirhash 12 2K 2K 78643K 12 0 0 ACPI 1808 196K 290K 78643K 12765 0 0 file desc 4 9K 25K 78643K 3180 0 0 sigio 0 0K 0K 78643K 19 0 0 proc 62 63K 95K 78643K 1440 0 0 subproc 32 2K 2K 78643K 308 0 0 NFS srvsock 1 0K 0K 78643K 1 0 0 NFS daemon 1 16K 16K 78643K 1 0 0 ip_moptions 0 0K 0K 78643K 302 0 0 in_multi 30 1K 2K 78643K 404 0 0 ether_multi 1 0K 0K 78643K 36 0 0 mrt 0 0K 0K 78643K 36 0 0 ISOFS mount 1 32K 32K 78643K 1 0 0 MSDOSFS mount 1 16K 16K 78643K 1 0 0 ttys 96 424K 424K 78643K 96 0 0 exec 0 0K 1K 78643K 684 0 0 pagedep 1 8K 8K 78643K 1 0 0 inodedep 1 32K 32K 78643K 1 0 0 newblk 1 0K 0K 78643K 1 0 0 VM swap 7 26K 26K 78643K 7 0 0 UVM amap 129 136K 145K 78643K 14306 0 0 UVM aobj 106 5K 5K 78643K 109 0 0 memdesc 1 4K 4K 78643K 1 0 0 crypto data 1 1K 1K 78643K 1 0 0 ip6_options 0 0K 0K 78643K 703 0 0 NDP 23 0K 1K 78643K 199 0 0 temp 255 3561K 4201K 78643K 125841 0 0 kqueue 0 0K 0K 78643K 24 0 0 SYN cache 2 16K 16K 78643K 2 0 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle arp 64 36 0 31 1 0 1 1 0 8 0 plcache 128 20 0 0 1 0 1 1 0 8 0 rtpcb 80 334 0 332 1 0 1 1 0 8 0 rtentry 112 294 0 259 2 0 2 2 0 8 0 unpcb 120 1590 0 1579 1 0 1 1 0 8 0 syncache 264 12 0 12 6 6 0 1 0 8 0 sackhl 24 9 0 9 6 6 0 1 0 8 0 tcpqe 32 8174 0 8174 3 3 0 2 0 8 0 tcpcb 544 1419 0 1415 14 13 1 13 0 8 0 inpcb 280 7241 0 7234 25 22 3 9 0 8 2 rttmr 72 11 0 11 10 9 1 1 0 8 1 nd6 48 49 0 47 3 2 1 1 0 8 0 pkpcb 40 19 0 19 7 7 0 1 0 8 0 ppxss 1128 74 0 74 22 21 1 1 0 8 1 pffrag 232 47 0 47 22 22 0 1 0 482 0 pffrnode 88 47 0 47 22 22 0 1 0 8 0 pffrent 40 1177 0 1177 22 22 0 1 0 8 0 pfosfp 40 846 0 423 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfstitem 24 301 0 266 1 0 1 1 0 8 0 pfstkey 112 301 0 266 2 0 2 2 0 8 0 pfstate 328 301 0 266 5 1 4 5 0 8 0 pfrule 1360 21 0 16 2 1 1 2 0 8 0 art_heap8 4096 5 0 4 4 2 2 2 0 8 1 art_heap4 256 1360 0 1186 28 15 13 15 0 8 0 art_table 32 1365 0 1190 2 0 2 2 0 8 0 art_node 16 292 0 260 1 0 1 1 0 8 0 sysvmsgpl 40 54 0 32 2 1 1 1 0 8 0 semupl 112 2 0 2 1 1 0 1 0 8 0 semapl 112 998 0 988 1 0 1 1 0 8 0 shmpl 112 107 0 3 3 0 3 3 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino1pl 128 6356 0 4950 46 0 46 46 0 8 0 ffsino 272 6356 0 4950 96 1 95 95 0 8 0 nchpl 144 11420 0 10953 61 41 20 61 0 8 0 uvmvnodes 72 5926 0 0 108 0 108 108 0 8 0 vnodes 208 5926 0 0 312 0 312 312 0 8 0 namei 1024 46874 0 46874 4 3 1 1 0 8 1 percpumem 16 30 0 0 1 0 1 1 0 8 0 vcpupl 1984 12 0 1 2 0 2 2 0 8 0 vmpool 552 24 0 13 4 3 1 1 0 8 0 scsiplug 64 5 0 5 4 4 0 1 0 8 0 scxspl 192 37057 0 37057 36 35 1 7 0 8 1 plimitpl 152 282 0 274 1 0 1 1 0 8 0 sigapl 432 3334 0 3320 3 1 2 3 0 8 0 futexpl 56 78647 0 78647 2 1 1 1 0 8 1 knotepl 112 937 0 918 4 3 1 2 0 8 0 kqueuepl 104 1026 0 1024 7 6 1 4 0 8 0 pipepl 112 5066 0 5047 14 12 2 2 0 8 1 fdescpl 488 3335 0 3320 3 0 3 3 0 8 0 filepl 152 37477 0 37377 45 39 6 14 0 8 1 lockfpl 104 1065 0 1064 1 0 1 1 0 8 0 lockfspl 48 361 0 360 1 0 1 1 0 8 0 sessionpl 112 36 0 25 1 0 1 1 0 8 0 pgrppl 48 58 0 47 1 0 1 1 0 8 0 ucredpl 96 9571 0 9561 1 0 1 1 0 8 0 zombiepl 144 3320 0 3320 3 2 1 1 0 8 1 processpl 896 3352 0 3320 4 0 4 4 0 8 0 procpl 632 12691 0 12650 6 0 6 6 0 8 1 srpgc 64 28 0 28 13 13 0 1 0 8 0 sosppl 128 28 0 28 11 11 0 1 0 8 0 sockpl 384 9214 0 9194 47 42 5 14 0 8 2 mcl64k 65536 265 0 0 34 1 33 34 0 8 1 mcl16k 16384 12 0 0 2 0 2 2 0 8 0 mcl12k 12288 18 0 0 2 0 2 2 0 8 0 mcl9k 9216 14 0 0 1 0 1 1 0 8 0 mcl8k 8192 17 0 0 3 0 3 3 0 8 0 mcl4k 4096 31 0 0 4 2 2 3 0 8 0 mcl2k2 2112 7 0 0 1 0 1 1 0 8 0 mcl2k 2048 299 0 0 33 15 18 32 0 8 0 mtagpl 80 42 0 0 1 0 1 1 0 8 0 mbufpl 256 781 0 0 28 0 28 28 0 8 0 bufpl 256 18981 0 11933 441 0 441 441 0 8 0 anonpl 16 448246 0 430947 232 151 81 93 0 124 0 amapchunkpl 152 27640 0 27526 49 43 6 13 0 158 0 amappl16 192 20712 0 19660 230 174 56 66 0 8 2 amappl15 184 432 0 430 1 0 1 1 0 8 0 amappl14 176 528 0 526 2 1 1 1 0 8 0 amappl13 168 245 0 244 7 6 1 1 0 8 0 amappl12 160 477 0 477 6 6 0 1 0 8 0 amappl11 152 604 0 589 1 0 1 1 0 8 0 amappl10 144 177 0 170 1 0 1 1 0 8 0 amappl9 136 1502 0 1495 1 0 1 1 0 8 0 amappl8 128 1054 0 1022 4 2 2 2 0 8 0 amappl7 120 278 0 270 1 0 1 1 0 8 0 amappl6 112 607 0 594 1 0 1 1 0 8 0 amappl5 104 756 0 743 1 0 1 1 0 8 0 amappl4 96 3476 0 3443 1 0 1 1 0 8 0 amappl3 88 751 0 744 1 0 1 1 0 8 0 amappl2 80 25237 0 25165 4 2 2 3 0 8 0 amappl1 72 89006 0 88583 25 15 10 20 0 8 0 amappl 80 13220 0 13176 2 0 2 2 0 84 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 17 0 17 1 1 0 1 0 8 0 aobjpl 64 108 0 3 2 0 2 2 0 8 0 uaddrrnd 24 3359 0 3320 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 3359 0 3320 1 0 1 1 0 8 0 vmmpekpl 168 44415 0 44377 3 0 3 3 0 8 0 vmmpepl 168 441769 0 439648 495 352 143 143 0 357 38 vmsppl 368 3334 0 3320 2 0 2 2 0 8 0 pdppl 4096 6725 0 6677 10 3 7 7 0 8 0 pvpl 32 1187795 0 1167616 455 275 180 216 0 265 0 pmappl 232 3358 0 3333 9 7 2 2 0 8 0 extentpl 40 41 0 26 1 0 1 1 0 8 0 phpool 112 710 0 58 19 0 19 19 0 8 0