panic: pool_cache_item_magic_check: mbufpl cpu free list modified: item addr 0xfffffd806d796d00+16 0x0!=0x614a2d3523e14806 Stopped at db_enter+0x18: addq $0x8,%rsp TID PID UID PRFLAGS PFLAGS CPU COMMAND 337130 79321 0 0x2 0x4000000 0K syz-fuzzer *200644 34891 0 0x12 0 1 sshd db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic() at panic+0x15c sys/kern/subr_prf.c:207 pool_cache_get(ffffffff82647520) at pool_cache_get+0x323 pool_cache_item_magic_check sys/kern/subr_pool.c:1789 [inline] pool_cache_get(ffffffff82647520) at pool_cache_get+0x323 sys/kern/subr_pool.c:1892 pool_get() at pool_get+0x91 sys/kern/subr_pool.c:572 m_copym(fffffd806f0a6b00,0,580,2) at m_copym+0x174 m_get sys/kern/uipc_mbuf.c:250 [inline] m_copym(fffffd806f0a6b00,0,580,2) at m_copym+0x174 sys/kern/uipc_mbuf.c:667 tcp_output(ffff800000a7b540) at tcp_output+0x151c sys/netinet/tcp_output.c:673 tcp_usrreq(fffffd806e97ec08,9,fffffd806f0a6b00,0,0,ffff800020ac0ee0) at tcp_usrreq+0xa45 sosend(fffffd806e97ec08,0,ffff800020aeba58,0,0,80) at sosend+0x645 sys/kern/uipc_socket.c:524 dofilewritev(ffff800020ac0ee0,4,ffff800020aeba58,0,ffff800020aebb40) at dofilewritev+0x1b7 sys/kern/sys_generic.c:364 sys_write(ffff800020ac0ee0,ffff800020aebaf0,ffff800020aebb40) at sys_write+0x83 sys/kern/sys_generic.c:284 syscall(ffff800020aebbc0) at syscall+0x4a4 mi_syscall sys/sys/syscall_mi.h:92 [inline] syscall(ffff800020aebbc0) at syscall+0x4a4 sys/arch/amd64/amd64/trap.c:555 Xsyscall(6,4,14aa4fa1616b,4,4,14ad13357b00) at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffc7920, count: 3 https://www.openbsd.org/ddb.html describes the minimum info required in bug reports. Insufficient info makes it difficult to find and fix bugs. ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic pool_cache_item_magic_check: mbufpl cpu free list modified: item addr 0xfffffd806d796d00+16 0x0!=0x614a2d3523e14806 ddb{1}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic() at panic+0x15c sys/kern/subr_prf.c:207 pool_cache_get(ffffffff82647520) at pool_cache_get+0x323 pool_cache_item_magic_check sys/kern/subr_pool.c:1789 [inline] pool_cache_get(ffffffff82647520) at pool_cache_get+0x323 sys/kern/subr_pool.c:1892 pool_get() at pool_get+0x91 sys/kern/subr_pool.c:572 m_copym(fffffd806f0a6b00,0,580,2) at m_copym+0x174 m_get sys/kern/uipc_mbuf.c:250 [inline] m_copym(fffffd806f0a6b00,0,580,2) at m_copym+0x174 sys/kern/uipc_mbuf.c:667 tcp_output(ffff800000a7b540) at tcp_output+0x151c sys/netinet/tcp_output.c:673 tcp_usrreq(fffffd806e97ec08,9,fffffd806f0a6b00,0,0,ffff800020ac0ee0) at tcp_usrreq+0xa45 sosend(fffffd806e97ec08,0,ffff800020aeba58,0,0,80) at sosend+0x645 sys/kern/uipc_socket.c:524 dofilewritev(ffff800020ac0ee0,4,ffff800020aeba58,0,ffff800020aebb40) at dofilewritev+0x1b7 sys/kern/sys_generic.c:364 sys_write(ffff800020ac0ee0,ffff800020aebaf0,ffff800020aebb40) at sys_write+0x83 sys/kern/sys_generic.c:284 syscall(ffff800020aebbc0) at syscall+0x4a4 mi_syscall sys/sys/syscall_mi.h:92 [inline] syscall(ffff800020aebbc0) at syscall+0x4a4 sys/arch/amd64/amd64/trap.c:555 Xsyscall(6,4,14aa4fa1616b,4,4,14ad13357b00) at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffc7920, count: -12 ddb{1}> show registers rdi 0 rsi 0x1 rbp 0xffff800020aeb400 rbx 0xffff800020aeb4b0 rdx 0xffff800020ac0ee0 rcx 0 rax 0 r8 0xffffffff81cd62ff kprintf+0x16f r9 0x1 r10 0x25 r11 0x89083db9b1e7bbfc r12 0x3000000008 r13 0xffff800020aeb410 r14 0x100 r15 0x1 rip 0xffffffff81823988 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff800020aeb3f0 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb{1}> show proc PROC (sshd) pid=200644 stat=onproc flags process=12 proc=0 pri=50, usrpri=50, nice=20 forw=0xffffffffffffffff, list=0xffff800020ac18c0,0xffff800020ac0788 process=0xffff800020a8a710 user=0xffff800020ae6000, vmspace=0xfffffd806e7dc008 estcpu=0, cpticks=1, pctcpu=0.1 user=0, sys=1, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 7252 133471 79321 0 2 0x2 syz-executor.0 2864 383778 0 0 3 0x14200 acct acct 77544 239749 1 0 3 0x100083 ttyin getty 84004 519587 0 0 3 0x14200 bored sosplice 79321 414618 23786 0 3 0x82 nanosleep syz-fuzzer 79321 374881 23786 0 3 0x4000082 nanosleep syz-fuzzer 79321 82572 23786 0 3 0x4000082 thrsleep syz-fuzzer 79321 337130 23786 0 7 0x4000002 syz-fuzzer 79321 465771 23786 0 3 0x4000082 thrsleep syz-fuzzer 79321 429277 23786 0 3 0x4000082 kqread syz-fuzzer 79321 251756 23786 0 3 0x4000082 thrsleep syz-fuzzer 79321 291039 23786 0 3 0x4000082 thrsleep syz-fuzzer 79321 376300 23786 0 3 0x4000082 thrsleep syz-fuzzer 79321 142310 23786 0 3 0x4000082 thrsleep syz-fuzzer 23786 86147 34891 0 3 0x10008a pause ksh *34891 200644 15171 0 7 0x12 sshd 15171 8162 1 0 3 0x80 select sshd 7604 412952 9315 74 3 0x100092 bpf pflogd 9315 309771 1 0 3 0x80 netio pflogd 9146 459983 90536 73 3 0x100090 kqread syslogd 90536 498838 1 0 3 0x100082 netio syslogd 83793 500976 1 77 3 0x100090 poll dhclient 97283 356595 1 0 3 0x80 poll dhclient 49375 128020 0 0 3 0x14200 pgzero zerothread 76051 405066 0 0 3 0x14200 aiodoned aiodoned 51962 506785 0 0 3 0x14200 syncer update 55432 53316 0 0 3 0x14200 cleaner cleaner 19714 336937 0 0 3 0x14200 reaper reaper 5595 293854 0 0 3 0x14200 pgdaemon pagedaemon 90045 492544 0 0 3 0x14200 bored crynlk 74266 338276 0 0 3 0x14200 bored crypto 81755 470417 0 0 3 0x40014200 acpi0 acpi0 24758 24285 0 0 3 0x40014200 idle1 56829 515337 0 0 3 0x14200 bored softnet 95990 438084 0 0 3 0x14200 bored systqmp 22360 196892 0 0 3 0x14200 bored systq 33776 141950 0 0 3 0x40014200 bored softclock 74169 72730 0 0 3 0x40014200 idle0 95098 11134 0 0 3 0x14200 bored smr 1 311565 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{1}> show all locks Process 7252 (syz-executor.0) thread 0xffff800020ab0290 (133471) exclusive rrwlock inode r = 0 (0xfffffd806e1ee6f8) #0 witness_lock+0x52e sys/kern/subr_witness.c:1163 #1 rw_enter+0x447 sys/kern/kern_rwlock.c:306 #2 rrw_enter+0x4f sys/kern/kern_rwlock.c:435 #3 ufs_ihashins+0x45 sys/ufs/ufs/ufs_ihash.c:140 #4 ffs_vget+0x13e sys/ufs/ffs/ffs_vfsops.c:1352 #5 ffs_inode_alloc+0x1cf sys/ufs/ffs/ffs_alloc.c:392 #6 ufs_mkdir+0xf4 sys/ufs/ufs/ufs_vnops.c:1164 #7 VOP_MKDIR+0xc6 sys/kern/vfs_vops.c:450 #8 domkdirat+0x121 sys/kern/vfs_syscalls.c:2983 #9 syscall+0x4a4 mi_syscall sys/sys/syscall_mi.h:92 [inline] #9 syscall+0x4a4 sys/arch/amd64/amd64/trap.c:555 #10 Xsyscall+0x128 exclusive rrwlock inode r = 0 (0xfffffd8062e1ef80) #0 witness_lock+0x52e sys/kern/subr_witness.c:1163 #1 rw_enter+0x447 sys/kern/kern_rwlock.c:306 #2 rrw_enter+0x4f sys/kern/kern_rwlock.c:435 #3 VOP_LOCK+0xf0 sys/kern/vfs_vops.c:615 #4 vn_lock+0x81 sys/kern/vfs_vnops.c:562 #5 vfs_lookup+0xe6 sys/kern/vfs_lookup.c:419 #6 namei+0x63c sys/kern/vfs_lookup.c:249 #7 domkdirat+0x75 sys/kern/vfs_syscalls.c:2968 #8 syscall+0x4a4 mi_syscall sys/sys/syscall_mi.h:92 [inline] #8 syscall+0x4a4 sys/arch/amd64/amd64/trap.c:555 #9 Xsyscall+0x128 Process 79321 (syz-fuzzer) thread 0xffff800020ac1b38 (337130) shared rwlock vmmaplk r = 0 (0xfffffd806e7dcba0) #0 witness_lock+0x52e sys/kern/subr_witness.c:1163 #1 uvm_map_inentry_fix+0xa3 vm_map_lock_read_ln sys/uvm/uvm_map.c:5361 [inline] #1 uvm_map_inentry_fix+0xa3 sys/uvm/uvm_map.c:1840 #2 uvm_map_inentry+0xd2 sys/uvm/uvm_map.c:1872 #3 syscall+0x397 mi_syscall sys/sys/syscall_mi.h:71 [inline] #3 syscall+0x397 sys/arch/amd64/amd64/trap.c:555 #4 Xsyscall+0x128 exclusive kernel_lock &kernel_lock r = 0 (0xffffffff82666748) #0 witness_lock+0x52e sys/kern/subr_witness.c:1163 #1 uvm_map_inentry+0xbe sys/uvm/uvm_map.c:1872 #2 syscall+0x397 mi_syscall sys/sys/syscall_mi.h:71 [inline] #2 syscall+0x397 sys/arch/amd64/amd64/trap.c:555 #3 Xsyscall+0x128 Process 34891 (sshd) thread 0xffff800020ac0ee0 (200644) exclusive rwlock netlock r = 0 (0xffffffff824d6738) #0 witness_lock+0x52e sys/kern/subr_witness.c:1163 #1 solock+0x5a sys/kern/uipc_socket2.c:282 #2 sosend+0x51b sys/kern/uipc_socket.c:512 #3 dofilewritev+0x1b7 sys/kern/sys_generic.c:364 #4 sys_write+0x83 sys/kern/sys_generic.c:284 #5 syscall+0x4a4 mi_syscall sys/sys/syscall_mi.h:92 [inline] #5 syscall+0x4a4 sys/arch/amd64/amd64/trap.c:555 #6 Xsyscall+0x128 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim Kern Lim devbuf 9592 6435K 8210K 78643K 17103 0 0 pcb 13 10K 12K 78643K 391 0 0 rtable 103 12K 13K 78643K 2193 0 0 ifaddr 85 17K 19K 78643K 719 0 0 counters 39 33K 33K 78643K 39 0 0 ioctlops 0 0K 4K 78643K 1538 0 0 iov 0 0K 32K 78643K 484 0 0 mount 1 1K 1K 78643K 1 0 0 vnodes 1220 77K 78K 78643K 3724 0 0 UFS quota 1 32K 32K 78643K 1 0 0 UFS mount 5 36K 36K 78643K 5 0 0 shm 2 1K 9K 78643K 34 0 0 VM map 2 1K 1K 78643K 6 0 0 sem 12 0K 1K 78643K 614 0 0 dirhash 12 2K 2K 78643K 12 0 0 ACPI 1808 196K 290K 78643K 12765 0 0 file desc 3 5K 25K 78643K 2595 0 0 sigio 0 0K 0K 78643K 33 0 0 proc 62 63K 83K 78643K 1155 0 0 subproc 16 1K 2K 78643K 238 0 0 NFS srvsock 1 0K 0K 78643K 1 0 0 NFS daemon 1 16K 16K 78643K 1 0 0 ip_moptions 0 0K 0K 78643K 137 0 0 in_multi 24 1K 2K 78643K 263 0 0 ether_multi 1 0K 0K 78643K 11 0 0 mrt 0 0K 0K 78643K 21 0 0 ISOFS mount 1 32K 32K 78643K 1 0 0 MSDOSFS mount 1 16K 16K 78643K 1 0 0 ttys 90 397K 397K 78643K 90 0 0 exec 0 0K 1K 78643K 558 0 0 pfkey data 0 0K 4K 78643K 650 0 0 pagedep 1 8K 8K 78643K 1 0 0 inodedep 1 32K 32K 78643K 1 0 0 newblk 1 0K 0K 78643K 1 0 0 VM swap 7 26K 26K 78643K 7 0 0 UVM amap 97 13K 31K 78643K 9520 0 0 UVM aobj 130 4K 4K 78643K 140 0 0 memdesc 1 4K 4K 78643K 1 0 0 crypto data 1 1K 1K 78643K 1 0 0 ip6_options 0 0K 1K 78643K 419 0 0 NDP 20 0K 0K 78643K 140 0 0 temp 213 3560K 4201K 78643K 44113 0 0 kqueue 0 0K 0K 78643K 15 0 0 SYN cache 2 16K 16K 78643K 2 0 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle arp 64 59 0 55 1 0 1 1 0 8 0 plcache 128 20 0 0 1 0 1 1 0 8 0 rtpcb 80 183 0 181 1 0 1 1 0 8 0 rtentry 112 600 0 563 2 0 2 2 0 8 0 unpcb 120 908 0 898 1 0 1 1 0 8 0 syncache 264 14 0 14 6 6 0 1 0 8 0 tcpqe 32 5348 0 5348 1 1 0 1 0 8 0 tcpcb 544 1245 0 1241 12 4 8 12 0 8 7 inpcb 280 3499 0 3492 17 9 8 9 0 8 7 rttmr 72 9 0 8 6 5 1 1 0 8 0 ip6q 72 1 0 1 1 1 0 1 0 8 0 nd6 48 35 0 33 2 1 1 1 0 8 0 pkpcb 40 743 0 743 1 0 1 1 0 8 1 swfcl 56 2 0 0 1 0 1 1 0 8 0 ppxss 1128 54 0 54 8 7 1 1 0 8 1 pffrag 232 23 0 23 4 4 0 1 0 482 0 pffrnode 88 23 0 23 4 4 0 1 0 8 0 pffrent 40 57 0 57 4 4 0 1 0 8 0 pfosfp 40 846 0 423 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfstitem 24 138 0 103 1 0 1 1 0 8 0 pfstkey 112 138 0 103 2 0 2 2 0 8 0 pfstate 328 138 0 103 5 0 5 5 0 8 0 pfrule 1360 21 0 16 2 1 1 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 1216 0 1016 28 12 16 16 0 8 0 art_table 32 1217 0 1016 2 0 2 2 0 8 0 art_node 16 275 0 243 1 0 1 1 0 8 0 sysvmsgpl 40 35 0 33 1 0 1 1 0 8 0 semupl 112 1 0 1 1 1 0 1 0 8 0 semapl 112 610 0 600 1 0 1 1 0 8 0 shmpl 112 138 0 10 4 0 4 4 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino1pl 128 5429 0 4019 46 0 46 46 0 8 0 ffsino 272 5429 0 4019 95 0 95 95 0 8 0 nchpl 144 9307 0 8844 61 41 20 61 0 8 0 uvmvnodes 72 5926 0 0 108 0 108 108 0 8 0 vnodes 208 5926 0 0 312 0 312 312 0 8 0 namei 1024 35705 0 35704 3 2 1 1 0 8 0 percpumem 16 30 0 0 1 0 1 1 0 8 0 vmpool 552 4 0 4 2 2 0 1 0 8 0 scsiplug 64 3 0 3 2 2 0 1 0 8 0 scxspl 192 28267 0 28267 26 24 2 7 0 8 2 plimitpl 152 201 0 194 1 0 1 1 0 8 0 sigapl 432 2760 0 2747 3 1 2 3 0 8 0 futexpl 56 52504 0 52504 1 0 1 1 0 8 1 knotepl 112 548 0 535 2 1 1 2 0 8 0 kqueuepl 104 730 0 728 4 1 3 4 0 8 2 pipepl 112 1286 0 1273 6 5 1 2 0 8 0 fdescpl 488 2761 0 2747 3 0 3 3 0 8 0 filepl 152 24451 0 24374 23 12 11 14 0 8 7 lockfpl 104 827 0 826 1 0 1 1 0 8 0 lockfspl 48 294 0 293 1 0 1 1 0 8 0 sessionpl 112 31 0 21 1 0 1 1 0 8 0 pgrppl 48 48 0 38 1 0 1 1 0 8 0 ucredpl 96 7745 0 7736 1 0 1 1 0 8 0 zombiepl 144 2747 0 2747 3 2 1 1 0 8 1 processpl 896 2778 0 2747 4 0 4 4 0 8 0 procpl 632 8122 0 8082 5 0 5 5 0 8 1 srpgc 64 20 0 20 8 7 1 1 0 8 1 sosppl 128 22 0 22 8 7 1 1 0 8 1 sockpl 384 5421 0 5402 23 14 9 14 0 8 7 mcl64k 65536 18 0 0 3 0 3 3 0 8 0 mcl16k 16384 15 0 0 2 0 2 2 0 8 0 mcl12k 12288 25 0 0 2 0 2 2 0 8 0 mcl9k 9216 17 0 0 2 0 2 2 0 8 0 mcl8k 8192 17 0 0 3 0 3 3 0 8 0 mcl4k 4096 13 0 0 2 0 2 2 0 8 0 mcl2k2 2112 7 0 0 1 0 1 1 0 8 0 mcl2k 2048 134 0 0 15 0 15 15 0 8 0 mtagpl 80 34 0 0 1 0 1 1 0 8 0 mbufpl 256 284 0 0 13 0 13 13 0 8 0 bufpl 256 14198 0 7150 441 0 441 441 0 8 0 anonpl 16 313401 0 300801 122 51 71 83 0 124 0 amapchunkpl 152 18350 0 18243 32 26 6 14 0 158 0 amappl16 192 14384 0 13466 105 58 47 59 0 8 0 amappl15 184 435 0 435 2 2 0 1 0 8 0 amappl14 176 1020 0 1015 2 1 1 1 0 8 0 amappl13 168 152 0 152 2 2 0 1 0 8 0 amappl12 160 201 0 201 4 3 1 1 0 8 1 amappl11 152 331 0 316 1 0 1 1 0 8 0 amappl10 144 182 0 178 1 0 1 1 0 8 0 amappl9 136 775 0 770 1 0 1 1 0 8 0 amappl8 128 320 0 292 2 0 2 2 0 8 0 amappl7 120 264 0 256 1 0 1 1 0 8 0 amappl6 112 337 0 324 1 0 1 1 0 8 0 amappl5 104 508 0 495 1 0 1 1 0 8 0 amappl4 96 2233 0 2201 2 1 1 2 0 8 0 amappl3 88 2246 0 2240 1 0 1 1 0 8 0 amappl2 80 20878 0 20816 4 2 2 3 0 8 0 amappl1 72 69208 0 68793 25 15 10 20 0 8 0 amappl 80 8596 0 8562 2 0 2 2 0 84 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 17 0 17 1 1 0 1 0 8 0 aobjpl 64 139 0 10 3 0 3 3 0 8 0 uaddrrnd 24 2765 0 2747 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 2765 0 2747 1 0 1 1 0 8 0 vmmpekpl 168 24034 0 23995 3 0 3 3 0 8 0 vmmpepl 168 346374 0 344496 230 105 125 130 0 357 32 vmsppl 368 2760 0 2747 2 0 2 2 0 8 0 pdppl 4096 5537 0 5502 7 1 6 6 0 8 0 pvpl 32 859429 0 843760 296 113 183 202 0 265 20 pmappl 232 2764 0 2751 4 3 1 2 0 8 0 extentpl 40 41 0 26 1 0 1 1 0 8 0 phpool 112 634 0 22 18 0 18 18 0 8 0