panic: pool_cache_item_magic_check: mbufpl cpu free list modified: item addr 0xfffffd80753a0b00+24 0x73e9a1269bf30303!=0x8c165ca6f1d25403 Stopped at db_enter+0x18: addq $0x8,%rsp TID PID UID PRFLAGS PFLAGS CPU COMMAND *383960 95843 0 0x100002 0 0K arp db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic() at panic+0x15c sys/kern/subr_prf.c:207 pool_cache_get(ffffffff82639e38) at pool_cache_get+0x323 pool_cache_item_magic_check sys/kern/subr_pool.c:1789 [inline] pool_cache_get(ffffffff82639e38) at pool_cache_get+0x323 sys/kern/subr_pool.c:1892 pool_get() at pool_get+0x91 sys/kern/subr_pool.c:572 m_get(2,3) at m_get+0x4c sys/kern/uipc_mbuf.c:250 sbappendaddr(fffffd8062e40928,fffffd8062e409b0,ffffffff822d6f50,fffffd8066ac2f00,0) at sbappendaddr+0x140 sys/kern/uipc_socket2.c:802 rtm_sendup(fffffd8062e40928,fffffd8066abfc00,1) at rtm_sendup+0xe7 sys/net/rtsock.c:594 route_input(fffffd8066abfc00,fffffd8062e40928,2) at route_input+0x1cc sys/net/rtsock.c:560 route_output(fffffd8066abfc00,fffffd8062e40928,0,0) at route_output+0x80c sys/net/rtsock.c:857 route_usrreq(fffffd8062e40928,9,fffffd8066abfc00,0,0,ffff800020acf3e0) at route_usrreq+0x363 sys/net/rtsock.c:275 sosend(fffffd8062e40928,0,ffff800023b8d7d8,0,0,0) at sosend+0x645 sys/kern/uipc_socket.c:524 dofilewritev(ffff800020acf3e0,3,ffff800023b8d7d8,0,ffff800023b8d8c0) at dofilewritev+0x1b7 sys/kern/sys_generic.c:364 sys_write(ffff800020acf3e0,ffff800023b8d870,ffff800023b8d8c0) at sys_write+0x83 sys/kern/sys_generic.c:284 syscall(ffff800023b8d940) at syscall+0x4a4 mi_syscall sys/sys/syscall_mi.h:92 [inline] syscall(ffff800023b8d940) at syscall+0x4a4 sys/arch/amd64/amd64/trap.c:555 end trace frame: 0xffff800023b8d9c0, count: 0 https://www.openbsd.org/ddb.html describes the minimum info required in bug reports. Insufficient info makes it difficult to find and fix bugs. ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic pool_cache_item_magic_check: mbufpl cpu free list modified: item addr 0xfffffd80753a0b00+24 0x73e9a1269bf30303!=0x8c165ca6f1d25403 ddb{0}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic() at panic+0x15c sys/kern/subr_prf.c:207 pool_cache_get(ffffffff82639e38) at pool_cache_get+0x323 pool_cache_item_magic_check sys/kern/subr_pool.c:1789 [inline] pool_cache_get(ffffffff82639e38) at pool_cache_get+0x323 sys/kern/subr_pool.c:1892 pool_get() at pool_get+0x91 sys/kern/subr_pool.c:572 m_get(2,3) at m_get+0x4c sys/kern/uipc_mbuf.c:250 sbappendaddr(fffffd8062e40928,fffffd8062e409b0,ffffffff822d6f50,fffffd8066ac2f00,0) at sbappendaddr+0x140 sys/kern/uipc_socket2.c:802 rtm_sendup(fffffd8062e40928,fffffd8066abfc00,1) at rtm_sendup+0xe7 sys/net/rtsock.c:594 route_input(fffffd8066abfc00,fffffd8062e40928,2) at route_input+0x1cc sys/net/rtsock.c:560 route_output(fffffd8066abfc00,fffffd8062e40928,0,0) at route_output+0x80c sys/net/rtsock.c:857 route_usrreq(fffffd8062e40928,9,fffffd8066abfc00,0,0,ffff800020acf3e0) at route_usrreq+0x363 sys/net/rtsock.c:275 sosend(fffffd8062e40928,0,ffff800023b8d7d8,0,0,0) at sosend+0x645 sys/kern/uipc_socket.c:524 dofilewritev(ffff800020acf3e0,3,ffff800023b8d7d8,0,ffff800023b8d8c0) at dofilewritev+0x1b7 sys/kern/sys_generic.c:364 sys_write(ffff800020acf3e0,ffff800023b8d870,ffff800023b8d8c0) at sys_write+0x83 sys/kern/sys_generic.c:284 syscall(ffff800023b8d940) at syscall+0x4a4 mi_syscall sys/sys/syscall_mi.h:92 [inline] syscall(ffff800023b8d940) at syscall+0x4a4 sys/arch/amd64/amd64/trap.c:555 Xsyscall(6,4,7f7ffffd37bf,4,7f7ffffd3708,bcecdfffe00) at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffd35d0, count: -15 ddb{0}> show registers rdi 0 rsi 0x1 rbp 0xffff800023b8d0e0 rbx 0xffff800023b8d190 rdx 0xffff800020acf3e0 rcx 0 rax 0 r8 0xffffffff81bed2cf kprintf+0x16f r9 0x1 r10 0x25 r11 0x93062a8c37b93005 r12 0x3000000008 r13 0xffff800023b8d0f0 r14 0x100 r15 0x1 rip 0xffffffff81463878 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff800023b8d0d0 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb{0}> show proc PROC (arp) pid=383960 stat=onproc flags process=100002 proc=0 pri=80, usrpri=80, nice=20 forw=0xffffffffffffffff, list=0xffff800020acf658,0xffffffff826576c8 process=0xffff800020add180 user=0xffff800023b88000, vmspace=0xfffffd807f00ba10 estcpu=36, cpticks=2, pctcpu=0.0 user=0, sys=2, intr=0 ddb{0}> ps PID TID PPID UID S FLAGS WAIT COMMAND *95843 383960 17423 0 7 0x100002 arp 17423 320754 2693 0 3 0x10008a pause sh 2693 330155 64838 0 3 0x82 wait syz-executor.0 40196 452403 0 0 3 0x14200 acct acct 96397 186041 64838 0 3 0x82 piperd syz-executor.1 88542 464817 1 0 3 0x100083 ttyopn getty 71534 413579 0 0 3 0x14200 bored sosplice 64838 503450 87534 0 3 0x82 thrsleep syz-fuzzer 64838 33933 87534 0 3 0x4000082 thrsleep syz-fuzzer 64838 136652 87534 0 3 0x4000082 thrsleep syz-fuzzer 64838 513745 87534 0 3 0x4000082 thrsleep syz-fuzzer 64838 323741 87534 0 3 0x4000082 thrsleep syz-fuzzer 64838 446045 87534 0 3 0x4000082 thrsleep syz-fuzzer 64838 522817 87534 0 3 0x4000082 thrsleep syz-fuzzer 64838 466021 87534 0 3 0x4000082 thrsleep syz-fuzzer 64838 387188 87534 0 3 0x4000082 kqread syz-fuzzer 64838 87822 87534 0 3 0x4000082 thrsleep syz-fuzzer 87534 29987 75435 0 3 0x10008a pause ksh 75435 137964 91688 0 3 0x92 select sshd 91688 150893 1 0 3 0x80 select sshd 4443 429093 20926 74 3 0x100092 bpf pflogd 20926 160619 1 0 3 0x80 netio pflogd 25766 438363 34049 73 3 0x100090 kqread syslogd 34049 152398 1 0 3 0x100082 netio syslogd 75119 134431 1 77 3 0x100090 poll dhclient 29997 417676 1 0 3 0x80 poll dhclient 12419 415582 0 0 3 0x14200 pgzero zerothread 75627 217573 0 0 3 0x14200 aiodoned aiodoned 27432 368467 0 0 3 0x14200 syncer update 75686 220914 0 0 3 0x14200 cleaner cleaner 55480 127271 0 0 3 0x14200 reaper reaper 91394 61033 0 0 3 0x14200 pgdaemon pagedaemon 48255 386243 0 0 3 0x14200 bored crynlk 17501 186267 0 0 3 0x14200 bored crypto 98993 487338 0 0 3 0x40014200 acpi0 acpi0 20822 292977 0 0 7 0x40014200 idle1 97756 478226 0 0 3 0x14200 bored softnet 48710 436785 0 0 3 0x14200 bored systqmp 33967 374551 0 0 3 0x14200 bored systq 16442 365028 0 0 3 0x40014200 bored softclock 75650 214739 0 0 3 0x40014200 idle0 42948 122536 0 0 3 0x14200 bored smr 1 77653 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{0}> show all locks Process 95843 (arp) thread 0xffff800020acf3e0 (383960) exclusive kernel_lock &kernel_lock r = 1 (0xffffffff8266a578) #0 witness_lock+0x52e sys/kern/subr_witness.c:1163 #1 solock+0x66 sys/kern/uipc_socket2.c:292 #2 sosend+0x51b sys/kern/uipc_socket.c:512 #3 dofilewritev+0x1b7 sys/kern/sys_generic.c:364 #4 sys_write+0x83 sys/kern/sys_generic.c:284 #5 syscall+0x4a4 mi_syscall sys/sys/syscall_mi.h:92 [inline] #5 syscall+0x4a4 sys/arch/amd64/amd64/trap.c:555 #6 Xsyscall+0x128 ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim Kern Lim devbuf 9626 7104K 7946K 78643K 20648 0 0 pcb 13 10K 12K 78643K 925 0 0 rtable 97 13K 14K 78643K 3068 0 0 ifaddr 95 20K 22K 78643K 979 0 0 counters 39 33K 33K 78643K 39 0 0 ioctlops 0 0K 4K 78643K 1650 0 0 iov 0 0K 32K 78643K 899 0 0 mount 1 1K 1K 78643K 1 0 0 vnodes 1213 76K 77K 78643K 5634 0 0 UFS quota 1 32K 32K 78643K 1 0 0 UFS mount 5 36K 36K 78643K 5 0 0 shm 2 1K 5K 78643K 44 0 0 VM map 2 1K 1K 78643K 33 0 0 sem 12 1K 1K 78643K 1001 0 0 dirhash 12 2K 2K 78643K 12 0 0 ACPI 1808 196K 290K 78643K 12765 0 0 file desc 6 17K 25K 78643K 4939 0 0 sigio 0 0K 0K 78643K 73 0 0 proc 62 63K 95K 78643K 1976 0 0 subproc 32 2K 2K 78643K 514 0 0 NFS srvsock 1 0K 0K 78643K 1 0 0 NFS daemon 1 16K 16K 78643K 1 0 0 ip_moptions 0 0K 0K 78643K 286 0 0 in_multi 13 0K 2K 78643K 461 0 0 ether_multi 1 0K 0K 78643K 20 0 0 mrt 0 0K 0K 78643K 18 0 0 ISOFS mount 1 32K 32K 78643K 1 0 0 MSDOSFS mount 1 16K 16K 78643K 1 0 0 ttys 90 397K 397K 78643K 90 0 0 exec 0 0K 1K 78643K 991 0 0 pfkey data 0 0K 4K 78643K 4 0 0 pagedep 1 8K 8K 78643K 1 0 0 inodedep 1 32K 32K 78643K 1 0 0 newblk 1 0K 0K 78643K 1 0 0 VM swap 7 26K 26K 78643K 7 0 0 UVM amap 119 22K 31K 78643K 18007 0 0 UVM aobj 130 4K 4K 78643K 137 0 0 memdesc 1 4K 4K 78643K 1 0 0 crypto data 1 1K 1K 78643K 1 0 0 ip6_options 0 0K 0K 78643K 995 0 0 NDP 25 0K 1K 78643K 300 0 0 temp 264 3561K 4201K 78643K 121445 0 0 kqueue 0 0K 0K 78643K 16 0 0 SYN cache 2 16K 16K 78643K 2 0 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle arp 64 127 0 123 1 0 1 1 0 8 0 plcache 128 20 0 0 1 0 1 1 0 8 0 rtpcb 80 324 0 321 1 0 1 1 0 8 0 rtentry 112 480 0 451 2 0 2 2 0 8 0 unpcb 120 2410 0 2397 1 0 1 1 0 8 0 syncache 264 23 0 23 10 10 0 1 0 8 0 tcpqe 32 5802 0 5802 2 2 0 2 0 8 0 tcpcb 544 2291 0 2287 16 7 9 14 0 8 8 inpcb 280 10175 0 10166 20 11 9 9 0 8 8 rttmr 72 7 0 6 1 0 1 1 0 8 0 ip6q 72 1 0 1 1 1 0 1 0 8 0 ip6af 40 2 0 2 1 1 0 1 0 8 0 nd6 48 64 0 64 4 3 1 1 0 8 1 pkpcb 40 32 0 32 13 13 0 1 0 8 0 ppxss 1128 141 0 141 13 12 1 1 0 8 1 pffrag 232 41 0 41 14 14 0 1 0 482 0 pffrnode 88 41 0 41 14 14 0 1 0 8 0 pffrent 40 105 0 105 14 14 0 1 0 8 0 pfosfp 40 846 0 423 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfstitem 24 565 0 404 2 0 2 2 0 8 0 pfstkey 112 565 0 404 6 0 6 6 0 8 0 pfstate 328 565 0 404 16 1 15 15 0 8 0 pfrule 1360 21 0 16 2 1 1 2 0 8 0 art_heap8 4096 9 0 8 8 7 1 3 0 8 0 art_heap4 256 1958 0 1803 27 13 14 17 0 8 0 art_table 32 1967 0 1811 6 4 2 3 0 8 0 art_node 16 471 0 444 1 0 1 1 0 8 0 sysvmsgpl 40 26 0 23 1 0 1 1 0 8 0 semupl 112 3 0 3 1 1 0 1 0 8 0 semapl 112 997 0 987 1 0 1 1 0 8 0 shmpl 112 135 0 7 4 0 4 4 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino1pl 128 9353 0 7931 47 0 47 47 0 8 0 ffsino 272 9353 0 7931 96 0 96 96 0 8 0 nchpl 144 17808 0 17334 61 40 21 61 0 8 0 uvmvnodes 72 6162 0 0 113 0 113 113 0 8 0 vnodes 208 6162 0 0 325 0 325 325 0 8 0 namei 1024 57857 0 57857 3 2 1 1 0 8 1 percpumem 16 30 0 0 1 0 1 1 0 8 0 vmpool 552 31 0 31 12 12 0 1 0 8 0 scsiplug 64 15 0 15 6 6 0 1 0 8 0 scxspl 192 52804 0 52804 39 36 3 7 0 8 3 plimitpl 152 430 0 422 1 0 1 1 0 8 0 sigapl 432 5058 0 5042 4 2 2 3 0 8 0 futexpl 56 100795 0 100795 1 0 1 1 0 8 1 knotepl 112 1473 0 1452 5 4 1 3 0 8 0 kqueuepl 104 1585 0 1583 7 3 4 4 0 8 3 pipepl 112 2930 0 2911 9 8 1 2 0 8 0 fdescpl 488 5059 0 5042 3 0 3 3 0 8 0 filepl 152 43219 0 43118 40 28 12 14 0 8 8 lockfpl 104 1790 0 1789 1 0 1 1 0 8 0 lockfspl 48 566 0 565 1 0 1 1 0 8 0 sessionpl 112 49 0 38 1 0 1 1 0 8 0 pgrppl 48 100 0 89 1 0 1 1 0 8 0 ucredpl 96 5203 0 5193 1 0 1 1 0 8 0 zombiepl 144 5045 0 5045 3 2 1 1 0 8 1 processpl 896 5079 0 5045 5 1 4 5 0 8 0 procpl 632 15840 0 15797 5 0 5 5 0 8 0 srpgc 64 52 0 52 17 16 1 1 0 8 1 sosppl 128 56 0 56 13 12 1 1 0 8 1 sockpl 384 12991 0 12968 32 21 11 15 0 8 8 mcl64k 65536 262 0 0 33 19 14 33 0 8 1 mcl16k 16384 11 0 0 2 0 2 2 0 8 0 mcl12k 12288 33 0 0 2 0 2 2 0 8 0 mcl9k 9216 19 0 0 2 0 2 2 0 8 0 mcl8k 8192 17 0 0 3 0 3 3 0 8 0 mcl4k 4096 17 0 0 3 0 3 3 0 8 0 mcl2k2 2112 10 0 0 1 0 1 1 0 8 0 mcl2k 2048 256 0 0 28 2 26 28 0 8 0 mtagpl 80 62 0 0 1 0 1 1 0 8 0 mbufpl 256 802 0 0 31 0 31 31 0 8 0 bufpl 256 21495 0 14447 441 0 441 441 0 8 0 anonpl 16 554831 0 541973 173 103 70 85 0 124 0 amapchunkpl 152 35983 0 35841 66 60 6 12 0 158 0 amappl16 192 25408 0 24460 143 85 58 60 0 8 8 amappl15 184 505 0 504 3 2 1 1 0 8 0 amappl14 176 757 0 755 1 0 1 1 0 8 0 amappl13 168 550 0 549 5 4 1 1 0 8 0 amappl12 160 1303 0 1301 1 0 1 1 0 8 0 amappl11 152 1363 0 1348 1 0 1 1 0 8 0 amappl10 144 52 0 48 1 0 1 1 0 8 0 amappl9 136 1212 0 1205 1 0 1 1 0 8 0 amappl8 128 727 0 687 5 3 2 2 0 8 0 amappl7 120 178 0 169 1 0 1 1 0 8 0 amappl6 112 1310 0 1296 1 0 1 1 0 8 0 amappl5 104 1654 0 1639 1 0 1 1 0 8 0 amappl4 96 4425 0 4390 1 0 1 1 0 8 0 amappl3 88 1486 0 1481 1 0 1 1 0 8 0 amappl2 80 38692 0 38625 3 1 2 3 0 8 0 amappl1 72 123229 0 122759 25 15 10 20 0 8 0 amappl 80 16566 0 16527 2 0 2 2 0 84 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 17 0 17 1 1 0 1 0 8 0 aobjpl 64 136 0 7 3 0 3 3 0 8 0 uaddrrnd 24 5090 0 5042 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 5090 0 5042 1 0 1 1 0 8 0 vmmpekpl 168 44655 0 44619 2 0 2 2 0 8 0 vmmpepl 168 638194 0 636124 322 193 129 139 0 357 31 vmsppl 368 5058 0 5042 2 0 2 2 0 8 0 pdppl 4096 10187 0 10146 7 1 6 6 0 8 0 pvpl 32 1471723 0 1455509 364 192 172 201 0 265 5 pmappl 232 5089 0 5073 10 9 1 2 0 8 0 extentpl 40 41 0 26 1 0 1 1 0 8 0 phpool 112 707 0 54 20 0 20 20 0 8 0