kernel: protection fault trap, code=0 Stopped at pool_do_put+0x115: movq 0x8(%rbx),%rbx ddb{1}> show panic the kernel did not panic ddb{1}> trace pool_do_put(ffffffff82c64528,fffffd806f3ac3e8) at pool_do_put+0x115 pool_put(ffffffff82c64528,fffffd806f3ac3e8) at pool_put+0x8f sys/kern/subr_pool.c:799 soclose(fffffd806f3ac3e8,0) at soclose+0x4ba sys/kern/uipc_socket.c:440 soo_close(fffffd806b86dc90,ffff8000212b3aa0) at soo_close+0x44 fdrop(fffffd806b86dc90,ffff8000212b3aa0) at fdrop+0xcb sys/kern/kern_descrip.c:1274 closef(fffffd806b86dc90,ffff8000212b3aa0) at closef+0x120 sys/kern/kern_descrip.c:1258 fdfree(ffff8000212b3aa0) at fdfree+0xf4 sys/kern/kern_descrip.c:1190 exit1(ffff8000212b3aa0,1,0,1) at exit1+0x3e4 sys/kern/kern_exit.c:206 sys_exit(ffff8000212b3aa0,ffff800022d11520,ffff800022d11570) at sys_exit+0x1a sys/kern/kern_exit.c:89 syscall(ffff800022d115f0) at syscall+0x5e2 mi_syscall sys/sys/syscall_mi.h:110 [inline] syscall(ffff800022d115f0) at syscall+0x5e2 sys/arch/amd64/amd64/trap.c:623 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x79cfc46d6fd0, count: -11 ddb{1}> show registers rdi 0 rsi 0xfffffd806f3ac7b8 rbp 0xffff800022d11280 rbx 0x27164cc402aec9d rdx 0 rcx 0xffff8000212b3aa0 rax 0xffff800020d58ff0 r8 0x90 r9 0 r10 0x93f333f72b0084b4 r11 0x8134e3bcb87210fd r12 0xfffffd806f3ac3e8 r13 0xed63f8c9094c9ac2 r14 0xffffffff82c64528 socket_pool r15 0xfffffd806f3acf90 rip 0xffffffff8126b5d5 pool_do_put+0x115 cs 0x8 rflags 0x10296 __ALIGN_SIZE+0xf296 rsp 0xffff800022d111d0 ss 0x10 pool_do_put+0x115: movq 0x8(%rbx),%rbx ddb{1}> show proc PROC (login_passwd) pid=57566 stat=onproc flags process=10003b proc=2000 pri=26, usrpri=43, nice=16 forw=0xffffffffffffffff, list=0xffff8000212b37f8,0xffffffff82c58ae0 process=0xffff8000212925c8 user=0xffff800022d0c000, vmspace=0xfffffd8069b4ecc8 estcpu=1, cpticks=3, pctcpu=0.3 user=2, sys=1, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 26475 406765 1 0 3 0x1 kernel: protection fault trap, code=0 Faulted in DDB; continuing... ddb{1}> show all locks CPU 1: exclusive mutex sockpl r = 0 (0xffffffff82c64538) #0 witness_lock+0x447 #1 mtx_enter_try+0x104 #2 mtx_enter+0x4f sys/kern/kern_lock.c:266 #3 pool_put+0x84 sys/kern/subr_pool.c:797 #4 soclose+0x4ba sys/kern/uipc_socket.c:440 #5 soo_close+0x44 #6 fdrop+0xcb sys/kern/kern_descrip.c:1274 #7 closef+0x120 sys/kern/kern_descrip.c:1258 #8 fdfree+0xf4 sys/kern/kern_descrip.c:1190 #9 exit1+0x3e4 sys/kern/kern_exit.c:206 #10 sys_exit+0x1a sys/kern/kern_exit.c:89 #11 syscall+0x5e2 mi_syscall sys/sys/syscall_mi.h:110 [inline] #11 syscall+0x5e2 sys/arch/amd64/amd64/trap.c:623 #12 Xsyscall+0x128 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10192 6475K 7259K 78643K 12746 0 pcb 13 10K 12K 78643K 42 0 rtable 238 6K 6K 78643K 362 0 pf 32 9K 10K 78643K 58 0 ifaddr 45 15K 15K 78643K 55 0 ifgroup 55 2K 2K 78643K 76 0 sysctl 2 0K 0K 78643K 2 0 counters 60 35K 36K 78643K 72 0 ioctlops 0 0K 4K 78643K 1486 0 iov 0 0K 16K 78643K 153 0 mount 1 1K 1K 78643K 1 0 log 0 0K 0K 78643K 4 0 vnodes 1412 88K 88K 78643K 1710 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 5K 78643K 5 0 VM map 2 1K 1K 78643K 2 0 sem 12 1K 1K 78643K 20 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1697 195K 286K 78643K 12548 0 file desc 10 33K 81K 78643K 587 0 sigio 0 0K 0K 78643K 97 0 proc 62 79K 115K 78643K 535 0 subproc 104 6K 6K 78643K 104 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 9 0 in_multi 99 7K 7K 78643K 105 0 ether_multi 1 0K 0K 78643K 4 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 61 281K 281K 78643K 61 0 exec 0 0K 1K 78643K 417 0 tdb 3 0K 0K 78643K 3 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 8 62K 64K 78643K 10 0 UVM amap 303 85K 89K 78643K 8136 0 UVM aobj 79 3K 3K 78643K 79 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 0K 78643K 7 0 NDP 12 0K 1K 78643K 34 0 temp 96 5928K 5992K 78643K 5991 0 kqueue 12 18K 20K 78643K 35 0 SYN cache 2 16K 16K 78643K 2 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 22 0 0 1 0 1 1 0 8 0 rtpcb 120 82 0 79 2 1 1 2 0 8 0 rtentry 112 114 0 2 4 0 4 4 0 8 0 unpcb 144 304 0 289 4 3 1 4 0 8 0 syncache 304 7 0 7 2 2 0 1 0 8 0 tcpqe 32 467 0 467 3 3 0 2 0 8 0 tcpcb 808 247 0 243 11 10 1 11 0 8 0 arp 120 19 0 1 1 0 1 1 0 8 0 inpcb 368 485 0 478 10 9 1 10 0 8 0 nd6 136 26 0 0 1 0 1 1 0 8 0 pkpcb 40 4 0 4 1 1 0 1 0 8 0 kcovpl 48 8 0 0 1 0 1 1 0 8 0 ppxss 1256 3 0 3 1 1 0 1 0 8 0 pffrent 40 3 0 3 1 1 0 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfstitem 24 48 0 46 1 0 1 1 0 8 0 pfstkey 128 48 0 46 2 1 1 2 0 8 0 pfstate 376 48 0 46 4 3 1 4 0 8 0 pfrule 1344 21 0 16 2 1 1 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 468 0 1 30 0 30 30 0 8 0 art_table 32 469 0 1 4 0 4 4 0 8 0 art_node 16 113 0 11 1 0 1 1 0 8 0 sysvmsgpl 40 2 0 1 1 0 1 1 0 8 0 semapl 112 18 0 8 1 0 1 1 0 8 0 shmpl 112 76 0 0 3 0 3 3 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 2085 0 632 91 0 91 91 0 8 0 ffsino 272 2085 0 632 97 0 97 97 0 8 0 nchpl 144 2960 0 1266 63 0 63 63 0 8 0 uvmvnodes 80 2562 0 0 53 0 53 53 0 8 0 vnodes 216 2562 0 0 143 0 143 143 0 8 0 namei 1024 10694 0 10691 3 2 1 2 0 8 0 percpumem 16 49 0 6 1 0 1 1 0 8 0 vmpool 696 3 0 3 1 1 0 1 0 8 0 kstatmem 264 36 0 12 2 0 2 2 0 8 0 scxspl 216 9924 0 9924 10 9 1 8 1 8 1 plimitpl 152 54 0 38 1 0 1 1 0 8 0 sigapl 424 912 0 871 7 1 6 6 0 8 0 futexpl 64 4176 0 4176 1 1 0 1 0 8 0 knotepl 120 108 0 0 4 0 4 4 0 8 0 kqueuepl 216 117 0 109 5 4 1 5 0 8 0 pipepl 320 322 0 294 8 5 3 8 0 8 0 fdescpl 496 895 0 871 5 0 5 5 0 8 0 filepl 152 5438 0 5198 15 5 10 15 0 8 0 lockfpl 104 121 0 119 1 0 1 1 0 8 0 lockfspl 48 59 0 57 1 0 1 1 0 8 0 sessionpl 144 24 0 7 1 0 1 1 0 8 0 pgrppl 48 24 0 7 1 0 1 1 0 8 0 ucredpl 104 416 0 403 1 0 1 1 0 8 0 zombiepl 144 872 0 871 3 2 1 1 0 8 0 processpl 1072 912 0 871 4 0 4 4 0 8 0 procpl 680 1782 0 1722 8 1 7 7 0 8 0 sockpl 488 910 0 883 18 14 4 17 0 8 0 sockpl: pool(0xffffffff82c64528:sockpl): free list modified: page 0xfffffd806f3ac000; item ordinal 2; addr 0xfffffd806f3ac019 (p 0xfffffd806f3ac000); offset 0x0=0x122ac84eb4800e84 pool(sockpl): free list modified: page 0xfffffd806f3ac000; item ordinal 2; addr 0xfffffd806f3ac019 (p 0xfffffd806f3ac000); offset 0x0=0xefdeadbe sockpl: pool(0xffffffff82c64528:sockpl): page inconsistency: page 0xfffffd806f3ac000; item ordinal 3; addr 0x27164cc402aec9d mcl64k 65536 4 0 0 1 0 1 1 0 8 0 mcl16k 16384 4 0 0 1 0 1 1 0 8 0 mcl12k 12288 7 0 0 1 0 1 1 0 8 0 mcl9k 9216 2 0 0 1 0 1 1 0 8 0 mcl8k 8192 10 0 0 2 0 2 2 0 8 0 mcl4k 4096 7 0 0 1 0 1 1 0 8 0 mcl2k2 2112 2 0 0 1 0 1 1 0 8 0 mcl2k 2048 299 0 0 35 7 28 35 0 8 0 mtagpl 96 82 0 0 2 0 2 2 0 8 0 mbufpl 256 1036 0 0 64 13 51 64 0 8 1 bufpl 288 4844 0 148 336 0 336 336 0 8 0 anonpl 24 217687 0 206006 84 2 82 82 0 186 0 amapchunkpl 152 25781 0 25007 40 3 37 39 0 158 0 amappl16 200 5509 0 5145 20 0 20 20 0 8 0 amappl15 192 13 0 13 1 1 0 1 0 8 0 amappl14 184 170 0 152 2 1 1 2 0 8 0 amappl13 176 11 0 11 2 2 0 1 0 8 0 amappl12 168 1567 0 1542 3 1 2 2 0 8 0 amappl11 160 63 0 48 1 0 1 1 0 8 0 amappl10 152 36 0 25 1 0 1 1 0 8 0 amappl9 144 176 0 175 1 0 1 1 0 8 0 amappl8 136 196 0 139 2 0 2 2 0 8 0 amappl7 128 75 0 63 2 0 2 2 0 8 0 amappl6 120 255 0 234 2 1 1 2 0 8 0 amappl5 112 184 0 172 1 0 1 1 0 8 0 amappl4 104 592 0 550 3 1 2 3 0 8 0 amappl3 96 5561 0 5496 4 1 3 3 0 8 0 amappl2 88 1166 0 1100 3 1 2 3 0 8 0 amappl1 80 11850 0 11286 23 10 13 23 0 8 0 amappl 88 7566 0 7379 6 0 6 6 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 78 0 0 2 0 2 2 0 8 0 uaddrrnd 24 898 0 874 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 898 0 874 1 0 1 1 0 8 0 vmmpekpl 168 13008 0 12953 3 0 3 3 0 8 0 vmmpepl 168 75597 0 73538 133 32 101 133 0 357 0 vmsppl 464 897 0 874 5 1 4 5 0 8 0 rwobjpl 56 27940 0 23847 61 3 58 58 0 8 0 pdppl 4096 1804 0 1748 122 66 56 78 0 8 0 pvpl 32 504557 0 487763 364 207 157 363 0 265 0 pmappl 248 897 0 874 3 0 3 3 0 8 0 extentpl 40 56 0 38 1 0 1 1 0 8 0 phpool 112 820 0 111 22 0 22 22 0 8 0 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+0x1e: addq $0x8,%rsp ddb{0}> trace x86_ipi_db(ffffffff82bd4ff0) at x86_ipi_db+0x1e sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 __sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc+0x30 kd_curproc sys/dev/kcov.c:590 [inline] __sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc+0x30 sys/dev/kcov.c:158 __mp_lock(ffffffff82cdebd8) at __mp_lock+0x133 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff82cdebd8) at __mp_lock+0x133 sys/kern/kern_lock.c:147 softintr_dispatch(0) at softintr_dispatch+0x52 sys/arch/amd64/amd64/softintr.c:88 Xsoftclock() at Xsoftclock+0x27 __sanitizer_cov_trace_cmp4(0,9) at __sanitizer_cov_trace_cmp4+0x37 sys/dev/kcov.c:212 splraise(9) at splraise+0x58 sys/arch/amd64/amd64/intr.c:702 mtx_enter_try(ffffffff82c64528) at mtx_enter_try+0x73 sys/kern/kern_lock.c:291 mtx_enter(ffffffff82c64528) at mtx_enter+0x7f sys/kern/kern_lock.c:266 pool_put(ffffffff82c64528,fffffd806ef3e990) at pool_put+0x84 sys/kern/subr_pool.c:797 soclose(fffffd806ef3e990,0) at soclose+0x4ba sys/kern/uipc_socket.c:440 soo_close(fffffd806b86dd28,ffff8000211efd48) at soo_close+0x44 fdrop(fffffd806b86dd28,ffff8000211efd48) at fdrop+0xcb sys/kern/kern_descrip.c:1274 closef(fffffd806b86dd28,ffff8000211efd48) at closef+0x120 sys/kern/kern_descrip.c:1258 syscall(ffff800021225410) at syscall+0x606 mi_syscall sys/sys/syscall_mi.h:110 [inline] syscall(ffff800021225410) at syscall+0x606 sys/arch/amd64/amd64/trap.c:623 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7b0c42470250, count: -18 ddb{0}> machine ddbcpu 1 Stopped at pool_do_put+0x115: movq 0x8(%rbx),%rbx ddb{1}> trace pool_do_put(ffffffff82c64528,fffffd806f3ac3e8) at pool_do_put+0x115 pool_put(ffffffff82c64528,fffffd806f3ac3e8) at pool_put+0x8f sys/kern/subr_pool.c:799 soclose(fffffd806f3ac3e8,0) at soclose+0x4ba sys/kern/uipc_socket.c:440 soo_close(fffffd806b86dc90,ffff8000212b3aa0) at soo_close+0x44 fdrop(fffffd806b86dc90,ffff8000212b3aa0) at fdrop+0xcb sys/kern/kern_descrip.c:1274 closef(fffffd806b86dc90,ffff8000212b3aa0) at closef+0x120 sys/kern/kern_descrip.c:1258 fdfree(ffff8000212b3aa0) at fdfree+0xf4 sys/kern/kern_descrip.c:1190 exit1(ffff8000212b3aa0,1,0,1) at exit1+0x3e4 sys/kern/kern_exit.c:206 sys_exit(ffff8000212b3aa0,ffff800022d11520,ffff800022d11570) at sys_exit+0x1a sys/kern/kern_exit.c:89 syscall(ffff800022d115f0) at syscall+0x5e2 mi_syscall sys/sys/syscall_mi.h:110 [inline] syscall(ffff800022d115f0) at syscall+0x5e2 sys/arch/amd64/amd64/trap.c:623 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x79cfc46d6fd0, count: -11