kernel: protection fault trap, code=0 Stopped at pool_do_put+0x115: movq 0x8(%rbx),%rbx ddb> ddb> set $lines = 0 ddb> set $maxwidth = 0 ddb> show panic the kernel did not panic ddb> trace pool_do_put(ffffffff82d2dba8,fffffd8069bfca08) at pool_do_put+0x115 pool_put(ffffffff82d2dba8,fffffd8069bfca08) at pool_put+0x6b sys/kern/subr_pool.c:799 in_pcbdetach(fffffd8070659550) at in_pcbdetach+0x34 sys/netinet/in_pcb.c:578 tcp_close(ffff800000d5a970) at tcp_close+0x14d sys/netinet/tcp_subr.c:539 tcp_timer_2msl(ffff800000d5a970) at tcp_timer_2msl+0x19b sys/netinet/tcp_timer.c:530 timeout_run(ffff800000d5aa58) at timeout_run+0x8f sys/kern/kern_timeout.c:665 softclock_thread(ffff8000215e9000) at softclock_thread+0xc4 sys/kern/kern_timeout.c:809 end trace frame: 0x0, count: -7 ddb> show registers rdi 0 rsi 0xfffffd8069bfc120 rbp 0xffff8000215f6540 rbx 0x919f43a5fae9ad4b rdx 0 rcx 0x1 rax 0xffff8000215e9000 r8 0 r9 0 r10 0x45541ed0818532c0 r11 0x834c56d7bd3d2ba1 r12 0xfffffd8069bfca08 r13 0x3c21ac99db453490 r14 0xffffffff82d2dba8 socket_pool r15 0xfffffd8069bfcf90 rip 0xffffffff820315f5 pool_do_put+0x115 cs 0x8 rflags 0x10216 __ALIGN_SIZE+0xf216 rsp 0xffff8000215f6490 ss 0x10 pool_do_put+0x115: movq 0x8(%rbx),%rbx ddb> show proc PROC (softclock) tid=285717 pid=10657 tcnt=1 stat=onproc flags process=14000 proc=40000200 runpri=0, usrpri=50, slppri=0, nice=20 wchan=0x0, wmesg=, ps_single=0x0 forw=0xffffffffffffffff, list=0xffff8000215e92a8,0xffff8000215e9560 process=0xffff8000ffffe3f0 user=0xffff8000215f1000, vmspace=0xffffffff82d2e8c8 estcpu=0, cpticks=0, pctcpu=0.0, user=0, sys=0, intr=0 ddb> ps PID TID PPID UID S FLAGS WAIT COMMAND 90125 30787 28334 0 3 0x1 kernel: protection fault trap, code=0 Faulted in DDB; continuing... ddb> show all locks No such command ddb> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10171 6411K 6802K 78643K 13557 0 pcb 13 12K 13K 78643K 129 0 rtable 241 7K 7K 78643K 510 0 pf 29 8K 9K 78643K 52 0 ifaddr 43 11K 11K 78643K 60 0 ifgroup 50 2K 2K 78643K 81 0 sysctl 2 1K 1K 78643K 2 0 counters 28 17K 17K 78643K 38 0 ioctlops 0 0K 2K 78643K 114 0 iov 0 0K 12K 78643K 62 0 mount 1 1K 1K 78643K 1 0 log 0 0K 0K 78643K 4 0 vnodes 1344 84K 84K 78643K 2112 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 5K 78643K 14 0 VM map 2 1K 1K 78643K 2 0 sem 12 0K 0K 78643K 310 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1697 195K 286K 78643K 12548 0 file desc 11 37K 73K 78643K 1522 0 sigio 0 0K 0K 78643K 4 0 proc 57 59K 75K 78643K 585 0 subproc 104 6K 6K 78643K 117 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 165 0 in_multi 99 7K 7K 78643K 156 0 ether_multi 1 0K 0K 78643K 7 0 mrt 1 0K 0K 78643K 4 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 61 281K 281K 78643K 61 0 exec 0 0K 1K 78643K 560 0 tdb 3 0K 0K 78643K 3 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 8 62K 64K 78643K 10 0 UVM amap 292 131K 136K 78643K 16111 0 UVM aobj 92 3K 3K 78643K 95 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 0K 78643K 31 0 NDP 11 0K 2K 78643K 39 0 temp 77 5916K 5972K 78643K 9279 0 kqueue 12 18K 24K 78643K 149 0 SYN cache 2 16K 16K 78643K 2 0 ddb> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle rtpcb 120 86 0 83 1 0 1 1 0 8 0 rtentry 112 125 0 13 4 0 4 4 0 8 0 unpcb 144 1692 0 1678 13 12 1 6 0 8 0 syncache 304 13 0 13 4 4 0 1 0 8 0 tcpqe 32 432 0 432 4 4 0 1 0 8 0 tcpcb 808 2550 0 2539 27 23 4 18 0 8 0 arp 88 21 0 2 1 0 1 1 0 8 0 ipq 40 1 0 1 1 1 0 1 0 8 0 ipqe 40 4 0 4 1 1 0 1 0 8 0 inpcb 336 3222 0 3201 26 22 4 15 0 8 0 nd6 104 29 0 4 1 0 1 1 0 8 0 pkpcb 40 35 0 35 1 1 0 1 0 8 0 kcovpl 48 9 0 1 1 0 1 1 0 8 0 ppxss 1160 6 0 6 2 2 0 1 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 520 0 57 30 1 29 30 0 8 0 art_table 32 521 0 57 4 0 4 4 0 8 0 art_node 16 124 0 22 1 0 1 1 0 8 0 semupl 112 2 0 2 1 1 0 1 0 8 0 semapl 112 208 0 198 1 0 1 1 0 8 0 shmpl 112 92 0 3 3 0 3 3 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 3587 0 2140 91 0 91 91 0 8 0 ffsino 240 3587 0 2140 86 0 86 86 0 8 0 nchpl 144 5939 0 5444 63 42 21 63 0 8 0 uvmvnodes 80 4116 0 0 84 0 84 84 0 8 0 vnodes 216 4116 0 0 229 0 229 229 0 8 0 namei 1024 20318 0 20318 3 3 0 3 0 8 0 vcpupl 2048 3 0 0 1 0 1 1 0 8 0 vmpool 664 6 0 3 1 0 1 1 0 8 0 kstatmem 264 42 0 20 2 0 2 2 0 8 0 scxspl 216 17888 0 17888 14 13 1 8 1 8 1 plimitpl 152 387 0 370 1 0 1 1 0 8 0 sigapl 424 1822 0 1776 6 0 6 6 0 8 0 futexpl 64 16935 0 16933 1 0 1 1 0 8 0 knotepl 120 16181 0 16101 3 0 3 3 0 8 0 kqueuepl 184 413 0 405 9 8 1 4 0 8 0 pipepl 288 567 0 539 9 7 2 7 0 8 0 fdescpl 432 1804 0 1782 4 0 4 4 0 8 0 filepl 120 15058 0 14817 23 14 9 16 0 8 0 lockfpl 104 569 0 567 2 1 1 2 0 8 0 lockfspl 48 141 0 139 1 0 1 1 0 8 0 sessionpl 144 24 0 8 1 0 1 1 0 8 0 pgrppl 48 65 0 49 1 0 1 1 0 8 0 ucredpl 104 1755 0 1740 1 0 1 1 0 8 0 zombiepl 144 1782 0 1776 1 0 1 1 0 8 0 processpl 1008 1822 0 1776 6 0 6 6 0 8 0 procpl 680 3986 0 3924 8 2 6 7 0 8 0 sosppl 168 12 0 12 1 1 0 1 0 8 0 sockpl 456 5079 0 5046 85 78 7 29 0 8 0 sockpl: pool(0xffffffff82d2dba8:sockpl): free list modified: page 0xfffffd8069bfc000; item ordinal 1; addr 0xfffffd8069bfc843 (p 0xfffffd8069bfc000); offset 0x0=0x45349088d2ba82d7 pool(sockpl): free list modified: page 0xfffffd8069bfc000; item ordinal 1; addr 0xfffffd8069bfc843 (p 0xfffffd8069bfc000); offset 0x0=0xadbeefde sockpl: pool(0xffffffff82d2dba8:sockpl): page inconsistency: page 0xfffffd8069bfc000; item ordinal 2; addr 0x919f43a5fae9ad4b mcl64k 65536 70 0 70 1 1 0 1 0 8 0 mcl16k 16384 35 0 35 3 3 0 1 0 8 0 mcl12k 12288 62 0 62 3 3 0 1 0 8 0 mcl9k 9216 11 0 11 3 3 0 1 0 8 0 mcl8k 8192 65 0 65 4 4 0 1 0 8 0 mcl4k 4096 175 0 175 1 1 0 1 0 8 0 mcl2k2 2112 14 0 14 3 3 0 1 0 8 0 mcl2k 2048 68576 0 68511 53 44 9 35 0 8 0 mtagpl 96 357 0 167 9 1 8 8 0 8 0 mbufpl 256 124095 0 123797 94 69 25 77 0 8 0 bufpl 288 6889 0 495 457 0 457 457 0 8 0 anonpl 24 326102 0 312644 111 25 86 105 0 188 0 amapchunkpl 152 51874 0 51145 42 8 34 37 0 158 0 amappl16 200 8319 0 7904 36 13 23 34 0 8 0 amappl15 192 12 0 12 1 1 0 1 0 8 0 amappl14 184 155 0 143 2 1 1 2 0 8 0 amappl13 176 13 0 13 1 1 0 1 0 8 0 amappl12 168 2462 0 2435 3 1 2 2 0 8 0 amappl11 160 53 0 42 1 0 1 1 0 8 0 amappl10 152 24 0 16 2 1 1 1 0 8 0 amappl9 144 272 0 271 1 0 1 1 0 8 0 amappl8 136 196 0 143 2 0 2 2 0 8 0 amappl7 128 172 0 155 2 0 2 2 0 8 0 amappl6 120 254 0 247 1 0 1 1 0 8 0 amappl5 112 148 0 140 1 0 1 1 0 8 0 amappl4 104 436 0 415 2 1 1 2 0 8 0 amappl3 96 10153 0 10088 3 0 3 3 0 8 0 amappl2 88 2256 0 2188 3 1 2 3 0 8 0 amappl1 80 14661 0 14172 22 10 12 22 0 8 0 amappl 88 15556 0 15367 6 0 6 6 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 94 0 3 2 0 2 2 0 8 0 uaddrrnd 24 1810 0 1785 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 1810 0 1785 1 0 1 1 0 8 0 vmmpekpl 168 19614 0 19568 3 0 3 3 0 8 0 vmmpepl 168 125739 0 123800 138 39 99 112 0 357 0 vmsppl 368 1809 0 1785 3 0 3 3 0 8 0 rwobjpl 24 41119 0 35541 34 0 34 34 0 8 0 pdppl 4096 3626 0 3573 155 102 53 68 0 8 0 pvpl 32 736185 0 717890 334 126 208 334 0 265 51 pmappl 216 1809 0 1785 3 1 2 2 0 8 0 extentpl 40 56 0 38 1 0 1 1 0 8 0 phpool 112 1059 0 284 24 0 24 24 0 8 0 ddb> machine ddbcpu 0 No such command ddb> trace pool_do_put(ffffffff82d2dba8,fffffd8069bfca08) at pool_do_put+0x115 pool_put(ffffffff82d2dba8,fffffd8069bfca08) at pool_put+0x6b sys/kern/subr_pool.c:799 in_pcbdetach(fffffd8070659550) at in_pcbdetach+0x34 sys/netinet/in_pcb.c:578 tcp_close(ffff800000d5a970) at tcp_close+0x14d sys/netinet/tcp_subr.c:539 tcp_timer_2msl(ffff800000d5a970) at tcp_timer_2msl+0x19b sys/netinet/tcp_timer.c:530 timeout_run(ffff800000d5aa58) at timeout_run+0x8f sys/kern/kern_timeout.c:665 softclock_thread(ffff8000215e9000) at softclock_thread+0xc4 sys/kern/kern_timeout.c:809 end trace frame: 0x0, count: -7 ddb> machine ddbcpu 1 No such command ddb> trace pool_do_put(ffffffff82d2dba8,fffffd8069bfca08) at pool_do_put+0x115 pool_put(ffffffff82d2dba8,fffffd8069bfca08) at pool_put+0x6b sys/kern/subr_pool.c:799 in_pcbdetach(fffffd8070659550) at in_pcbdetach+0x34 sys/netinet/in_pcb.c:578 tcp_close(ffff800000d5a970) at tcp_close+0x14d sys/netinet/tcp_subr.c:539 tcp_timer_2msl(ffff800000d5a970) at tcp_timer_2msl+0x19b sys/netinet/tcp_timer.c:530 timeout_run(ffff800000d5aa58) at timeout_run+0x8f sys/kern/kern_timeout.c:665 softclock_thread(ffff8000215e9000) at softclock_thread+0xc4 sys/kern/kern_timeout.c:809 end trace frame: 0x0, count: -7