kernel: protection fault trap, code=0 Stopped at pool_do_put+0x115: movq 0x8(%rbx),%rbx ddb> ddb> set $lines = 0 ddb> set $maxwidth = 0 ddb> show panic the kernel did not panic ddb> trace pool_do_put(ffffffff82cc03a8,fffffd8067781ce0) at pool_do_put+0x115 pool_put(ffffffff82cc03a8,fffffd8067781ce0) at pool_put+0x6b sys/kern/subr_pool.c:799 in_pcbdetach(fffffd8076a332d8) at in_pcbdetach+0x34 sys/netinet/in_pcb.c:580 tcp_close(ffff800000ea1bb0) at tcp_close+0x14d sys/netinet/tcp_subr.c:535 tcp_timer_2msl(ffff800000ea1bb0) at tcp_timer_2msl+0x191 sys/netinet/tcp_timer.c:530 timeout_run(ffff800000ea1c98) at timeout_run+0x8f sys/kern/kern_timeout.c:641 softclock_thread(ffff8000ffffeae0) at softclock_thread+0xf4 sys/kern/kern_timeout.c:765 end trace frame: 0x0, count: -7 ddb> show registers rdi 0 rsi 0xfffffd80677813f8 rbp 0xffff8000215f4180 rbx 0xffed8b38b4bae49b rdx 0 rcx 0x1 rax 0xffff8000ffffeae0 r8 0 r9 0 r10 0xd0c55f10854ac7ce r11 0x2200c61dd29c3d32 r12 0xfffffd8067781ce0 r13 0x505326971861f3e4 r14 0xffffffff82cc03a8 socket_pool r15 0xfffffd8067781f90 rip 0xffffffff82716ec5 pool_do_put+0x115 cs 0x8 rflags 0x10212 __ALIGN_SIZE+0xf212 rsp 0xffff8000215f40d0 ss 0x10 pool_do_put+0x115: movq 0x8(%rbx),%rbx ddb> show proc PROC (softclock) pid=13470 stat=onproc flags process=14000 proc=40000200 pri=0, usrpri=50, nice=20 forw=0xffffffffffffffff, list=0xffff8000ffffed98,0xffff8000ffffe010 process=0xffff8000ffffcfc0 user=0xffff8000215ef000, vmspace=0xffffffff82c80338 estcpu=0, cpticks=0, pctcpu=0.0 user=0, sys=0, intr=0 ddb> ps PID TID PPID UID S FLAGS WAIT COMMAND 62652 311539 2124 0 3 0x3 kernel: protection fault trap, code=0 Faulted in DDB; continuing... ddb> show all locks No such command ddb> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10187 6426K 6989K 78643K 13471 0 pcb 13 16K 21K 78643K 1074 0 rtable 211 15K 17K 78643K 1615 0 ifaddr 81 24K 26K 78643K 595 0 sysctl 2 0K 0K 78643K 4 0 counters 27 17K 17K 78643K 269 0 ioctlops 0 0K 2K 78643K 883 0 iov 0 0K 32K 78643K 665 0 mount 1 1K 1K 78643K 1 0 log 0 0K 0K 78643K 4 0 vnodes 1475 92K 92K 78643K 5395 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 9K 78643K 88 0 VM map 2 1K 1K 78643K 2 0 sem 25 1K 1K 78643K 101 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1697 195K 286K 78643K 12548 0 file desc 10 33K 69K 78643K 6848 0 sigio 1 0K 0K 78643K 68 0 proc 55 43K 75K 78643K 1476 0 subproc 104 6K 6K 78643K 455 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 332 0 in_multi 88 5K 6K 78643K 507 0 ether_multi 1 0K 0K 78643K 32 0 mrt 1 0K 0K 78643K 30 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 193 864K 864K 78643K 193 0 exec 0 0K 1K 78643K 1586 0 pfkey data 0 0K 0K 78643K 41 0 tdb 3 0K 0K 78643K 3 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 8 62K 64K 78643K 10 0 UVM amap 284 91K 101K 78643K 65256 0 UVM aobj 131 4K 4K 78643K 146 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 0K 78643K 205 0 NDP 12 0K 1K 78643K 205 0 temp 130 5850K 39638K 78643K 92470 0 kqueue 6 10K 26K 78643K 457 0 SYN cache 2 8K 16K 78643K 3 0 ddb> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle rtpcb 120 676 0 675 10 9 1 3 0 8 0 rtentry 112 499 0 408 4 1 3 4 0 8 0 unpcb 144 9154 0 9148 79 78 1 10 0 8 0 syncache 296 53 0 53 13 13 0 1 0 8 0 tcpqe 32 119 0 119 9 9 0 1 0 8 0 tcpcb 776 1822 0 1816 59 57 2 8 0 8 0 arp 88 87 0 71 1 0 1 1 0 8 0 ipq 40 4 0 4 2 2 0 1 0 8 0 ipqe 40 7 0 7 2 2 0 1 0 8 0 inpcb 336 7376 0 7371 148 145 3 17 0 8 1 nd6 104 114 0 93 1 0 1 1 0 8 0 pkpcb 40 259 0 259 13 13 0 1 0 8 0 kcovpl 48 35 0 27 1 0 1 1 0 8 0 mppekey 1024 3 0 3 1 1 0 1 0 8 0 ppxss 1160 186 0 186 13 13 0 1 0 8 0 pppxif 1360 117 0 117 4 4 0 1 0 8 0 pfstscr 40 72 0 57 1 0 1 1 0 8 0 pfosfp 40 85 0 83 1 0 1 1 0 8 0 pfosfpen 112 85 0 83 1 0 1 1 0 8 0 pfanchor 1288 643 0 139 47 5 42 43 0 8 0 pfqueue 264 6 0 6 2 2 0 1 0 8 0 pfstitem 24 42 0 24 1 0 1 1 0 8 0 pfstkey 128 132 0 106 1 0 1 1 0 8 0 pfstate 352 66 0 51 2 0 2 2 0 8 0 rttmr 136 9 0 9 2 2 0 1 0 8 0 art_heap8 4096 4 0 3 4 3 1 2 0 8 0 art_heap4 256 2022 0 1610 46 19 27 30 0 8 0 art_table 32 2026 0 1613 4 0 4 4 0 8 0 art_node 16 494 0 416 1 0 1 1 0 8 0 sysvmsgpl 40 48 0 8 1 0 1 1 0 8 0 semapl 112 96 0 73 1 0 1 1 0 8 0 shmpl 112 143 0 15 4 0 4 4 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 11253 0 9784 93 0 93 93 0 8 0 ffsino 240 11253 0 9784 87 0 87 87 0 8 0 nchpl 144 20256 0 18606 63 0 63 63 0 8 0 uvmvnodes 80 5926 0 0 121 0 121 121 0 8 0 vnodes 216 5926 0 0 330 0 330 330 0 8 0 namei 1024 76297 0 76294 9 8 1 3 0 8 0 kstatmem 264 268 0 244 3 1 2 3 0 8 0 scsiplug 72 3 0 3 1 1 0 1 0 8 0 scxspl 216 60838 0 60838 22 22 0 8 0 8 0 plimitpl 152 1150 0 1136 1 0 1 1 0 8 0 sigapl 424 7115 0 7055 8 0 8 8 0 8 0 futexpl 64 81545 0 81545 6 6 0 1 0 8 0 knotepl 120 78602 0 78538 49 46 3 11 0 8 0 kqueuepl 184 1496 0 1491 26 25 1 10 0 8 0 pipepl 288 1840 0 1812 36 33 3 7 0 8 0 fdescpl 432 7076 0 7058 4 0 4 4 0 8 0 filepl 120 55065 0 54849 86 78 8 16 0 8 0 lockfpl 104 1277 0 1276 3 2 1 2 0 8 0 lockfspl 48 345 0 344 1 0 1 1 0 8 0 sessionpl 144 52 0 37 1 0 1 1 0 8 0 pgrppl 48 163 0 148 1 0 1 1 0 8 0 ucredpl 104 5720 0 5711 1 0 1 1 0 8 0 zombiepl 144 7059 0 7055 3 2 1 1 0 8 0 processpl 1008 7115 0 7055 10 1 9 9 0 8 0 procpl 696 17542 0 17469 18 9 9 10 0 8 0 sosppl 168 51 0 51 15 15 0 1 0 8 0 sockpl 456 17471 0 17459 382 378 4 34 0 8 1 sockpl: pool(0xffffffff82cc03a8:sockpl): free list modified: page 0xfffffd8067781000; item ordinal 2; addr 0xfffffd806778106b (p 0xfffffd8067781000); offset 0x0=0x19e8fcf8c47c31a8 pool(sockpl): free list modified: page 0xfffffd8067781000; item ordinal 2; addr 0xfffffd806778106b (p 0xfffffd8067781000); offset 0x0=0xafbeadde sockpl: pool(0xffffffff82cc03a8:sockpl): page inconsistency: page 0xfffffd8067781000; item ordinal 3; addr 0xffed8b38b4bae49b mcl64k 65536 254 0 254 21 21 0 1 0 8 0 mcl16k 16384 143 0 143 22 22 0 1 0 8 0 mcl12k 12288 280 0 280 23 23 0 1 0 8 0 mcl9k 9216 86 0 86 17 17 0 1 0 8 0 mcl8k 8192 546 0 545 15 14 1 1 0 8 0 mcl4k 4096 1101 0 1101 13 13 0 1 0 8 0 mcl2k2 2112 56 0 56 20 20 0 1 0 8 0 mcl2k 2048 88077 0 88034 56 49 7 35 0 8 0 mtagpl 96 234 0 234 4 4 0 3 0 8 0 mbufpl 256 232688 0 232552 983 965 18 113 0 8 0 bufpl 288 14942 0 8554 457 0 457 457 0 8 0 anonpl 24 802621 0 787950 144 40 104 119 0 188 0 amapchunkpl 152 208720 0 207999 77 38 39 50 0 158 0 amappl16 200 15171 0 14696 56 29 27 37 0 8 0 amappl15 192 29 0 28 1 0 1 1 0 8 0 amappl14 184 237 0 227 2 1 1 2 0 8 0 amappl13 176 12 0 12 1 1 0 1 0 8 0 amappl12 168 8106 0 8086 2 0 2 2 0 8 0 amappl11 160 55 0 50 1 0 1 1 0 8 0 amappl10 152 61 0 50 1 0 1 1 0 8 0 amappl9 144 207 0 205 2 1 1 2 0 8 0 amappl8 136 295 0 228 3 0 3 3 0 8 0 amappl7 128 71 0 52 1 0 1 1 0 8 0 amappl6 120 490 0 476 2 1 1 2 0 8 0 amappl5 112 331 0 328 1 0 1 1 0 8 0 amappl4 104 1060 0 1031 3 2 1 2 0 8 0 amappl3 96 40931 0 40888 3 0 3 3 0 8 0 amappl2 88 7438 0 7401 4 2 2 3 0 8 0 amappl1 80 35724 0 35334 23 13 10 22 0 8 0 amappl 88 64374 0 64197 10 3 7 7 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 145 0 15 3 0 3 3 0 8 0 uaddrrnd 24 7076 0 7058 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 7076 0 7058 1 0 1 1 0 8 0 vmmpekpl 168 54953 0 54897 3 0 3 3 0 8 0 vmmpepl 168 439074 0 437338 234 125 109 137 0 357 0 vmsppl 360 7075 0 7058 3 0 3 3 0 8 0 rwobjpl 24 110641 0 103295 47 1 46 46 0 8 0 pdppl 4096 14158 0 14116 454 412 42 66 0 8 0 pvpl 32 2146868 0 2127719 366 188 178 359 0 265 0 pmappl 216 7075 0 7058 2 0 2 2 0 8 0 extentpl 40 56 0 38 1 0 1 1 0 8 0 phpool 112 2455 0 1711 25 0 25 25 0 8 0 ddb> machine ddbcpu 0 No such command ddb> trace pool_do_put(ffffffff82cc03a8,fffffd8067781ce0) at pool_do_put+0x115 pool_put(ffffffff82cc03a8,fffffd8067781ce0) at pool_put+0x6b sys/kern/subr_pool.c:799 in_pcbdetach(fffffd8076a332d8) at in_pcbdetach+0x34 sys/netinet/in_pcb.c:580 tcp_close(ffff800000ea1bb0) at tcp_close+0x14d sys/netinet/tcp_subr.c:535 tcp_timer_2msl(ffff800000ea1bb0) at tcp_timer_2msl+0x191 sys/netinet/tcp_timer.c:530 timeout_run(ffff800000ea1c98) at timeout_run+0x8f sys/kern/kern_timeout.c:641 softclock_thread(ffff8000ffffeae0) at softclock_thread+0xf4 sys/kern/kern_timeout.c:765 end trace frame: 0x0, count: -7 ddb> machine ddbcpu 1 No such command ddb> trace pool_do_put(ffffffff82cc03a8,fffffd8067781ce0) at pool_do_put+0x115 pool_put(ffffffff82cc03a8,fffffd8067781ce0) at pool_put+0x6b sys/kern/subr_pool.c:799 in_pcbdetach(fffffd8076a332d8) at in_pcbdetach+0x34 sys/netinet/in_pcb.c:580 tcp_close(ffff800000ea1bb0) at tcp_close+0x14d sys/netinet/tcp_subr.c:535 tcp_timer_2msl(ffff800000ea1bb0) at tcp_timer_2msl+0x191 sys/netinet/tcp_timer.c:530 timeout_run(ffff800000ea1c98) at timeout_run+0x8f sys/kern/kern_timeout.c:641 softclock_thread(ffff8000ffffeae0) at softclock_thread+0xf4 sys/kern/kern_timeout.c:765 end trace frame: 0x0, count: -7