kernel: protection fault trap, code=0 Stopped at pool_do_put+0x115: movq 0x8(%rbx),%rbx ddb> ddb> set $lines = 0 ddb> set $maxwidth = 0 ddb> show panic the kernel did not panic ddb> trace pool_do_put(ffffffff82d4b998,fffffd8078d3c270) at pool_do_put+0x115 pool_put(ffffffff82d4b998,fffffd8078d3c270) at pool_put+0x6b sys/kern/subr_pool.c:799 in_pcbdetach(fffffd806568d2a0) at in_pcbdetach+0x34 sys/netinet/in_pcb.c:578 tcp_close(ffff800000d7f658) at tcp_close+0x14d sys/netinet/tcp_subr.c:539 tcp_timer_2msl(ffff800000d7f658) at tcp_timer_2msl+0x19b sys/netinet/tcp_timer.c:530 timeout_run(ffff800000d7f740) at timeout_run+0x8f sys/kern/kern_timeout.c:640 softclock_thread(ffff8000215e9d48) at softclock_thread+0xd4 sys/kern/kern_timeout.c:763 end trace frame: 0x0, count: -7 ddb> show registers rdi 0 rsi 0xfffffd8078d3c990 rbp 0xffff8000215f6d30 rbx 0x618207a0a497bc67 rdx 0 rcx 0x1 rax 0xffff8000215e9d48 r8 0 r9 0 r10 0x358ee952391feba0 r11 0x51a6756c2e33b666 r12 0xfffffd8078d3c270 r13 0xdf6d27327e25ba91 r14 0xffffffff82d4b998 socket_pool r15 0xfffffd8078d3cf90 rip 0xffffffff81704545 pool_do_put+0x115 cs 0x8 rflags 0x10296 __ALIGN_SIZE+0xf296 rsp 0xffff8000215f6c80 ss 0x10 pool_do_put+0x115: movq 0x8(%rbx),%rbx ddb> show proc PROC (softclock) pid=372251 stat=onproc flags process=14000 proc=40000200 pri=0, usrpri=50, nice=20 forw=0xffffffffffffffff, list=0xffff8000215ea540,0xffff8000215e9808 process=0xffff8000ffffe7e0 user=0xffff8000215f1000, vmspace=0xffffffff82d17558 estcpu=0, cpticks=1, pctcpu=0.0 user=0, sys=0, intr=0 ddb> ps PID TID PPID UID S FLAGS WAIT COMMAND 16428 479243 0 0 3 0x14200 acct acct 84452 118545 50971 0 3 0x3 kernel: protection fault trap, code=0 Faulted in DDB; continuing... ddb> show all locks No such command ddb> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10168 6401K 6808K 78643K 12346 0 pcb 13 10K 12K 78643K 23 0 rtable 242 6K 7K 78643K 391 0 pf 29 8K 9K 78643K 49 0 ifaddr 43 11K 11K 78643K 60 0 ifgroup 50 2K 2K 78643K 84 0 counters 28 17K 17K 78643K 38 0 ioctlops 0 0K 2K 78643K 34 0 iov 0 0K 14K 78643K 66 0 mount 1 1K 1K 78643K 1 0 log 0 0K 0K 78643K 4 0 vnodes 1341 84K 84K 78643K 1773 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 5K 78643K 8 0 VM map 2 1K 1K 78643K 2 0 sem 12 0K 0K 78643K 112 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1697 195K 286K 78643K 12548 0 file desc 10 33K 81K 78643K 954 0 sigio 0 0K 0K 78643K 89 0 proc 57 59K 83K 78643K 512 0 subproc 104 6K 6K 78643K 117 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 370 0 in_multi 99 7K 7K 78643K 117 0 ether_multi 1 0K 0K 78643K 1 0 mrt 1 0K 0K 78643K 1 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 49 228K 228K 78643K 49 0 exec 0 0K 1K 78643K 431 0 tdb 3 0K 0K 78643K 3 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 8 62K 64K 78643K 10 0 UVM amap 265 82K 84K 78643K 10597 0 UVM aobj 131 4K 4K 78643K 131 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 0K 78643K 3 0 NDP 11 0K 1K 78643K 39 0 temp 77 5912K 5990K 78643K 12364 0 kqueue 12 18K 24K 78643K 63 0 SYN cache 2 16K 16K 78643K 2 0 ddb> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle rtpcb 120 44 0 41 1 0 1 1 0 8 0 rtentry 112 126 0 12 4 0 4 4 0 8 0 unpcb 144 1209 0 1196 11 10 1 6 0 8 0 syncache 304 9 0 9 3 3 0 1 0 8 0 tcpqe 32 168 0 168 3 3 0 1 0 8 0 tcpcb 808 155 0 147 10 7 3 8 0 8 0 arp 88 21 0 2 1 0 1 1 0 8 0 ipq 40 8 0 8 1 1 0 1 0 8 0 ipqe 40 17 0 17 1 1 0 1 0 8 0 inpcb 336 689 0 673 14 10 4 13 0 8 1 nd6 104 30 0 3 1 0 1 1 0 8 0 pkpcb 40 4 0 4 1 1 0 1 0 8 0 kcovpl 48 9 0 1 1 0 1 1 0 8 0 ppxss 1160 3 0 3 1 1 0 1 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 503 0 44 29 0 29 29 0 8 0 art_table 32 504 0 44 4 0 4 4 0 8 0 art_node 16 125 0 21 1 0 1 1 0 8 0 sysvmsgpl 40 6 0 4 1 0 1 1 0 8 0 semupl 112 6 0 6 1 1 0 1 0 8 0 semapl 112 104 0 94 1 0 1 1 0 8 0 shmpl 112 128 0 0 4 0 4 4 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 2516 0 1076 91 0 91 91 0 8 0 ffsino 240 2516 0 1076 85 0 85 85 0 8 0 nchpl 144 3857 0 3369 63 41 22 63 0 8 0 uvmvnodes 80 3044 0 0 63 0 63 63 0 8 0 vnodes 216 3044 0 0 170 0 170 170 0 8 0 namei 1024 12376 0 12376 2 2 0 2 0 8 0 kstatmem 264 42 0 20 2 0 2 2 0 8 0 scxspl 216 12833 0 12833 10 9 1 8 1 8 1 plimitpl 152 191 0 176 1 0 1 1 0 8 0 sigapl 424 1251 0 1210 6 0 6 6 0 8 0 futexpl 64 7649 0 7649 1 1 0 1 0 8 0 knotepl 120 14845 0 14765 18 14 4 12 0 8 0 kqueuepl 184 145 0 137 4 3 1 4 0 8 0 pipepl 288 308 0 280 9 7 2 7 0 8 0 fdescpl 432 1233 0 1212 4 0 4 4 0 8 0 filepl 120 6539 0 6303 17 9 8 14 0 8 0 lockfpl 104 758 0 756 2 1 1 2 0 8 0 lockfspl 48 246 0 244 1 0 1 1 0 8 0 sessionpl 144 24 0 8 1 0 1 1 0 8 0 pgrppl 48 24 0 8 1 0 1 1 0 8 0 ucredpl 104 364 0 354 1 0 1 1 0 8 0 zombiepl 144 1212 0 1210 1 0 1 1 0 8 0 processpl 1008 1251 0 1210 8 2 6 7 0 8 0 procpl 680 2401 0 2347 7 0 7 7 0 8 0 sosppl 168 2 0 2 1 1 0 1 0 8 0 sockpl 456 1946 0 1919 45 39 6 22 0 8 1 sockpl: pool(0xffffffff82d4b998:sockpl): free list modified: page 0xfffffd8078d3c000; item ordinal 1; addr 0xfffffd8078d3c43a (p 0xfffffd8078d3c000); offset 0x0=0x7d5973f83d813ce9 pool(sockpl): free list modified: page 0xfffffd8078d3c000; item ordinal 1; addr 0xfffffd8078d3c43a (p 0xfffffd8078d3c000); offset 0x0=0xbeefdead sockpl: pool(0xffffffff82d4b998:sockpl): page inconsistency: page 0xfffffd8078d3c000; item ordinal 2; addr 0x618207a0a497bc67 mcl64k 65536 33 0 33 1 1 0 1 0 8 0 mcl16k 16384 14 0 14 3 3 0 1 0 8 0 mcl12k 12288 17 0 17 3 3 0 1 0 8 0 mcl9k 9216 10 0 10 2 2 0 1 0 8 0 mcl8k 8192 84 0 84 3 3 0 1 0 8 0 mcl4k 4096 78 0 78 3 3 0 1 0 8 0 mcl2k2 2112 3 0 3 2 2 0 1 0 8 0 mcl2k 2048 65580 0 65535 36 28 8 33 0 8 0 mtagpl 96 128 0 49 4 2 2 3 0 8 0 mbufpl 256 109334 0 109076 33 11 22 22 0 8 0 bufpl 288 5538 0 143 386 0 386 386 0 8 0 anonpl 24 278710 0 266231 105 25 80 104 0 188 0 amapchunkpl 152 34550 0 33870 40 7 33 38 0 158 0 amappl16 200 7570 0 7205 34 14 20 33 0 8 0 amappl15 192 15 0 14 1 0 1 1 0 8 0 amappl14 184 171 0 158 2 1 1 2 0 8 0 amappl13 176 19 0 18 1 0 1 1 0 8 0 amappl12 168 1871 0 1850 2 0 2 2 0 8 0 amappl11 160 47 0 37 1 0 1 1 0 8 0 amappl10 152 20 0 11 2 1 1 1 0 8 0 amappl9 144 221 0 220 1 0 1 1 0 8 0 amappl8 136 170 0 121 2 0 2 2 0 8 0 amappl7 128 61 0 54 1 0 1 1 0 8 0 amappl6 120 233 0 216 2 1 1 2 0 8 0 amappl5 112 204 0 195 1 0 1 1 0 8 0 amappl4 104 531 0 496 3 1 2 3 0 8 0 amappl3 96 7256 0 7206 3 1 2 3 0 8 0 amappl2 88 1490 0 1433 3 1 2 3 0 8 0 amappl1 80 12723 0 12229 22 11 11 22 0 8 0 amappl 88 10080 0 9911 6 0 6 6 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 130 0 0 3 0 3 3 0 8 0 uaddrrnd 24 1233 0 1212 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 1233 0 1212 1 0 1 1 0 8 0 vmmpekpl 168 14392 0 14345 3 0 3 3 0 8 0 vmmpepl 168 94329 0 92504 135 39 96 123 0 357 0 vmsppl 368 1232 0 1212 3 0 3 3 0 8 0 rwobjpl 24 33314 0 28838 30 1 29 29 0 8 0 pdppl 4096 2472 0 2424 130 82 48 72 0 8 0 pvpl 32 606754 0 589373 361 108 253 361 0 265 101 pmappl 216 1232 0 1212 2 0 2 2 0 8 0 extentpl 40 56 0 38 1 0 1 1 0 8 0 phpool 112 852 0 159 21 0 21 21 0 8 0 ddb> machine ddbcpu 0 No such command ddb> trace pool_do_put(ffffffff82d4b998,fffffd8078d3c270) at pool_do_put+0x115 pool_put(ffffffff82d4b998,fffffd8078d3c270) at pool_put+0x6b sys/kern/subr_pool.c:799 in_pcbdetach(fffffd806568d2a0) at in_pcbdetach+0x34 sys/netinet/in_pcb.c:578 tcp_close(ffff800000d7f658) at tcp_close+0x14d sys/netinet/tcp_subr.c:539 tcp_timer_2msl(ffff800000d7f658) at tcp_timer_2msl+0x19b sys/netinet/tcp_timer.c:530 timeout_run(ffff800000d7f740) at timeout_run+0x8f sys/kern/kern_timeout.c:640 softclock_thread(ffff8000215e9d48) at softclock_thread+0xd4 sys/kern/kern_timeout.c:763 end trace frame: 0x0, count: -7 ddb> machine ddbcpu 1 No such command ddb> trace pool_do_put(ffffffff82d4b998,fffffd8078d3c270) at pool_do_put+0x115 pool_put(ffffffff82d4b998,fffffd8078d3c270) at pool_put+0x6b sys/kern/subr_pool.c:799 in_pcbdetach(fffffd806568d2a0) at in_pcbdetach+0x34 sys/netinet/in_pcb.c:578 tcp_close(ffff800000d7f658) at tcp_close+0x14d sys/netinet/tcp_subr.c:539 tcp_timer_2msl(ffff800000d7f658) at tcp_timer_2msl+0x19b sys/netinet/tcp_timer.c:530 timeout_run(ffff800000d7f740) at timeout_run+0x8f sys/kern/kern_timeout.c:640 softclock_thread(ffff8000215e9d48) at softclock_thread+0xd4 sys/kern/kern_timeout.c:763 end trace frame: 0x0, count: -7