kernel: protection fault trap, code=0 Stopped at pool_do_put+0x115: movq 0x8(%rbx),%rbx ddb> ddb> set $lines = 0 ddb> set $maxwidth = 0 ddb> show panic the kernel did not panic ddb> trace pool_do_put(ffffffff82dc50c8,fffffd8063f427e8) at pool_do_put+0x115 pool_put(ffffffff82dc50c8,fffffd8063f427e8) at pool_put+0x6b sys/kern/subr_pool.c:799 in_pcbdetach(fffffd807e5979a0) at in_pcbdetach+0x34 sys/netinet/in_pcb.c:599 tcp_close(ffff800000e5ac90) at tcp_close+0x14d sys/netinet/tcp_subr.c:540 tcp_timer_2msl(ffff800000e5ac90) at tcp_timer_2msl+0x19b sys/netinet/tcp_timer.c:536 timeout_run(ffff800000e5ad78) at timeout_run+0x8f sys/kern/kern_timeout.c:665 softclock_thread(ffff80002a57d298) at softclock_thread+0xc4 sys/kern/kern_timeout.c:809 end trace frame: 0x0, count: -7 ddb> show registers rdi 0 rsi 0xfffffd8063f42bb8 rbp 0xffff80002a589090 rbx 0x4ef96160f046cb23 rdx 0 rcx 0x1 rax 0xffff80002a57d298 r8 0 r9 0 r10 0x47df71e841d5ed82 r11 0x53e60be94359a8d1 r12 0xfffffd8063f427e8 r13 0x5e58c65b2b0e3132 r14 0xffffffff82dc50c8 socket_pool r15 0xfffffd8063f42f90 rip 0xffffffff820da5d5 pool_do_put+0x115 cs 0x8 rflags 0x10286 __ALIGN_SIZE+0xf286 rsp 0xffff80002a588fe0 ss 0x10 pool_do_put+0x115: movq 0x8(%rbx),%rbx ddb> show proc PROC (softclock) tid=491062 pid=93167 tcnt=1 stat=onproc flags process=14000 proc=40000200 runpri=0, usrpri=50, slppri=0, nice=20 wchan=0x0, wmesg=, ps_single=0x0 forw=0xffffffffffffffff, list=0xffff80002a57d540,0xffff80002a57d000 process=0xffff8000ffffe9e0 user=0xffff80002a584000, vmspace=0xffffffff82e81ee8 estcpu=0, cpticks=0, pctcpu=0.0, user=0, sys=0, intr=0 ddb> ps PID TID PPID UID S FLAGS WAIT COMMAND 96672 296895 22188 0 3 0x3 kernel: protection fault trap, code=0 Faulted in DDB; continuing... ddb> show all locks No such command ddb> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10188 6444K 7523K 166960K 28878 0 pcb 15 20K 21K 166960K 816 0 rtable 243 15K 17K 166960K 1795 0 pf 35 9K 10K 166960K 243 0 ifaddr 46 12K 13K 166960K 247 0 ifgroup 62 2K 2K 166960K 416 0 sysctl 4 1K 1K 166960K 8 0 counters 33 17K 18K 166960K 121 0 ioctlops 0 0K 2K 166960K 563 0 iov 0 0K 16K 166960K 671 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1538 96K 97K 166960K 7964 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 1K 9K 166960K 63 0 VM map 2 1K 1K 166960K 2 0 sem 12 0K 0K 166960K 1221 0 dirhash 12 2K 2K 166960K 12 0 ACPI 1697 195K 286K 166960K 12548 0 file desc 10 33K 77K 166960K 7406 0 sigio 0 0K 0K 166960K 190 0 proc 61 59K 92K 166960K 1701 0 subproc 104 6K 6K 166960K 562 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 459 0 in_multi 99 7K 7K 166960K 532 0 ether_multi 1 0K 0K 166960K 7 0 mrt 0 0K 0K 166960K 6 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 79 360K 360K 166960K 79 0 exec 0 0K 1K 166960K 1639 0 pfkey data 0 0K 0K 166960K 5 0 tdb 3 0K 0K 166960K 3 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 368 322K 327K 166960K 68636 0 UVM aobj 131 7K 7K 166960K 134 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 0K 166960K 205 0 NDP 14 0K 2K 166960K 188 0 temp 87 6764K 7080K 166960K 92813 0 kqueue 12 18K 26K 166960K 574 0 SYN cache 2 16K 16K 166960K 2 0 ddb> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle rtpcb 120 409 0 406 3 2 1 3 0 8 0 rtentry 112 579 0 469 4 0 4 4 0 8 0 unpcb 144 7066 0 7053 14 7 7 10 0 8 6 syncache 320 70 0 70 2 1 1 1 0 8 1 tcpqe 32 206 0 206 2 1 1 1 0 8 1 tcpcb 808 1957 0 1946 16 5 11 12 0 8 8 arp 88 115 0 97 1 0 1 1 0 8 0 ipq 40 7 0 7 2 1 1 1 0 8 1 ipqe 40 20 0 20 2 1 1 1 0 8 1 inpcb 344 5975 0 5961 27 16 11 15 0 8 7 nd6 104 133 0 109 1 0 1 1 0 8 0 pkpcb 40 93 0 93 2 1 1 1 0 8 1 kcovpl 48 43 0 35 1 0 1 1 0 8 0 ppxss 1072 15 0 15 2 1 1 1 0 8 1 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 2164 0 1705 60 31 29 30 0 8 0 art_table 32 2165 0 1705 4 0 4 4 0 8 0 art_node 16 575 0 475 1 0 1 1 0 8 0 sysvmsgpl 40 4 0 4 1 1 0 1 0 8 0 semapl 112 1219 0 1209 1 0 1 1 0 8 0 shmpl 112 131 0 3 4 0 4 4 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 12070 0 10606 92 0 92 92 0 8 0 ffsino 240 12070 0 10606 87 0 87 87 0 8 0 nchpl 144 21774 0 20127 63 1 62 63 0 8 0 uvmvnodes 80 5926 0 0 121 0 121 121 0 8 0 vnodes 216 5926 0 0 330 0 330 330 0 8 0 namei 1024 81252 0 81252 3 2 1 3 0 8 1 vcpupl 2048 109 0 1 14 0 14 14 0 8 0 vmpool 664 116 0 8 9 0 9 9 0 8 0 kstatmem 264 204 0 176 4 1 3 3 0 8 1 scxspl 216 71231 0 71231 9 7 2 8 1 8 2 plimitpl 152 841 0 826 1 0 1 1 0 8 0 sigapl 424 7689 0 7631 10 2 8 8 0 8 0 futexpl 64 68750 0 68750 1 0 1 1 0 8 1 knotepl 120 66709 0 66627 16 13 3 16 0 8 0 kqueuepl 184 1932 0 1924 8 6 2 7 0 8 1 pipepl 288 1987 0 1959 20 10 10 11 0 8 8 fdescpl 432 7607 0 7586 4 0 4 4 0 8 0 filepl 120 53164 0 52930 25 11 14 17 0 8 6 lockfpl 104 2356 0 2354 3 1 2 2 0 8 1 lockfspl 48 924 0 922 1 0 1 1 0 8 0 sessionpl 144 58 0 42 1 0 1 1 0 8 0 pgrppl 48 116 0 100 1 0 1 1 0 8 0 ucredpl 104 8156 0 8141 1 0 1 1 0 8 0 zombiepl 144 7631 0 7631 1 0 1 1 0 8 1 processpl 1072 7689 0 7631 5 0 5 5 0 8 0 procpl 680 18088 0 18017 11 2 9 9 0 8 1 sosppl 168 96 0 96 1 0 1 1 0 8 1 sockpl 488 13556 0 13523 205 189 16 39 0 8 8 sockpl: pool(0xffffffff82dc50c8:sockpl): free list modified: page 0xfffffd8063f42000; item ordinal 1; addr 0xfffffd8063f42231 (p 0xfffffd8063f42000); offset 0x0=0x7a36bdadbd7475a6 pool(sockpl): free list modified: page 0xfffffd8063f42000; item ordinal 1; addr 0xfffffd8063f42231 (p 0xfffffd8063f42000); offset 0x0=0x10dead41 sockpl: pool(0xffffffff82dc50c8:sockpl): page inconsistency: page 0xfffffd8063f42000; item ordinal 2; addr 0x4ef96160f046cb23 mcl64k 65536 302 0 302 1 0 1 1 0 8 1 mcl16k 16384 137 0 137 1 0 1 1 0 8 1 mcl12k 12288 305 0 305 2 1 1 1 0 8 1 mcl9k 9216 124 0 124 2 1 1 1 0 8 1 mcl8k 8192 632 0 632 2 1 1 1 0 8 1 mcl4k 4096 761 0 761 4 3 1 3 0 8 1 mcl2k2 2112 53 0 53 2 1 1 1 0 8 1 mcl2k 2048 83895 0 83857 31 21 10 27 0 8 4 mtagpl 96 2187 0 2177 24 16 8 16 0 8 7 mbufpl 256 217818 0 217615 145 127 18 69 0 8 4 bufpl 280 17613 0 11224 457 0 457 457 0 8 0 anonpl 24 807004 0 794635 129 19 110 113 0 188 9 amapchunkpl 152 217717 0 217015 52 4 48 48 0 158 8 amappl16 200 15899 0 15481 47 20 27 35 0 8 3 amappl15 192 23 0 22 1 0 1 1 0 8 0 amappl14 184 255 0 244 2 1 1 2 0 8 0 amappl13 176 26 0 26 2 1 1 1 0 8 1 amappl12 168 8832 0 8805 2 0 2 2 0 8 0 amappl11 160 50 0 40 1 0 1 1 0 8 0 amappl10 152 62 0 53 1 0 1 1 0 8 0 amappl9 144 182 0 180 1 0 1 1 0 8 0 amappl8 136 431 0 347 3 0 3 3 0 8 0 amappl7 128 252 0 228 2 0 2 2 0 8 0 amappl6 120 812 0 804 1 0 1 1 0 8 0 amappl5 112 256 0 248 1 0 1 1 0 8 0 amappl4 104 706 0 683 2 1 1 2 0 8 0 amappl3 96 43113 0 43054 3 0 3 3 0 8 0 amappl2 88 8468 0 8402 3 1 2 3 0 8 0 amappl1 80 37683 0 37183 22 10 12 22 0 8 0 amappl 88 67693 0 67490 7 0 7 7 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 133 0 3 3 0 3 3 0 8 0 uaddrrnd 24 7723 0 7594 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 7723 0 7594 1 0 1 1 0 8 0 vmmpekpl 168 58821 0 58761 4 0 4 4 0 8 0 vmmpepl 168 470545 0 468505 151 19 132 132 0 357 19 vmsppl 352 7722 0 7594 13 0 13 13 0 8 0 rwobjpl 24 115611 0 108124 48 1 47 47 0 8 0 pdppl 4096 15452 0 15296 564 392 172 175 0 8 16 pvpl 32 2221901 0 2204370 409 187 222 362 0 265 42 pmappl 216 7722 0 7594 8 0 8 8 0 8 0 extentpl 40 56 0 38 1 0 1 1 0 8 0 phpool 112 1058 0 596 15 0 15 15 0 8 0 ddb> machine ddbcpu 0 No such command ddb> trace pool_do_put(ffffffff82dc50c8,fffffd8063f427e8) at pool_do_put+0x115 pool_put(ffffffff82dc50c8,fffffd8063f427e8) at pool_put+0x6b sys/kern/subr_pool.c:799 in_pcbdetach(fffffd807e5979a0) at in_pcbdetach+0x34 sys/netinet/in_pcb.c:599 tcp_close(ffff800000e5ac90) at tcp_close+0x14d sys/netinet/tcp_subr.c:540 tcp_timer_2msl(ffff800000e5ac90) at tcp_timer_2msl+0x19b sys/netinet/tcp_timer.c:536 timeout_run(ffff800000e5ad78) at timeout_run+0x8f sys/kern/kern_timeout.c:665 softclock_thread(ffff80002a57d298) at softclock_thread+0xc4 sys/kern/kern_timeout.c:809 end trace frame: 0x0, count: -7 ddb> machine ddbcpu 1 No such command ddb> trace pool_do_put(ffffffff82dc50c8,fffffd8063f427e8) at pool_do_put+0x115 pool_put(ffffffff82dc50c8,fffffd8063f427e8) at pool_put+0x6b sys/kern/subr_pool.c:799 in_pcbdetach(fffffd807e5979a0) at in_pcbdetach+0x34 sys/netinet/in_pcb.c:599 tcp_close(ffff800000e5ac90) at tcp_close+0x14d sys/netinet/tcp_subr.c:540 tcp_timer_2msl(ffff800000e5ac90) at tcp_timer_2msl+0x19b sys/netinet/tcp_timer.c:536 timeout_run(ffff800000e5ad78) at timeout_run+0x8f sys/kern/kern_timeout.c:665 softclock_thread(ffff80002a57d298) at softclock_thread+0xc4 sys/kern/kern_timeout.c:809 end trace frame: 0x0, count: -7