kernel: protection fault trap, code=0 Stopped at pool_do_put+0x115: movq 0x8(%rbx),%rbx ddb> ddb> set $lines = 0 ddb> set $maxwidth = 0 ddb> show panic the kernel did not panic ddb> trace pool_do_put(ffffffff82a95eb0,fffffd80745f6138) at pool_do_put+0x115 pool_put(ffffffff82a95eb0,fffffd80745f6138) at pool_put+0x67 sys/kern/subr_pool.c:799 in_pcbdetach(fffffd806f4e6e70) at in_pcbdetach+0x30 sys/netinet/in_pcb.c:579 tcp_close(ffff800000e4e968) at tcp_close+0x13d sys/netinet/tcp_subr.c:535 tcp_timer_2msl(ffff800000e4e968) at tcp_timer_2msl+0x16f sys/netinet/tcp_timer.c:531 timeout_run(ffff800000e4ea50) at timeout_run+0x8b sys/kern/kern_timeout.c:678 softclock_thread(ffff8000fffff260) at softclock_thread+0xf4 sys/kern/kern_timeout.c:802 end trace frame: 0x0, count: -7 ddb> show registers rdi 0 rsi 0xfffffd80745f6db0 rbp 0xffff800021684940 rbx 0xd6f35257799bfa4a rdx 0 rcx 0x1 rax 0xffff8000fffff260 r8 0 r9 0 r10 0xbf7a8d0a6157607d r11 0x73b7fda80d535593 r12 0xfffffd80745f6138 r13 0x97e33a4bbe503045 r14 0xffffffff82a95eb0 socket_pool r15 0xfffffd80745f6f90 rip 0xffffffff81248215 pool_do_put+0x115 cs 0x8 rflags 0x10216 __ALIGN_SIZE+0xf216 rsp 0xffff800021684890 ss 0x10 pool_do_put+0x115: movq 0x8(%rbx),%rbx ddb> show proc PROC (softclock) pid=250110 stat=onproc flags process=14000 proc=40000200 pri=0, usrpri=50, nice=20 forw=0xffffffffffffffff, list=0xffff8000fffff500,0xffff8000ffffe7f0 process=0xffff8000ffffcfa0 user=0xffff80002167f000, vmspace=0xffffffff82a94450 estcpu=0, cpticks=0, pctcpu=0.0 user=0, sys=0, intr=0 ddb> ps PID TID PPID UID S FLAGS WAIT COMMAND 17580 371903 68599 0 3 0x3 kernel: protection fault trap, code=0 Faulted in DDB; continuing... ddb> show all locks No such command ddb> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10220 6435K 7267K 78643K 38429 0 pcb 13 18K 22K 78643K 2582 0 rtable 267 23K 24K 78643K 18359 0 ifaddr 176 41K 42K 78643K 6482 0 sysctl 3 1K 3K 78643K 10 0 counters 27 17K 17K 78643K 419 0 ioctlops 0 0K 4K 78643K 12781 0 iov 0 0K 24K 78643K 1228 0 mount 1 1K 1K 78643K 1 0 log 0 0K 0K 78643K 4 0 vnodes 1516 95K 95K 78643K 11760 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 5K 78643K 94 0 VM map 2 0K 0K 78643K 2 0 sem 10 5K 10K 78643K 16 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1697 195K 286K 78643K 12548 0 file desc 10 33K 73K 78643K 16082 0 sigio 0 0K 0K 78643K 982 0 proc 63 59K 75K 78643K 3116 0 subproc 104 6K 6K 78643K 1209 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 465 0 in_multi 85 5K 6K 78643K 1227 0 ether_multi 2 0K 0K 78643K 41 0 mrt 3 0K 0K 78643K 30 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 241 1076K 1076K 78643K 241 0 exec 0 0K 2K 78643K 4607 0 tdb 3 0K 0K 78643K 3 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 8 62K 64K 78643K 10 0 UVM amap 473 1132K 1133K 78643K 85504 0 UVM aobj 131 4K 4K 78643K 137 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 0K 78643K 284 0 NDP 15 0K 2K 78643K 422 0 temp 142 4726K 5750K 78643K 173513 0 kqueue 12 18K 28K 78643K 737 0 SYN cache 2 16K 16K 78643K 2 0 ddb> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle rtpcb 120 11964 0 11961 73 72 1 5 0 8 0 rtentry 112 1295 0 1205 5 2 3 4 0 8 0 unpcb 144 12021 0 12008 108 107 1 10 0 8 0 syncache 296 74 0 74 21 21 0 1 0 8 0 sackhl 24 1 2 1 1 1 0 1 0 8 0 tcpqe 32 32 0 32 8 8 0 1 0 8 0 tcpcb 768 9298 0 9292 240 238 2 22 0 8 0 arp 88 193 0 177 1 0 1 1 0 8 0 ipq 40 9 0 9 5 5 0 1 0 8 0 ipqe 40 29 0 29 5 5 0 1 0 8 0 inpcb 336 15715 0 15706 220 218 2 17 0 8 0 nd6 48 291 0 270 1 0 1 1 0 8 0 pkpcb 40 14 0 14 4 4 0 1 0 8 0 kcovpl 48 93 0 85 1 0 1 1 0 8 0 ppxss 1160 260 0 260 20 20 0 1 0 8 0 pfstscr 40 18 0 8 3 2 1 1 0 8 0 pfosfp 40 11 0 10 1 0 1 1 0 8 0 pfosfpen 112 11 0 10 1 0 1 1 0 8 0 pfrke_plain 168 7 0 7 3 3 0 1 0 8 0 pfrktable 1344 2648 0 2630 2 0 2 2 0 8 0 pfanchor 1280 353 0 18 28 0 28 28 0 8 0 pftag 88 12 0 6 3 2 1 1 0 8 0 pfqueue 264 3 0 3 2 2 0 1 0 8 0 pfstitem 24 22 0 2 2 1 1 1 0 8 0 pfstkey 120 42 0 37 3 2 1 1 0 8 0 pfstate 336 21 0 11 3 2 1 1 0 8 0 pfrule 1360 3840 0 3322 44 0 44 44 0 8 0 rttmr 136 10 0 10 3 3 0 1 0 8 0 art_heap8 4096 5 0 4 5 4 1 2 0 8 0 art_heap4 256 5300 0 4919 74 49 25 30 0 8 0 art_table 32 5305 0 4923 5 1 4 4 0 8 0 art_node 16 1259 0 1182 1 0 1 1 0 8 0 semapl 112 8 0 0 1 0 1 1 0 8 0 shmpl 112 134 0 6 4 0 4 4 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 20587 0 19117 93 0 93 93 0 8 0 ffsino 240 20587 0 19117 87 0 87 87 0 8 0 nchpl 144 40865 0 40336 63 41 22 63 0 8 0 rtmask 32 20 0 20 6 6 0 1 0 8 0 uvmvnodes 80 5926 0 0 121 0 121 121 0 8 0 vnodes 216 5926 0 0 330 0 330 330 0 8 0 namei 1024 153199 0 153199 16 16 0 2 0 8 0 vcpupl 2048 148 0 0 19 0 19 19 0 8 0 vmpool 536 164 0 16 10 0 10 10 0 8 0 pfiaddrpl 120 3202 0 3152 2 0 2 2 0 8 0 kstatmem 264 454 0 424 5 2 3 3 0 8 0 scsiplug 72 7 0 7 2 2 0 1 0 8 0 scxspl 216 129839 0 129839 29 28 1 8 0 8 1 plimitpl 152 1374 0 1359 1 0 1 1 0 8 0 sigapl 424 16201 0 16137 8 0 8 8 0 8 0 futexpl 64 168966 0 168966 8 8 0 1 0 8 0 knotepl 120 174498 0 174418 76 72 4 12 0 8 0 kqueuepl 184 4168 0 4160 37 36 1 4 0 8 0 pipepl 288 3321 0 3293 72 69 3 7 0 8 0 fdescpl 432 16162 0 16141 7 3 4 4 0 8 0 filepl 120 124041 0 123808 160 150 10 17 0 8 0 lockfpl 104 3212 0 3210 9 8 1 2 0 8 0 lockfspl 48 941 0 939 1 0 1 1 0 8 0 sessionpl 144 108 0 92 1 0 1 1 0 8 0 pgrppl 48 432 0 416 1 0 1 1 0 8 0 ucredpl 104 11109 0 11098 1 0 1 1 0 8 0 zombiepl 144 16143 0 16137 7 6 1 1 0 8 0 processpl 1000 16201 0 16137 10 1 9 9 0 8 0 procpl 672 39259 0 39182 35 26 9 9 0 8 0 sosppl 168 98 0 98 12 12 0 1 0 8 0 sockpl 456 39722 0 39697 736 732 4 34 0 8 0 sockpl: pool(0xffffffff82a95eb0:sockpl): free list modified: page 0xfffffd80745f6000; item ordinal 1; addr 0xfffffd80745f685a (p 0xfffffd80745f6000); offset 0x0=0x56d5efcce06ed948 pool(sockpl): free list modified: page 0xfffffd80745f6000; item ordinal 1; addr 0xfffffd80745f685a (p 0xfffffd80745f6000); offset 0x0=0x4110dead sockpl: pool(0xffffffff82a95eb0:sockpl): page inconsistency: page 0xfffffd80745f6000; item ordinal 2; addr 0xd6f35257799bfa4a mcl64k 65536 651 0 651 31 31 0 1 0 8 0 mcl16k 16384 175 0 175 24 24 0 1 0 8 0 mcl12k 12288 496 0 496 27 27 0 1 0 8 0 mcl9k 9216 220 0 220 29 29 0 1 0 8 0 mcl8k 8192 939 0 939 30 30 0 1 0 8 0 mcl4k 4096 2764 0 2764 22 22 0 1 0 8 0 mcl2k2 2112 116 0 116 33 33 0 1 0 8 0 mcl2k 2048 99104 0 99044 55 47 8 18 0 8 0 mtagpl 96 4414 0 4058 54 43 11 20 0 8 0 mbufpl 256 554426 0 553875 11023 10970 53 1117 0 8 0 bufpl 288 27840 0 21422 459 0 459 459 0 8 0 anonpl 24 3405121 0 3387047 273 142 131 151 0 188 0 amapchunkpl 152 276216 0 275650 112 82 30 43 0 158 0 amappl16 200 73079 0 72260 132 85 47 56 0 8 0 amappl15 192 1320 0 1317 5 4 1 1 0 8 0 amappl14 184 717 0 715 1 0 1 1 0 8 0 amappl13 176 2173 0 2168 1 0 1 1 0 8 0 amappl12 168 2363 0 2362 1 0 1 1 0 8 0 amappl11 160 2713 0 2697 1 0 1 1 0 8 0 amappl10 152 1747 0 1741 1 0 1 1 0 8 0 amappl9 144 2309 0 2306 1 0 1 1 0 8 0 amappl8 136 5303 0 5201 4 0 4 4 0 8 0 amappl7 128 3761 0 3737 1 0 1 1 0 8 0 amappl6 120 2564 0 2544 2 1 1 2 0 8 0 amappl5 112 12785 0 12777 1 0 1 1 0 8 0 amappl4 104 6802 0 6771 2 1 1 2 0 8 0 amappl3 96 47441 0 47396 2 0 2 2 0 8 0 amappl2 88 4887 0 4833 3 1 2 2 0 8 0 amappl1 80 388564 0 387961 27 10 17 21 0 8 0 amappl 88 83402 0 83196 8 2 6 6 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 136 0 6 3 0 3 3 0 8 0 uaddrrnd 24 16326 0 16157 2 0 2 2 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 16326 0 16157 2 0 2 2 0 8 0 vmmpekpl 168 111556 0 111474 4 0 4 4 0 8 0 vmmpepl 168 1603772 0 1601015 356 205 151 167 0 357 0 vmsppl 272 16325 0 16157 14 2 12 12 0 8 0 rwobjpl 24 398573 0 390601 53 3 50 50 0 8 0 pdppl 4096 32658 0 32462 926 730 196 212 0 8 0 pvpl 32 6934986 0 6915582 579 390 189 259 0 265 0 pmappl 216 16325 0 16157 11 1 10 10 0 8 0 extentpl 40 56 0 38 1 0 1 1 0 8 0 phpool 112 13223 0 12216 129 87 42 58 0 8 0 ddb> machine ddbcpu 0 No such command ddb> trace pool_do_put(ffffffff82a95eb0,fffffd80745f6138) at pool_do_put+0x115 pool_put(ffffffff82a95eb0,fffffd80745f6138) at pool_put+0x67 sys/kern/subr_pool.c:799 in_pcbdetach(fffffd806f4e6e70) at in_pcbdetach+0x30 sys/netinet/in_pcb.c:579 tcp_close(ffff800000e4e968) at tcp_close+0x13d sys/netinet/tcp_subr.c:535 tcp_timer_2msl(ffff800000e4e968) at tcp_timer_2msl+0x16f sys/netinet/tcp_timer.c:531 timeout_run(ffff800000e4ea50) at timeout_run+0x8b sys/kern/kern_timeout.c:678 softclock_thread(ffff8000fffff260) at softclock_thread+0xf4 sys/kern/kern_timeout.c:802 end trace frame: 0x0, count: -7 ddb> machine ddbcpu 1 No such command ddb> trace pool_do_put(ffffffff82a95eb0,fffffd80745f6138) at pool_do_put+0x115 pool_put(ffffffff82a95eb0,fffffd80745f6138) at pool_put+0x67 sys/kern/subr_pool.c:799 in_pcbdetach(fffffd806f4e6e70) at in_pcbdetach+0x30 sys/netinet/in_pcb.c:579 tcp_close(ffff800000e4e968) at tcp_close+0x13d sys/netinet/tcp_subr.c:535 tcp_timer_2msl(ffff800000e4e968) at tcp_timer_2msl+0x16f sys/netinet/tcp_timer.c:531 timeout_run(ffff800000e4ea50) at timeout_run+0x8b sys/kern/kern_timeout.c:678 softclock_thread(ffff8000fffff260) at softclock_thread+0xf4 sys/kern/kern_timeout.c:802 end trace frame: 0x0, count: -7