kernel: protection fault trap, code=0 Stopped at pool_do_put+0x115: movq 0x8(%rbx),%rbx ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic the kernel did not panic ddb{0}> trace pool_do_put(ffffffff82ceba90,fffffd8075f049a8) at pool_do_put+0x115 pool_put(ffffffff82ceba90,fffffd8075f049a8) at pool_put+0x8b sys/kern/subr_pool.c:799 in_pcbdetach(fffffd8066de8cf8) at in_pcbdetach+0x30 sys/netinet/in_pcb.c:580 tcp_close(ffff800000d86c88) at tcp_close+0x13d sys/netinet/tcp_subr.c:534 tcp_timer_2msl(ffff800000d86c88) at tcp_timer_2msl+0x187 sys/netinet/tcp_timer.c:530 timeout_run(ffff800000d86d70) at timeout_run+0xcc sys/kern/kern_timeout.c:641 softclock_thread(ffff8000211795c0) at softclock_thread+0x134 sys/kern/kern_timeout.c:765 end trace frame: 0x0, count: -7 ddb{0}> show registers rdi 0 rsi 0xfffffd8075f04208 rbp 0xffff800021184f30 rbx 0x8404c333efbc76dd rdx 0 rcx 0x9 rax 0xffff8000211795c0 r8 0 r9 0 r10 0x76e6823295be85c4 r11 0xfaf241f5de11ed82 r12 0xfffffd8075f049a8 r13 0x6b90ac620dc442da r14 0xffffffff82ceba90 socket_pool r15 0xfffffd8075f04f90 rip 0xffffffff81a7e465 pool_do_put+0x115 cs 0x8 rflags 0x10212 __ALIGN_SIZE+0xf212 rsp 0xffff800021184e80 ss 0x10 pool_do_put+0x115: movq 0x8(%rbx),%rbx ddb{0}> show proc PROC (softclock) pid=32361 stat=onproc flags process=14000 proc=40000200 pri=0, usrpri=50, nice=20 forw=0xffffffffffffffff, list=0xffff800021179878,0xffff800021179318 process=0xffff8000ffffd4f0 user=0xffff800021180000, vmspace=0xffffffff82db7ec8 estcpu=0, cpticks=0, pctcpu=0.0 user=0, sys=0, intr=0 ddb{0}> ps PID TID PPID UID S FLAGS WAIT COMMAND 41108 504532 0 0 3 0x14280 nfsidl nfsio 509 262474 0 0 3 0x14280 nfsidl nfsio 44061 208417 0 0 3 0x14280 nfsidl nfsio 30756 161323 0 0 3 0x14280 nfsidl nfsio 46686 182409 0 0 3 0x14280 nfsidl nfsio 65888 359088 0 0 3 0x14280 nfsidl nfsio 19828 40985 0 0 3 0x14280 nfsidl nfsio 87022 167571 0 0 3 0x14280 nfsidl nfsio 56288 234765 0 0 3 0x14280 nfsidl nfsio 16568 149881 0 0 3 0x14280 nfsidl nfsio 40763 49384 0 0 3 0x14280 nfsidl nfsio 3949 175386 0 0 3 0x14280 nfsidl nfsio 55660 340619 0 0 3 0x14280 nfsidl nfsio 50380 485745 0 0 3 0x14280 nfsidl nfsio 88116 42917 0 0 3 0x14280 nfsidl nfsio 75566 113110 0 0 3 0x14280 nfsidl nfsio 27158 396734 0 0 3 0x14280 nfsidl nfsio 71767 333279 0 0 3 0x14280 nfsidl nfsio 89776 124393 0 0 3 0x14280 nfsidl nfsio 70714 129474 0 0 3 0x14280 nfsidl nfsio 54927 121348 25685 0 3 0x3 kernel: protection fault trap, code=0 Faulted in DDB; continuing... ddb{0}> show all locks CPU 0: exclusive mutex sockpl r = 0 (0xffffffff82cebaa0) #0 witness_lock+0x44d #1 mtx_enter_try+0x100 #2 mtx_enter+0x4b sys/kern/kern_lock.c:266 #3 pool_put+0x80 sys/kern/subr_pool.c:797 #4 in_pcbdetach+0x30 sys/netinet/in_pcb.c:580 #5 tcp_close+0x13d sys/netinet/tcp_subr.c:534 #6 tcp_timer_2msl+0x187 sys/netinet/tcp_timer.c:530 #7 timeout_run+0xcc sys/kern/kern_timeout.c:641 #8 softclock_thread+0x134 sys/kern/kern_timeout.c:765 #9 proc_trampoline+0x1c Process 10369 (softclock) thread 0xffff8000211795c0 (32361) exclusive rwlock netlock r = 0 (0xffffffff82b49670) #0 witness_lock+0x44d #1 tcp_timer_2msl+0x27 sys/netinet/tcp_timer.c:513 #2 timeout_run+0xcc sys/kern/kern_timeout.c:641 #3 softclock_thread+0x134 sys/kern/kern_timeout.c:765 #4 proc_trampoline+0x1c shared rwlock timeout r = 0 (0xffffffff82b63460) #0 witness_lock+0x44d #1 timeout_run+0xb7 sys/kern/kern_timeout.c:637 #2 softclock_thread+0x134 sys/kern/kern_timeout.c:765 #3 proc_trampoline+0x1c exclusive kernel_lock &kernel_lock r = 0 (0xffffffff82db6228) #0 witness_lock+0x44d #1 __mp_acquire_count+0x48 sys/kern/kern_lock.c:227 #2 mi_switch+0x3bb sys/kern/sched_bsd.c:405 #3 sleep_finish+0x180 sys/kern/kern_synch.c:417 #4 softclock_thread+0xd9 sys/kern/kern_timeout.c:760 #5 proc_trampoline+0x1c exclusive mutex sockpl r = 0 (0xffffffff82cebaa0) #0 witness_lock+0x44d #1 mtx_enter_try+0x100 #2 mtx_enter+0x4b sys/kern/kern_lock.c:266 #3 pool_put+0x80 sys/kern/subr_pool.c:797 #4 in_pcbdetach+0x30 sys/netinet/in_pcb.c:580 #5 tcp_close+0x13d sys/netinet/tcp_subr.c:534 #6 tcp_timer_2msl+0x187 sys/netinet/tcp_timer.c:530 #7 timeout_run+0xcc sys/kern/kern_timeout.c:641 #8 softclock_thread+0x134 sys/kern/kern_timeout.c:765 #9 proc_trampoline+0x1c ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10217 6480K 7064K 78643K 15448 0 pcb 13 14K 15K 78643K 336 0 rtable 175 6K 8K 78643K 913 0 ifaddr 68 23K 26K 78643K 325 0 sysctl 2 0K 2K 78643K 6 0 counters 56 35K 36K 78643K 182 0 ioctlops 0 0K 4K 78643K 1617 0 iov 0 0K 24K 78643K 184 0 mount 1 1K 1K 78643K 1 0 log 0 0K 0K 78643K 4 0 vnodes 1465 92K 92K 78643K 3057 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 5K 78643K 22 0 VM map 2 1K 1K 78643K 2 0 sem 12 0K 0K 78643K 226 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1697 195K 286K 78643K 12548 0 file desc 10 33K 81K 78643K 1945 0 sigio 0 0K 0K 78643K 41 0 proc 70 91K 127K 78643K 712 0 subproc 104 6K 6K 78643K 169 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 902 0 in_multi 70 4K 6K 78643K 238 0 ether_multi 1 0K 0K 78643K 33 0 mrt 1 0K 0K 78643K 4 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 151 678K 678K 78643K 151 0 exec 0 0K 1K 78643K 547 0 tdb 3 0K 0K 78643K 3 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 8 62K 64K 78643K 10 0 UVM amap 316 85K 98K 78643K 15310 0 UVM aobj 91 3K 3K 78643K 91 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 0K 78643K 131 0 NDP 11 0K 2K 78643K 79 0 temp 150 5786K 6798K 78643K 19837 0 kqueue 12 18K 24K 78643K 156 0 SYN cache 2 16K 16K 78643K 2 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 22 0 0 1 0 1 1 0 8 0 rtpcb 120 329 0 326 5 4 1 3 0 8 0 rtentry 112 272 0 195 4 0 4 4 0 8 0 unpcb 144 1050 0 1035 18 17 1 6 0 8 0 syncache 296 15 0 15 5 5 0 1 0 8 0 tcpqe 32 287 0 287 5 5 0 1 0 8 0 tcpcb 776 2034 0 2025 44 42 2 15 0 8 0 arp 120 28 0 16 1 0 1 1 0 8 0 inpcb 368 3188 0 3176 42 40 2 16 0 8 0 nd6 48 45 0 30 1 0 1 1 0 8 0 kcovpl 48 13 0 5 1 0 1 1 0 8 0 mppekey 1024 5 0 5 1 1 0 1 0 8 0 ppxss 1256 30 0 30 5 5 0 1 0 8 0 pppxif 1456 17 0 17 4 4 0 1 0 8 0 pfstscr 40 8 0 8 3 3 0 1 0 8 0 pffrag 232 11 0 9 2 1 1 1 0 482 0 pffrnode 88 11 0 9 2 1 1 1 0 8 0 pffrent 40 59 0 57 2 1 1 1 0 8 0 pfosfp 40 1431 0 1007 5 0 5 5 0 8 0 pfosfpen 112 1431 0 716 21 0 21 21 0 8 0 pfanchor 1280 136 0 67 11 5 6 11 0 8 0 pfstitem 24 10 0 6 1 0 1 1 0 8 0 pfstkey 128 27 0 23 1 0 1 1 0 8 0 pfstate 384 19 0 16 1 0 1 1 0 8 0 pfrule 1344 21 0 20 2 1 1 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 999 0 672 30 7 23 30 0 8 0 art_table 32 1000 0 672 4 0 4 4 0 8 0 art_node 16 204 0 134 1 0 1 1 0 8 0 sysvmsgpl 40 22 0 3 1 0 1 1 0 8 0 semupl 112 5 0 5 1 1 0 1 0 8 0 semapl 112 221 0 211 1 0 1 1 0 8 0 shmpl 112 88 0 0 3 0 3 3 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 3843 0 2400 92 1 91 91 0 8 0 ffsino 272 3843 0 2400 97 0 97 97 0 8 0 nchpl 144 6392 0 4753 63 0 63 63 0 8 0 uvmvnodes 80 5469 0 0 112 0 112 112 0 8 0 vnodes 216 5469 0 0 304 0 304 304 0 8 0 namei 1024 22844 0 22844 3 3 0 2 0 8 0 percpumem 16 104 0 63 1 0 1 1 0 8 0 vmpool 696 3 0 3 1 1 0 1 0 8 0 kstatmem 264 102 0 80 2 0 2 2 0 8 0 scxspl 216 19343 0 19343 14 13 1 8 0 8 1 plimitpl 152 136 0 120 1 0 1 1 0 8 0 sigapl 424 2276 0 2214 11 3 8 8 0 8 0 futexpl 64 17079 0 17079 1 1 0 1 0 8 0 knotepl 120 297 0 0 9 0 9 9 0 8 0 kqueuepl 216 354 0 346 7 6 1 5 0 8 0 pipepl 320 419 0 391 10 7 3 6 0 8 0 fdescpl 496 2237 0 2214 6 2 4 5 0 8 0 filepl 152 14566 0 14328 37 27 10 18 0 8 0 lockfpl 104 1302 0 1300 4 3 1 2 0 8 0 lockfspl 48 438 0 436 1 0 1 1 0 8 0 sessionpl 144 29 0 12 1 0 1 1 0 8 0 pgrppl 48 32 0 15 1 0 1 1 0 8 0 ucredpl 104 1347 0 1330 1 0 1 1 0 8 0 zombiepl 144 2214 0 2214 1 1 0 1 0 8 0 processpl 1072 2276 0 2214 6 1 5 5 0 8 0 procpl 696 5690 0 5610 13 3 10 10 0 8 0 srpgc 96 12 0 12 3 3 0 1 0 8 0 sosppl 168 9 0 9 2 2 0 1 0 8 0 sockpl 488 4567 0 4537 88 83 5 29 0 8 0 sockpl: pool(0xffffffff82ceba90:sockpl): free list modified: page 0xfffffd8075f04000; item ordinal 3; addr 0xfffffd8075f047c1 (p 0xfffffd8075f04000); offset 0x0=0x28e9eba4920c907 pool(sockpl): free list modified: page 0xfffffd8075f04000; item ordinal 3; addr 0xfffffd8075f047c1 (p 0xfffffd8075f04000); offset 0x0=0xefdeadbe sockpl: pool(0xffffffff82ceba90:sockpl): page inconsistency: page 0xfffffd8075f04000; item ordinal 4; addr 0x8404c333efbc76dd mcl64k 65536 19 0 0 3 1 2 3 0 8 0 mcl16k 16384 14 0 0 2 0 2 2 0 8 0 mcl12k 12288 10 0 0 1 0 1 1 0 8 0 mcl9k 9216 5 0 0 1 0 1 1 0 8 0 mcl8k 8192 17 0 0 3 0 3 3 0 8 0 mcl4k 4096 17 0 0 3 0 3 3 0 8 0 mcl2k2 2112 7 0 0 1 0 1 1 0 8 0 mcl2k 2048 421 0 0 51 0 51 51 0 8 0 mtagpl 96 104 0 0 2 0 2 2 0 8 0 mbufpl 256 2454 0 0 146 0 146 146 0 8 0 bufpl 288 6717 0 389 453 0 453 453 0 8 0 anonpl 24 478127 0 460059 137 20 117 126 0 186 0 amapchunkpl 152 43030 0 42283 47 12 35 39 0 158 0 amappl16 200 4857 0 4304 34 4 30 31 0 8 0 amappl15 192 9 0 9 2 2 0 1 0 8 0 amappl14 184 167 0 154 2 1 1 2 0 8 0 amappl13 176 3 0 3 1 1 0 1 0 8 0 amappl12 168 492 0 488 1 0 1 1 0 8 0 amappl11 160 49 0 35 1 0 1 1 0 8 0 amappl10 152 42 0 31 1 0 1 1 0 8 0 amappl9 144 970 0 969 2 1 1 1 0 8 0 amappl8 136 239 0 174 3 0 3 3 0 8 0 amappl7 128 173 0 145 2 0 2 2 0 8 0 amappl6 120 210 0 195 2 1 1 2 0 8 0 amappl5 112 177 0 168 1 0 1 1 0 8 0 amappl4 104 565 0 535 3 2 1 2 0 8 0 amappl3 96 6139 0 6094 3 1 2 2 0 8 0 amappl2 88 2703 0 2631 5 3 2 4 0 8 0 amappl1 80 53859 0 53189 31 13 18 28 0 8 0 amappl 88 14661 0 14488 8 2 6 6 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 90 0 0 2 0 2 2 0 8 0 uaddrrnd 24 2240 0 2217 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 2240 0 2217 1 0 1 1 0 8 0 vmmpekpl 168 23353 0 23303 3 0 3 3 0 8 0 vmmpepl 168 212393 0 209993 177 47 130 145 0 357 0 vmsppl 440 2239 0 2217 7 3 4 4 0 8 0 rwobjpl 56 63378 0 56136 110 5 105 105 0 8 0 pdppl 4096 4487 0 4434 207 150 57 79 0 8 4 pvpl 32 1001363 0 978122 368 162 206 367 0 265 0 pmappl 248 2239 0 2217 3 1 2 3 0 8 0 extentpl 40 56 0 38 1 0 1 1 0 8 0 phpool 112 1145 0 210 28 0 28 28 0 8 0 ddb{0}> machine ddbcpu 0 Invalid cpu 0 ddb{0}> trace pool_do_put(ffffffff82ceba90,fffffd8075f049a8) at pool_do_put+0x115 pool_put(ffffffff82ceba90,fffffd8075f049a8) at pool_put+0x8b sys/kern/subr_pool.c:799 in_pcbdetach(fffffd8066de8cf8) at in_pcbdetach+0x30 sys/netinet/in_pcb.c:580 tcp_close(ffff800000d86c88) at tcp_close+0x13d sys/netinet/tcp_subr.c:534 tcp_timer_2msl(ffff800000d86c88) at tcp_timer_2msl+0x187 sys/netinet/tcp_timer.c:530 timeout_run(ffff800000d86d70) at timeout_run+0xcc sys/kern/kern_timeout.c:641 softclock_thread(ffff8000211795c0) at softclock_thread+0x134 sys/kern/kern_timeout.c:765 end trace frame: 0x0, count: -7 ddb{0}> machine ddbcpu 1 Stopped at x86_ipi_db+0x1a: addq $0x8,%rsp ddb{1}> trace x86_ipi_db(ffff800020d68ff0) at x86_ipi_db+0x1a sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 acpicpu_idle() at acpicpu_idle+0x312 sys/dev/acpi/acpicpu.c:1206 sched_idle(ffff800020d68ff0) at sched_idle+0x417 sys/kern/kern_sched.c:175 end trace frame: 0x0, count: -5