kernel: page fault trap, code=0 Stopped at schedclock+0xa6: movzbl 0x344(%rax),%eax ddb> ddb> set $lines = 0 ddb> set $maxwidth = 0 ddb> show panic the kernel did not panic ddb> trace schedclock(ffff80002c10dd38) at schedclock+0xa6 sys/kern/sched_bsd.c:569 statclock(ffffffff82cbcbd0,ffff800037867480,0) at statclock+0x1dd sys/kern/kern_clock.c:323 clockintr_dispatch(ffff800037867480) at clockintr_dispatch+0x24c sys/kern/kern_clockintr.c:214 lapic_clockintr(0,0) at lapic_clockintr+0x3a sys/arch/amd64/amd64/lapic.c:482 Xresume_lapic_ltimer() at Xresume_lapic_ltimer+0x2a uvm_objtree_RBT_COMPARE(ffffffff8297f448,fffffd8074e7be28) at uvm_objtree_RBT_COMPARE+0x60 sys/uvm/uvm_page.c:82 uvm_pagelookup(fffffd8074e7be18,6359000) at uvm_pagelookup+0x48 uvm_objtree_RBT_FIND sys/uvm/uvm_object.h:93 [inline] uvm_pagelookup(fffffd8074e7be18,6359000) at uvm_pagelookup+0x48 sys/uvm/uvm_page.c:1221 buf_map(fffffd8074e7bd58) at buf_map+0x216 sys/kern/vfs_biomem.c:124 buf_get(0,0,12261fa7) at buf_get+0x758 sys/kern/vfs_bio.c:1175 geteblk(12261fa7) at geteblk+0x3c sys/kern/vfs_bio.c:1058 readdisklabel(2902,ffffffff8175ef20,ffff800000ee8400,0) at readdisklabel+0x149 sys/arch/amd64/amd64/disksubr.c:96 vndopen(2902,1,2000,ffff80002c10dd38) at vndopen+0x17e sys/dev/vnd.c:204 spec_open(ffff8000378679f8) at spec_open+0x3e3 sys/kern/spec_vnops.c:150 VOP_OPEN(fffffd8071c697f8,1,fffffd807f7d7618,ffff80002c10dd38) at VOP_OPEN+0x70 sys/kern/vfs_vops.c:138 vn_open(ffff800037867c48,1,0) at vn_open+0x452 sys/kern/vfs_vnops.c:177 doopenat(ffff80002c10dd38,ffffff9c,20000040,0,0,ffff800037867df0) at doopenat+0x26e sys/kern/vfs_syscalls.c:1126 syscall(ffff800037867ea0) at syscall+0x751 sys/arch/amd64/amd64/trap.c:577 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x28c99bd85f0, count: -18 ddb> show registers rdi 0x24 rsi 0x1 rbp 0xffff800037867370 rbx 0x24 rdx 0 rcx 0x2c1f __ALIGN_SIZE+0x1c1f rax 0 r8 0 r9 0 r10 0 r11 0x46caf17956eaa2ae r12 0x1 r13 0x1 r14 0xffff80002c10dd38 r15 0xc rip 0xffffffff81d6d5b6 schedclock+0xa6 cs 0x8 rflags 0x10297 __ALIGN_SIZE+0xf297 rsp 0xffff800037867330 ss 0x10 schedclock+0xa6: movzbl 0x344(%rax),%eax ddb> show proc kernel: page fault trap, code=0 Faulted in DDB; continuing... ddb> ps PID TID PPID UID S FLAGS WAIT COMMAND 11886 238665 0 0 3 0x14280 nfsidl nfsio 22754 466355 0 0 3 0x14280 nfsidl nfsio 24762 54912 0 0 3 0x14280 nfsidl nfsio 66920 387852 0 0 3 0x14280 nfsidl nfsio 3352 305639 0 0 3 0x14280 nfsidl nfsio 45094 499998 0 0 3 0x14280 nfsidl nfsio 9340 222746 0 0 3 0x14280 nfsidl nfsio 81475 412013 0 0 3 0x14280 nfsidl nfsio 34128 20729 0 0 3 0x14280 nfsidl nfsio 49091 293636 0 0 3 0x14280 nfsidl nfsio ddb> show all locks No such command ddb> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10199 6419K 6852K 166960K 16929 0 pcb 16 16K 18K 166960K 397 0 rtable 162 13K 15K 166960K 1018 0 pf 25 8K 9K 166960K 172 0 ifaddr 30 9K 12K 166960K 166 0 ifgroup 42 1K 2K 166960K 289 0 sysctl 3 0K 0K 166960K 3 0 counters 28 17K 17K 166960K 90 0 ioctlops 0 0K 2K 166960K 500 0 iov 0 0K 24K 166960K 393 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1407 88K 88K 166960K 3314 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 1K 5K 166960K 55 0 VM map 2 1K 1K 166960K 2 0 sem 12 0K 1K 166960K 879 0 dirhash 12 2K 2K 166960K 21 0 ACPI 1697 195K 286K 166960K 12548 0 file desc 16 57K 73K 166960K 3540 0 sigio 0 0K 0K 166960K 187 0 proc 60 59K 83K 166960K 1056 0 subproc 104 6K 6K 166960K 350 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 320 0 in_multi 61 4K 7K 166960K 324 0 ether_multi 1 0K 0K 166960K 1 0 mrt 0 0K 0K 166960K 3 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 223 996K 996K 166960K 223 0 exec 0 0K 1K 166960K 1294 0 tdb 3 0K 0K 166960K 3 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 401 194K 194K 166960K 34379 0 UVM aobj 131 4K 4K 166960K 134 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 0K 166960K 108 0 NDP 10 0K 2K 166960K 124 0 temp 74 6764K 6836K 166960K 27396 0 kqueue 13 20K 24K 166960K 271 0 SYN cache 2 16K 16K 166960K 2 0 ddb> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle rtpcb 120 323 0 320 5 2 3 3 0 8 2 rtentry 112 319 0 248 4 0 4 4 0 8 0 unpcb 144 3111 0 3095 16 10 6 6 0 8 5 syncache 336 50 0 50 3 2 1 1 0 8 1 sackhl 24 1 0 1 1 1 0 1 0 8 0 tcpqe 32 193 0 193 3 2 1 1 0 8 1 tcpcb 808 979 0 965 21 12 9 9 0 8 6 arp 88 54 0 44 1 0 1 1 0 8 0 ipq 40 2 0 2 1 1 0 1 0 8 0 ipqe 40 7 0 7 1 1 0 1 0 8 0 inpcb 360 2924 0 2905 28 17 11 14 0 8 8 nd6 104 85 0 71 1 0 1 1 0 8 0 pkpcb 40 9 0 9 1 1 0 1 0 8 0 kcovpl 48 26 0 18 1 0 1 1 0 8 0 ppxss 1072 12 0 12 1 1 0 1 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 1324 0 995 36 6 30 30 0 8 7 art_table 32 1325 0 995 4 0 4 4 0 8 0 art_node 16 318 0 253 1 0 1 1 0 8 0 sysvmsgpl 40 46 0 12 1 0 1 1 0 8 0 semupl 112 4 0 4 1 1 0 1 0 8 0 semapl 112 877 0 867 1 0 1 1 0 8 0 shmpl 112 131 0 3 4 0 4 4 0 8 0 dirhash 1024 23 0 6 3 0 3 3 0 8 0 dirhash: pool(0xffffffff82e47b28:dirhash): free list modified: page 0xffff80002a616000; item ordinal 0; addr 0xffff80002a617400 (p 0xfffffd8071ec4000); offset 0x0=0x0 pool(dirhash): free list modified: page 0xffff80002a616000; item ordinal 0; addr 0xffff80002a617400 (p 0xfffffd8071ec4000); offset 0x0=0x0 dirhash: pool(0xffffffff82e47b28:dirhash): page inconsistency: page 0xffff80002a616000; item ordinal 1; addr 0x4123554026be6962 dino2pl 256 6043 0 4602 91 0 91 91 0 8 0 ffsino 240 6043 0 4602 86 0 86 86 0 8 0 nchpl 144 10853 0 9204 63 0 63 63 0 8 0 uvmvnodes 80 5926 0 0 121 0 121 121 0 8 0 vnodes 216 5926 0 0 330 0 330 330 0 8 0 namei 1024 39775 0 39775 3 2 1 2 0 8 1 namei: pool(0xffffffff82daa728:namei): free list modified: page 0xffff80002a68e000; item ordinal 0; addr 0xffff80002a68e800 (p 0xfffffd8069878000); offset 0x0=0x0 namei: pool(0xffffffff82daa728:namei): page inconsistency: page 0xffff80002a68e000; item ordinal 1; addr 0x682b706f1fd68cf2 vcpupl 2048 191 0 0 24 0 24 24 0 8 0 vmpool 664 197 0 6 17 0 17 17 0 8 1 vmpool: pool(0xffffffff82db93d0:vmpool): page inconsistency: page 0x0; at page head addr 0xffff80002c0f1f90 (p 0xffff80002c0f0000) vmpool: pool(0xffffffff82db93d0:vmpool): page inconsistency: page 0x0; at page head addr 0xffff80002a671f90 (p 0xffff80002a670000) vmpool: pool(0xffffffff82db93d0:vmpool): page inconsistency: page 0x0; at page head addr 0xffff80002c0ebf90 (p 0xffff80002c0ea000) kstatmem 264 142 0 124 2 0 2 2 0 8 0 scxspl 216 33664 0 33663 9 8 1 8 1 8 0 plimitpl 152 451 0 436 1 0 1 1 0 8 0 sigapl 424 3881 0 3816 8 0 8 8 0 8 0 futexpl 64 32787 0 32779 1 0 1 1 0 8 0 knotepl 120 35899 0 35815 7 1 6 6 0 8 2 kqueuepl 184 653 0 643 8 4 4 4 0 8 3 pipepl 288 830 0 801 13 8 5 7 0 8 2 fdescpl 432 3784 0 3757 4 0 4 4 0 8 0 filepl 120 24201 0 23919 23 9 14 14 0 8 4 lockfpl 104 1119 0 1116 3 1 2 2 0 8 1 lockfspl 48 379 0 376 1 0 1 1 0 8 0 sessionpl 144 41 0 25 1 0 1 1 0 8 0 pgrppl 48 141 0 125 1 0 1 1 0 8 0 ucredpl 104 3413 0 3402 1 0 1 1 0 8 0 zombiepl 144 3817 0 3816 1 0 1 1 0 8 0 processpl 1072 3881 0 3816 5 0 5 5 0 8 0 processpl: pool(0xffffffff82d710b0:processpl): page inconsistency: page 0x0; at page head addr 0xffff80002a64ff90 (p 0xffff80002a64c000) procpl 680 8942 0 8851 11 2 9 9 0 8 1 procpl: pool(0xffffffff82d70f08:procpl): free list modified: page 0xffff80002f57a000; item ordinal 0; addr 0xffff80002f57a2a8 (p 0xfffffd807221c000); offset 0x0=0x0 pool(procpl): free list modified: page 0xffff80002f57a000; item ordinal 0; addr 0xffff80002f57a2a8 (p 0xfffffd807221c000); offset 0x0=0x0 procpl: pool(0xffffffff82d70f08:procpl): page inconsistency: page 0xffff80002f57a000; item ordinal 1; addr 0x16ed57313410cabf procpl: pool(0xffffffff82d70f08:procpl): free list modified: page 0xffff80002a63e000; item ordinal 0; addr 0xffff80002a63ed60 (p 0xfffffd8071be2000); offset 0x0=0x0 pool(procpl): free list modified: page 0xffff80002a63e000; item ordinal 0; addr 0xffff80002a63ed60 (p 0xfffffd8071be2000); offset 0x0=0x0 procpl: pool(0xffffffff82d70f08:procpl): page inconsistency: page 0xffff80002a63e000; item ordinal 1; addr 0xa1ee952b34830240 sosppl 168 42 0 42 3 2 1 1 0 8 1 sockpl 488 6384 0 6346 125 111 14 28 0 8 8 mcl64k 65536 145 0 145 3 2 1 1 0 8 1 mcl16k 16384 61 0 61 3 2 1 1 0 8 1 mcl12k 12288 104 0 104 3 2 1 1 0 8 1 mcl9k 9216 85 0 85 3 2 1 1 0 8 1 mcl8k 8192 250 0 250 3 2 1 1 0 8 1 mcl4k 4096 489 0 489 4 3 1 2 0 8 1 mcl2k2 2112 35 0 35 3 2 1 1 0 8 1 mcl2k 2048 77042 0 76960 38 22 16 27 0 8 4 mtagpl 96 585 0 524 5 2 3 4 0 8 0 mbufpl 256 152129 0 151955 74 50 24 60 0 8 8 bufpl 280 9904 0 3512 457 0 457 457 0 8 0 anonpl 24 506295 0 492351 132 28 104 107 0 188 8 amapchunkpl 152 108558 0 107666 42 5 37 38 0 158 0 amappl16 200 12183 0 11755 65 31 34 40 0 8 8 amappl15 192 8 0 8 1 1 0 1 0 8 0 amappl14 184 212 0 199 2 1 1 2 0 8 0 amappl13 176 24 0 24 2 1 1 1 0 8 1 amappl12 168 4692 0 4664 2 0 2 2 0 8 0 amappl11 160 46 0 36 1 0 1 1 0 8 0 amappl10 152 42 0 32 1 0 1 1 0 8 0 amappl9 144 126 0 125 1 0 1 1 0 8 0 amappl8 136 289 0 220 3 0 3 3 0 8 0 amappl7 128 203 0 181 2 0 2 2 0 8 0 amappl6 120 569 0 558 1 0 1 1 0 8 0 amappl5 112 217 0 207 1 0 1 1 0 8 0 amappl4 104 544 0 518 2 1 1 2 0 8 0 amappl3 96 21960 0 21874 3 0 3 3 0 8 0 amappl2 88 4439 0 4367 3 1 2 3 0 8 0 amappl1 80 22897 0 22389 22 11 11 22 0 8 0 amappl 88 33611 0 33360 8 1 7 7 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 133 0 3 3 0 3 3 0 8 0 uaddrrnd 24 3981 0 3763 2 0 2 2 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 3981 0 3763 2 0 2 2 0 8 0 vmmpekpl 168 31060 0 30976 4 0 4 4 0 8 0 vmmpepl 168 253739 0 251327 161 32 129 129 0 357 17 vmsppl 352 3980 0 3763 20 0 20 20 0 8 0 rwobjpl 24 70583 0 63051 48 0 48 48 0 8 0 pdppl 4096 7968 0 7717 415 160 255 255 0 8 4 pvpl 32 1289675 0 1270133 330 142 188 287 0 265 17 pmappl 216 3980 0 3763 13 0 13 13 0 8 0 extentpl 40 56 0 38 1 0 1 1 0 8 0 phpool 112 835 0 275 17 0 17 17 0 8 0 ddb> machine ddbcpu 0 No such command ddb> trace schedclock(ffff80002c10dd38) at schedclock+0xa6 sys/kern/sched_bsd.c:569 statclock(ffffffff82cbcbd0,ffff800037867480,0) at statclock+0x1dd sys/kern/kern_clock.c:323 clockintr_dispatch(ffff800037867480) at clockintr_dispatch+0x24c sys/kern/kern_clockintr.c:214 lapic_clockintr(0,0) at lapic_clockintr+0x3a sys/arch/amd64/amd64/lapic.c:482 Xresume_lapic_ltimer() at Xresume_lapic_ltimer+0x2a uvm_objtree_RBT_COMPARE(ffffffff8297f448,fffffd8074e7be28) at uvm_objtree_RBT_COMPARE+0x60 sys/uvm/uvm_page.c:82 uvm_pagelookup(fffffd8074e7be18,6359000) at uvm_pagelookup+0x48 uvm_objtree_RBT_FIND sys/uvm/uvm_object.h:93 [inline] uvm_pagelookup(fffffd8074e7be18,6359000) at uvm_pagelookup+0x48 sys/uvm/uvm_page.c:1221 buf_map(fffffd8074e7bd58) at buf_map+0x216 sys/kern/vfs_biomem.c:124 buf_get(0,0,12261fa7) at buf_get+0x758 sys/kern/vfs_bio.c:1175 geteblk(12261fa7) at geteblk+0x3c sys/kern/vfs_bio.c:1058 readdisklabel(2902,ffffffff8175ef20,ffff800000ee8400,0) at readdisklabel+0x149 sys/arch/amd64/amd64/disksubr.c:96 vndopen(2902,1,2000,ffff80002c10dd38) at vndopen+0x17e sys/dev/vnd.c:204 spec_open(ffff8000378679f8) at spec_open+0x3e3 sys/kern/spec_vnops.c:150 VOP_OPEN(fffffd8071c697f8,1,fffffd807f7d7618,ffff80002c10dd38) at VOP_OPEN+0x70 sys/kern/vfs_vops.c:138 vn_open(ffff800037867c48,1,0) at vn_open+0x452 sys/kern/vfs_vnops.c:177 doopenat(ffff80002c10dd38,ffffff9c,20000040,0,0,ffff800037867df0) at doopenat+0x26e sys/kern/vfs_syscalls.c:1126 syscall(ffff800037867ea0) at syscall+0x751 sys/arch/amd64/amd64/trap.c:577 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x28c99bd85f0, count: -18 ddb> machine ddbcpu 1 No such command ddb> trace schedclock(ffff80002c10dd38) at schedclock+0xa6 sys/kern/sched_bsd.c:569 statclock(ffffffff82cbcbd0,ffff800037867480,0) at statclock+0x1dd sys/kern/kern_clock.c:323 clockintr_dispatch(ffff800037867480) at clockintr_dispatch+0x24c sys/kern/kern_clockintr.c:214 lapic_clockintr(0,0) at lapic_clockintr+0x3a sys/arch/amd64/amd64/lapic.c:482 Xresume_lapic_ltimer() at Xresume_lapic_ltimer+0x2a uvm_objtree_RBT_COMPARE(ffffffff8297f448,fffffd8074e7be28) at uvm_objtree_RBT_COMPARE+0x60 sys/uvm/uvm_page.c:82 uvm_pagelookup(fffffd8074e7be18,6359000) at uvm_pagelookup+0x48 uvm_objtree_RBT_FIND sys/uvm/uvm_object.h:93 [inline] uvm_pagelookup(fffffd8074e7be18,6359000) at uvm_pagelookup+0x48 sys/uvm/uvm_page.c:1221 buf_map(fffffd8074e7bd58) at buf_map+0x216 sys/kern/vfs_biomem.c:124 buf_get(0,0,12261fa7) at buf_get+0x758 sys/kern/vfs_bio.c:1175 geteblk(12261fa7) at geteblk+0x3c sys/kern/vfs_bio.c:1058 readdisklabel(2902,ffffffff8175ef20,ffff800000ee8400,0) at readdisklabel+0x149 sys/arch/amd64/amd64/disksubr.c:96 vndopen(2902,1,2000,ffff80002c10dd38) at vndopen+0x17e sys/dev/vnd.c:204 spec_open(ffff8000378679f8) at spec_open+0x3e3 sys/kern/spec_vnops.c:150 VOP_OPEN(fffffd8071c697f8,1,fffffd807f7d7618,ffff80002c10dd38) at VOP_OPEN+0x70 sys/kern/vfs_vops.c:138 vn_open(ffff800037867c48,1,0) at vn_open+0x452 sys/kern/vfs_vnops.c:177 doopenat(ffff80002c10dd38,ffffff9c,20000040,0,0,ffff800037867df0) at doopenat+0x26e sys/kern/vfs_syscalls.c:1126 syscall(ffff800037867ea0) at syscall+0x751 sys/arch/amd64/amd64/trap.c:577 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x28c99bd85f0, count: -18