login: kernel: page fault trap, code=0 Stopped at schedclock+0xa6: movzbl 0x344(%rax),%eax ddb> ddb> set $lines = 0 ddb> set $maxwidth = 0 ddb> show panic the kernel did not panic ddb> trace schedclock(ffff80002a69caa8) at schedclock+0xa6 sys/kern/sched_bsd.c:569 statclock(ffffffff82c5eac0,ffff80002f524700,0) at statclock+0x1dd sys/kern/kern_clock.c:341 clockintr_dispatch(ffff80002f524700) at clockintr_dispatch+0x24c sys/kern/kern_clockintr.c:221 lapic_clockintr(0,0) at lapic_clockintr+0x3a sys/arch/amd64/amd64/lapic.c:482 Xresume_lapic_ltimer() at Xresume_lapic_ltimer+0x2a uvm_objtree_RBT_COMPARE(ffffffff82bd5aa8,fffffd807e0b3a10) at uvm_objtree_RBT_COMPARE+0x60 sys/uvm/uvm_page.c:82 uvm_pagelookup(fffffd807e0b3a00,1632000) at uvm_pagelookup+0x48 uvm_objtree_RBT_FIND sys/uvm/uvm_object.h:93 [inline] uvm_pagelookup(fffffd807e0b3a00,1632000) at uvm_pagelookup+0x48 sys/uvm/uvm_page.c:1225 buf_map(fffffd807e0b3940) at buf_map+0x216 sys/kern/vfs_biomem.c:124 buf_get(0,0,4124ddd1) at buf_get+0x763 sys/kern/vfs_bio.c:1189 geteblk(4124ddd1) at geteblk+0x3c sys/kern/vfs_bio.c:1071 readdisklabel(2902,ffffffff814e4c50,ffff800000e2a400,0) at readdisklabel+0x149 sys/arch/amd64/amd64/disksubr.c:96 vndopen(2902,81,2000,ffff80002a69caa8) at vndopen+0x17e sys/dev/vnd.c:204 spec_open(ffff80002f524c88) at spec_open+0x3e3 sys/kern/spec_vnops.c:150 VOP_OPEN(fffffd807e73bb58,81,fffffd807f7d74e0,ffff80002a69caa8) at VOP_OPEN+0x70 sys/kern/vfs_vops.c:138 vn_open(ffff80002f524ed8,81,0) at vn_open+0x452 sys/kern/vfs_vnops.c:177 doopenat(ffff80002a69caa8,ffffff9c,20000c80,80,0,ffff80002f5250a0) at doopenat+0x26e sys/kern/vfs_syscalls.c:1126 syscall(ffff80002f525160) at syscall+0x5ef sys/arch/amd64/amd64/trap.c:591 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x9346b953a40, count: -18 ddb> show registers rdi 0x24 rsi 0x1 rbp 0xffff80002f5245f0 rbx 0x24 rdx 0 rcx 0x24c rax 0 r8 0 r9 0 r10 0 r11 0x74cb244db8c0abfe r12 0x1 r13 0x1 r14 0xffff80002a69caa8 r15 0xc rip 0xffffffff8206e0b6 schedclock+0xa6 cs 0x8 rflags 0x10297 __ALIGN_SIZE+0xf297 rsp 0xffff80002f5245b0 ss 0x10 schedclock+0xa6: movzbl 0x344(%rax),%eax ddb> show proc kernel: page fault trap, code=0 Faulted in DDB; continuing... ddb> ps PID TID PPID UID S FLAGS WAIT COMMAND 0 0 -1 0 3 0x10200 scheduler swapper ddb> show all locks No such command ddb> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10187 6418K 7307K 166960K 16167 0 pcb 16 16K 18K 166960K 281 0 rtable 209 14K 15K 166960K 823 0 pf 29 8K 9K 166960K 95 0 ifaddr 38 10K 11K 166960K 98 0 ifgroup 50 2K 2K 166960K 142 0 sysctl 2 0K 0K 166960K 2 0 counters 30 17K 17K 166960K 60 0 ioctlops 0 0K 2K 166960K 89 0 iov 0 0K 24K 166960K 657 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1448 91K 91K 166960K 2809 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 1K 9K 166960K 54 0 VM map 2 1K 1K 166960K 2 0 sem 12 0K 1K 166960K 676 0 dirhash 12 2K 2K 166960K 27 0 ACPI 1697 195K 286K 166960K 12548 0 file desc 17 61K 81K 166960K 2822 0 sigio 0 0K 0K 166960K 101 0 proc 58 59K 75K 166960K 731 0 subproc 104 6K 6K 166960K 182 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 82 0 in_multi 83 6K 7K 166960K 186 0 ether_multi 1 0K 0K 166960K 1 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 73 334K 334K 166960K 73 0 exec 0 0K 1K 166960K 1210 0 pfkey data 0 0K 0K 166960K 6 0 tdb 3 0K 0K 166960K 3 0 pagedep 1 8K 8K 166960K 1 0 inodedep 1 32K 32K 166960K 1 0 newblk 1 0K 0K 166960K 1 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 349 216K 218K 166960K 28058 0 UVM aobj 120 3K 3K 166960K 126 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 0K 166960K 43 0 NDP 11 0K 2K 166960K 70 0 temp 74 6704K 6780K 166960K 36698 0 kqueue 13 20K 26K 166960K 248 0 SYN cache 2 16K 16K 166960K 2 0 ddb> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle rtpcb 120 230 0 227 5 2 3 3 0 8 2 rtentry 112 295 0 202 4 0 4 4 0 8 0 unpcb 144 1869 0 1856 22 16 6 6 0 8 5 syncache 312 36 0 36 4 4 0 1 0 8 0 tcpqe 32 253 0 253 6 6 0 1 0 8 0 tcpcb 808 1141 0 1116 22 16 6 17 0 8 0 arp 88 34 0 19 1 0 1 1 0 8 0 ipq 40 2 0 2 1 0 1 1 0 8 1 ipqe 40 6 0 6 1 0 1 1 0 8 1 inpcb 336 2719 0 2687 28 19 9 17 0 8 3 nd6 104 44 0 24 1 0 1 1 0 8 0 pkpcb 40 4 0 4 1 1 0 1 0 8 0 kcovpl 48 14 0 6 1 0 1 1 0 8 0 ppxss 1160 17 0 17 4 3 1 1 0 8 1 ppxss: pool(0xffffffff82dfce00:ppxss): page inconsistency: page 0x0; at page head addr 0xffff80002a6bff90 (p 0xffff80002a6bc000) art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 771 0 359 35 8 27 29 0 8 0 art_table 32 772 0 359 4 0 4 4 0 8 0 art_node 16 188 0 104 1 0 1 1 0 8 0 sysvmsgpl 40 52 0 12 1 0 1 1 0 8 0 semupl 112 3 0 3 1 1 0 1 0 8 0 semapl 112 674 0 664 1 0 1 1 0 8 0 shmpl 112 123 0 6 4 0 4 4 0 8 0 dirhash 1024 27 0 10 3 0 3 3 0 8 0 dirhash: pool(0xffffffff82d35d10:dirhash): free list modified: page 0xffff80002a626000; item ordinal 0; addr 0xffff80002a626000 (p 0xfffffd806ed8e000); offset 0x0=0x0 pool(dirhash): free list modified: page 0xffff80002a626000; item ordinal 0; addr 0xffff80002a626000 (p 0xfffffd806ed8e000); offset 0x0=0x0 dirhash: pool(0xffffffff82d35d10:dirhash): page inconsistency: page 0xffff80002a626000; item ordinal 1; addr 0x31b9ab7a2ff97631 dino2pl 256 5429 0 3971 92 0 92 92 0 8 0 ffsino 240 5429 0 3971 86 0 86 86 0 8 0 nchpl 144 9669 0 8021 63 0 63 63 0 8 0 uvmvnodes 80 5926 0 0 121 0 121 121 0 8 0 vnodes 216 5926 0 0 330 0 330 330 0 8 0 namei 1024 33301 0 33300 6 5 1 3 0 8 0 namei: pool(0xffffffff82e0eff0:namei): free list modified: page 0xffff80002a5f6000; item ordinal 0; addr 0xffff80002a5f6000 (p 0xfffffd80679fa000); offset 0x0=0x0 pool(namei): free list modified: page 0xffff80002a5f6000; item ordinal 0; addr 0xffff80002a5f6000 (p 0xfffffd80679fa000); offset 0x0=0x0 namei: pool(0xffffffff82e0eff0:namei): page inconsistency: page 0xffff80002a5f6000; item ordinal 1; addr 0x3164dd2d752a5876 vcpupl 2048 13 0 0 2 0 2 2 0 8 0 vmpool 664 13 0 0 2 0 2 2 0 8 0 vmpool: pool(0xffffffff82e0dea0:vmpool): page inconsistency: page 0x0; at page head addr 0xffff80002a6cff90 (p 0xffff80002a6ce000) vmpool: pool(0xffffffff82e0dea0:vmpool): page inconsistency: page 0x0; at page head addr 0xffff80002a693f90 (p 0xffff80002a692000) kstatmem 264 82 0 60 2 0 2 2 0 8 0 scxspl 216 29449 0 29448 12 10 2 8 1 8 1 plimitpl 152 206 0 191 1 0 1 1 0 8 0 sigapl 424 3213 0 3146 8 0 8 8 0 8 0 futexpl 64 33404 0 33399 1 0 1 1 0 8 0 knotepl 120 31915 0 31831 38 35 3 16 0 8 0 kqueuepl 184 841 0 832 11 10 1 4 0 8 0 pipepl 288 514 0 486 11 8 3 7 0 8 0 fdescpl 432 3095 0 3067 4 0 4 4 0 8 0 filepl 120 19114 0 18875 38 22 16 16 0 8 7 lockfpl 104 1115 0 1113 3 2 1 2 0 8 0 lockfspl 48 502 0 500 1 0 1 1 0 8 0 sessionpl 144 30 0 14 1 0 1 1 0 8 0 pgrppl 48 39 0 23 1 0 1 1 0 8 0 ucredpl 104 1698 0 1687 1 0 1 1 0 8 0 zombiepl 144 3147 0 3146 2 1 1 1 0 8 0 processpl 1072 3213 0 3146 5 0 5 5 0 8 0 procpl 680 7337 0 7249 13 4 9 10 0 8 1 procpl: pool(0xffffffff82d48900:procpl): free list modified: page 0xffff80002a602000; item ordinal 0; addr 0xffff80002a6037f8 (p 0xfffffd8065d8c000); offset 0x0=0x0 pool(procpl): free list modified: page 0xffff80002a602000; item ordinal 0; addr 0xffff80002a6037f8 (p 0xfffffd8065d8c000); offset 0x0=0x0 procpl: pool(0xffffffff82d48900:procpl): page inconsistency: page 0xffff80002a602000; item ordinal 1; addr 0xea360e3c504f7f9a procpl: pool(0xffffffff82d48900:procpl): free list modified: page 0xffff80002a62c000; item ordinal 0; addr 0xffff80002a62d000 (p 0xfffffd806f1bc000); offset 0x0=0x0 pool(procpl): free list modified: page 0xffff80002a62c000; item ordinal 0; addr 0xffff80002a62d000 (p 0xfffffd806f1bc000); offset 0x0=0x0 procpl: pool(0xffffffff82d48900:procpl): page inconsistency: page 0xffff80002a62c000; item ordinal 1; addr 0xbd26b14227ef891e procpl: pool(0xffffffff82d48900:procpl): free list modified: page 0xffff80002a68a000; item ordinal 0; addr 0xffff80002a68ad48 (p 0xfffffd8071124000); offset 0x0=0x0 pool(procpl): free list modified: page 0xffff80002a68a000; item ordinal 0; addr 0xffff80002a68ad48 (p 0xfffffd8071124000); offset 0x0=0x0 procpl: pool(0xffffffff82d48900:procpl): page inconsistency: page 0xffff80002a68a000; item ordinal 1; addr 0x783716b578d5361d procpl: pool(0xffffffff82d48900:procpl): free list modified: page 0xffff80002a5d6000; item ordinal 0; addr 0xffff80002a5d6d50 (p 0xfffffd807f7e4000); offset 0x0=0x0 pool(procpl): free list modified: page 0xffff80002a5d6000; item ordinal 0; addr 0xffff80002a5d6d50 (p 0xfffffd807f7e4000); offset 0x0=0x0 procpl: pool(0xffffffff82d48900:procpl): page inconsistency: page 0xffff80002a5d6000; item ordinal 1; addr 0xa7f35d56338f63e2 procpl: pool(0xffffffff82d48900:procpl): free list modified: page 0xffff80002a6a4000; item ordinal 0; addr 0xffff80002a6a52b8 (p 0xfffffd806a018000); offset 0x0=0x0 pool(procpl): free list modified: page 0xffff80002a6a4000; item ordinal 0; addr 0xffff80002a6a52b8 (p 0xfffffd806a018000); offset 0x0=0x0 procpl: pool(0xffffffff82d48900:procpl): page inconsistency: page 0xffff80002a6a4000; item ordinal 1; addr 0xaaa5721494d6cb6f procpl: pool(0xffffffff82d48900:procpl): free list modified: page 0xffff80002a69c000; item ordinal 0; addr 0xffff80002a69cff8 (p 0xfffffd8071124000); offset 0x0=0x0 pool(procpl): free list modified: page 0xffff80002a69c000; item ordinal 0; addr 0xffff80002a69cff8 (p 0xfffffd8071124000); offset 0x0=0x0 procpl: pool(0xffffffff82d48900:procpl): page inconsistency: page 0xffff80002a69c000; item ordinal 1; addr 0xdfb26df0f55157ec sosppl 168 40 0 40 2 1 1 1 0 8 1 sockpl 456 4836 0 4791 98 67 31 34 0 8 21 mcl64k 65536 90 0 90 3 2 1 1 0 8 1 mcl16k 16384 85 0 85 3 2 1 1 0 8 1 mcl12k 12288 122 0 122 2 1 1 1 0 8 1 mcl9k 9216 37 0 37 5 4 1 1 0 8 1 mcl8k 8192 507 0 507 3 2 1 1 0 8 1 mcl4k 4096 293 0 293 2 1 1 1 0 8 1 mcl2k2 2112 13 0 13 5 4 1 1 0 8 1 mcl2k 2048 74478 0 74423 36 27 9 31 0 8 1 mtagpl 96 489 0 101 11 1 10 10 0 8 0 mbufpl 256 144158 0 143587 143 95 48 85 0 8 8 bufpl 288 9366 0 2973 457 0 457 457 0 8 0 anonpl 24 426500 0 411531 147 33 114 114 0 188 18 amapchunkpl 152 89959 0 89085 47 5 42 42 0 158 5 amappl16 200 10199 0 9737 60 23 37 37 0 8 12 amappl15 192 13 0 13 1 1 0 1 0 8 0 amappl14 184 174 0 162 2 1 1 2 0 8 0 amappl13 176 15 0 12 1 0 1 1 0 8 0 amappl12 168 3836 0 3806 2 0 2 2 0 8 0 amappl11 160 49 0 39 1 0 1 1 0 8 0 amappl10 152 39 0 30 1 0 1 1 0 8 0 amappl9 144 200 0 199 1 0 1 1 0 8 0 amappl8 136 216 0 158 3 0 3 3 0 8 0 amappl7 128 170 0 150 2 0 2 2 0 8 0 amappl6 120 343 0 335 1 0 1 1 0 8 0 amappl5 112 146 0 138 1 0 1 1 0 8 0 amappl4 104 504 0 481 2 1 1 2 0 8 0 amappl3 96 18109 0 18025 3 0 3 3 0 8 0 amappl2 88 3827 0 3756 3 1 2 3 0 8 0 amappl1 80 19569 0 19061 22 11 11 22 0 8 0 amappl 88 27454 0 27222 6 0 6 6 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 125 0 6 3 0 3 3 0 8 0 uaddrrnd 24 3108 0 3067 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 3108 0 3067 1 0 1 1 0 8 0 vmmpekpl 168 27194 0 27136 3 0 3 3 0 8 0 vmmpepl 168 203544 0 201262 173 70 103 124 0 357 0 vmsppl 352 3107 0 3067 4 0 4 4 0 8 0 rwobjpl 24 60011 0 52495 48 1 47 47 0 8 0 pdppl 4096 6222 0 6147 171 94 77 77 0 8 2 pvpl 32 1046476 0 1025848 422 223 199 367 0 265 27 pmappl 216 3107 0 3067 3 0 3 3 0 8 0 extentpl 40 56 0 38 1 0 1 1 0 8 0 phpool 112 1124 0 289 26 0 26 26 0 8 0 ddb> machine ddbcpu 0 No such command ddb> trace schedclock(ffff80002a69caa8) at schedclock+0xa6 sys/kern/sched_bsd.c:569 statclock(ffffffff82c5eac0,ffff80002f524700,0) at statclock+0x1dd sys/kern/kern_clock.c:341 clockintr_dispatch(ffff80002f524700) at clockintr_dispatch+0x24c sys/kern/kern_clockintr.c:221 lapic_clockintr(0,0) at lapic_clockintr+0x3a sys/arch/amd64/amd64/lapic.c:482 Xresume_lapic_ltimer() at Xresume_lapic_ltimer+0x2a uvm_objtree_RBT_COMPARE(ffffffff82bd5aa8,fffffd807e0b3a10) at uvm_objtree_RBT_COMPARE+0x60 sys/uvm/uvm_page.c:82 uvm_pagelookup(fffffd807e0b3a00,1632000) at uvm_pagelookup+0x48 uvm_objtree_RBT_FIND sys/uvm/uvm_object.h:93 [inline] uvm_pagelookup(fffffd807e0b3a00,1632000) at uvm_pagelookup+0x48 sys/uvm/uvm_page.c:1225 buf_map(fffffd807e0b3940) at buf_map+0x216 sys/kern/vfs_biomem.c:124 buf_get(0,0,4124ddd1) at buf_get+0x763 sys/kern/vfs_bio.c:1189 geteblk(4124ddd1) at geteblk+0x3c sys/kern/vfs_bio.c:1071 readdisklabel(2902,ffffffff814e4c50,ffff800000e2a400,0) at readdisklabel+0x149 sys/arch/amd64/amd64/disksubr.c:96 vndopen(2902,81,2000,ffff80002a69caa8) at vndopen+0x17e sys/dev/vnd.c:204 spec_open(ffff80002f524c88) at spec_open+0x3e3 sys/kern/spec_vnops.c:150 VOP_OPEN(fffffd807e73bb58,81,fffffd807f7d74e0,ffff80002a69caa8) at VOP_OPEN+0x70 sys/kern/vfs_vops.c:138 vn_open(ffff80002f524ed8,81,0) at vn_open+0x452 sys/kern/vfs_vnops.c:177 doopenat(ffff80002a69caa8,ffffff9c,20000c80,80,0,ffff80002f5250a0) at doopenat+0x26e sys/kern/vfs_syscalls.c:1126 syscall(ffff80002f525160) at syscall+0x5ef sys/arch/amd64/amd64/trap.c:591 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x9346b953a40, count: -18 ddb> machine ddbcpu 1 No such command ddb> trace schedclock(ffff80002a69caa8) at schedclock+0xa6 sys/kern/sched_bsd.c:569 statclock(ffffffff82c5eac0,ffff80002f524700,0) at statclock+0x1dd sys/kern/kern_clock.c:341 clockintr_dispatch(ffff80002f524700) at clockintr_dispatch+0x24c sys/kern/kern_clockintr.c:221 lapic_clockintr(0,0) at lapic_clockintr+0x3a sys/arch/amd64/amd64/lapic.c:482 Xresume_lapic_ltimer() at Xresume_lapic_ltimer+0x2a uvm_objtree_RBT_COMPARE(ffffffff82bd5aa8,fffffd807e0b3a10) at uvm_objtree_RBT_COMPARE+0x60 sys/uvm/uvm_page.c:82 uvm_pagelookup(fffffd807e0b3a00,1632000) at uvm_pagelookup+0x48 uvm_objtree_RBT_FIND sys/uvm/uvm_object.h:93 [inline] uvm_pagelookup(fffffd807e0b3a00,1632000) at uvm_pagelookup+0x48 sys/uvm/uvm_page.c:1225 buf_map(fffffd807e0b3940) at buf_map+0x216 sys/kern/vfs_biomem.c:124 buf_get(0,0,4124ddd1) at buf_get+0x763 sys/kern/vfs_bio.c:1189 geteblk(4124ddd1) at geteblk+0x3c sys/kern/vfs_bio.c:1071 readdisklabel(2902,ffffffff814e4c50,ffff800000e2a400,0) at readdisklabel+0x149 sys/arch/amd64/amd64/disksubr.c:96 vndopen(2902,81,2000,ffff80002a69caa8) at vndopen+0x17e sys/dev/vnd.c:204 spec_open(ffff80002f524c88) at spec_open+0x3e3 sys/kern/spec_vnops.c:150 VOP_OPEN(fffffd807e73bb58,81,fffffd807f7d74e0,ffff80002a69caa8) at VOP_OPEN+0x70 sys/kern/vfs_vops.c:138 vn_open(ffff80002f524ed8,81,0) at vn_open+0x452 sys/kern/vfs_vnops.c:177 doopenat(ffff80002a69caa8,ffffff9c,20000c80,80,0,ffff80002f5250a0) at doopenat+0x26e sys/kern/vfs_syscalls.c:1126 syscall(ffff80002f525160) at syscall+0x5ef sys/arch/amd64/amd64/trap.c:591 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x9346b953a40, count: -18