uvm_fault(0xffffffff82883830, 0xfffffd0000000010, 0, 1) -> e kernel: page fault trap, code=0 Stopped at pool_cache_get+0x1b1: movq 0x10(%r14),%r13 ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic kernel page fault uvm_fault(0xffffffff82883830, 0xfffffd0000000010, 0, 1) -> e pool_cache_get(ffffffff828c79e0) at pool_cache_get+0x1b1 pool_cache_item_magic_check sys/kern/subr_pool.c:1770 [inline] pool_cache_get(ffffffff828c79e0) at pool_cache_get+0x1b1 sys/kern/subr_pool.c:1884 end trace frame: 0xffff800020e040b0, count: 0 ddb{1}> trace pool_cache_get(ffffffff828c79e0) at pool_cache_get+0x1b1 pool_cache_item_magic_check sys/kern/subr_pool.c:1770 [inline] pool_cache_get(ffffffff828c79e0) at pool_cache_get+0x1b1 sys/kern/subr_pool.c:1884 pool_get(ffffffff828c79e0,2) at pool_get+0x91 sys/kern/subr_pool.c:572 m_gethdr(2,2) at m_gethdr+0x4c sys/kern/uipc_mbuf.c:283 tcp_output(ffff800000ada320) at tcp_output+0x14e6 tcp_usrreq(fffffd806eadf968,9,fffffd806d1df600,0,0,ffff800020e40600) at tcp_usrreq+0xa55 sosend(fffffd806eadf968,0,ffff800020e044e8,0,0,80) at sosend+0x671 sys/kern/uipc_socket.c:555 dofilewritev(ffff800020e40600,4,ffff800020e044e8,0,ffff800020e045d0) at dofilewritev+0x1b6 sys/kern/sys_generic.c:365 sys_write(ffff800020e40600,ffff800020e04580,ffff800020e045d0) at sys_write+0x83 sys/kern/sys_generic.c:285 syscall(ffff800020e04650) at syscall+0x4a4 mi_syscall sys/sys/syscall_mi.h:102 [inline] syscall(ffff800020e04650) at syscall+0x4a4 sys/arch/amd64/amd64/trap.c:570 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffd8090, count: -10 ddb{1}> show registers rdi 0 rsi 0x7 rbp 0xffff800020e03ff0 rbx 0xf6a9d26a64d26088 rdx 0xffff800020e40600 rcx 0 rax 0 r8 0xffffffff815a43da tcp_output+0xbea r9 0x5 r10 0x3 r11 0x4cedd60fbc87141 r12 0xffffffff828c79e0 mbpool r13 0 r14 0xfffffd0000000000 r15 0xfffffd807f009e00 rip 0xffffffff81393ca1 pool_cache_get+0x1b1 cs 0x8 rflags 0x10286 __ALIGN_SIZE+0xf286 rsp 0xffff800020e03f90 ss 0x10 pool_cache_get+0x1b1: movq 0x10(%r14),%r13 ddb{1}> show proc PROC (sshd) pid=207973 stat=onproc flags process=12 proc=0 pri=50, usrpri=50, nice=20 forw=0xffffffffffffffff, list=0xffff800020e3f010,0xffff800020e3f500 process=0xffff8000ffffe7d8 user=0xffff800020dff000, vmspace=0xfffffd806e8f4008 estcpu=0, cpticks=1, pctcpu=0.0 user=0, sys=1, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 20912 301718 0 0 3 0x14280 nfsidl nfsio 27593 506996 0 0 3 0x14280 nfsidl nfsio 91595 172452 0 0 3 0x14280 nfsidl nfsio 21958 417874 0 0 3 0x14280 nfsidl nfsio 13985 298159 0 0 3 0x14280 nfsidl nfsio 50478 449733 0 0 3 0x14280 nfsidl nfsio 10399 122641 0 0 3 0x14280 nfsidl nfsio 88424 29925 0 0 3 0x14280 nfsidl nfsio 71074 415333 0 0 3 0x14280 nfsidl nfsio 8101 371601 0 0 3 0x14280 nfsidl nfsio 20535 313478 0 0 3 0x14280 nfsidl nfsio 40373 24501 0 0 3 0x14280 nfsidl nfsio 55371 281470 0 0 3 0x14280 nfsidl nfsio 61187 55114 0 0 3 0x14280 nfsidl nfsio 19551 417761 0 0 3 0x14280 nfsidl nfsio 35808 77475 0 0 3 0x14280 nfsidl nfsio 47016 216082 0 0 3 0x14280 nfsidl nfsio 2366 523299 0 0 3 0x14280 nfsidl nfsio 20955 65725 0 0 3 0x14280 nfsidl nfsio 45290 499360 0 0 3 0x14280 nfsidl nfsio 5885 422993 89642 0 3 0x82 nanosleep syz-executor.1 59016 149307 89642 0 3 0x2 biowait syz-executor.0 89642 485998 69348 0 3 0x82 thrsleep syz-fuzzer 89642 189489 69348 0 3 0x4000082 nanosleep syz-fuzzer 89642 303787 69348 0 3 0x4000082 thrsleep syz-fuzzer 89642 446650 69348 0 3 0x4000082 thrsleep syz-fuzzer 89642 136227 69348 0 3 0x4000082 thrsleep syz-fuzzer 89642 255106 69348 0 3 0x4000082 thrsleep syz-fuzzer 89642 147431 69348 0 3 0x4000082 thrsleep syz-fuzzer 89642 225264 69348 0 3 0x4000082 thrsleep syz-fuzzer 89642 254069 69348 0 3 0x4000082 kqread syz-fuzzer 89642 413857 69348 0 3 0x4000082 thrsleep syz-fuzzer 69348 271192 70921 0 3 0x10008a pause ksh *70921 207973 42898 0 7 0x12 sshd 21666 447668 1 0 3 0x100083 ttyin getty 42898 179725 1 0 3 0x80 select sshd 10496 150046 70014 74 3 0x100092 bpf pflogd 70014 70935 1 0 3 0x80 netio pflogd 23993 346062 44936 73 3 0x100090 kqread syslogd 44936 63976 1 0 3 0x100082 netio syslogd 1787 447113 1 77 3 0x100090 poll dhclient 20391 488588 1 0 3 0x80 poll dhclient 83760 517803 0 0 3 0x14200 bored smr 72691 63463 0 0 3 0x14200 pgzero zerothread 23087 340971 0 0 3 0x14200 aiodoned aiodoned 24478 317740 0 0 3 0x14200 syncer update 65131 104050 0 0 3 0x14200 cleaner cleaner 41752 497083 0 0 3 0x14200 reaper reaper 79533 16412 0 0 3 0x14200 pgdaemon pagedaemon 24486 433744 0 0 3 0x14200 bored crynlk 85762 489420 0 0 3 0x14200 bored crypto 27313 371090 0 0 3 0x40014200 acpi0 acpi0 86204 194336 0 0 3 0x40014200 idle1 70984 415856 0 0 3 0x14200 bored softnet 31098 225105 0 0 3 0x14200 bored systqmp 21706 279835 0 0 3 0x14200 bored systq 75646 274239 0 0 3 0x40014200 bored softclock 54452 414270 0 0 7 0x40014200 idle0 1 435580 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{1}> show all locks Process 59016 (syz-executor.0) thread 0xffff800020ddd868 (149307) exclusive rrwlock inode r = 0 (0xfffffd80675cdb48) #0 witness_lock+0x4c7 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4c7 sys/kern/subr_witness.c:1164 #1 rw_enter+0x453 sys/kern/kern_rwlock.c:311 #2 rrw_enter+0x88 sys/kern/kern_rwlock.c:462 #3 VOP_LOCK+0x4b sys/kern/vfs_vops.c:603 #4 vn_lock+0x81 sys/kern/vfs_vnops.c:575 #5 vget+0x1c8 sys/kern/vfs_subr.c:671 #6 ufs_ihashget+0x141 sys/ufs/ufs/ufs_ihash.c:119 #7 ffs_vget+0x74 sys/ufs/ffs/ffs_vfsops.c:1329 #8 ufs_lookup+0x14b7 sys/ufs/ufs/ufs_lookup.c:487 #9 VOP_LOOKUP+0x5b sys/kern/vfs_vops.c:90 #10 vfs_lookup+0x7a6 sys/kern/vfs_lookup.c:568 #11 namei+0x63c sys/kern/vfs_lookup.c:249 #12 dounlinkat+0x99 sys/kern/vfs_syscalls.c:1853 #13 syscall+0x4a4 mi_syscall sys/sys/syscall_mi.h:102 [inline] #13 syscall+0x4a4 sys/arch/amd64/amd64/trap.c:570 #14 Xsyscall+0x128 exclusive rrwlock inode r = 0 (0xfffffd80684fba30) #0 witness_lock+0x4c7 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4c7 sys/kern/subr_witness.c:1164 #1 rw_enter+0x453 sys/kern/kern_rwlock.c:311 #2 rrw_enter+0x88 sys/kern/kern_rwlock.c:462 #3 VOP_LOCK+0x4b sys/kern/vfs_vops.c:603 #4 vn_lock+0x81 sys/kern/vfs_vnops.c:575 #5 vfs_lookup+0xe6 sys/kern/vfs_lookup.c:419 #6 namei+0x63c sys/kern/vfs_lookup.c:249 #7 dounlinkat+0x99 sys/kern/vfs_syscalls.c:1853 #8 syscall+0x4a4 mi_syscall sys/sys/syscall_mi.h:102 [inline] #8 syscall+0x4a4 sys/arch/amd64/amd64/trap.c:570 #9 Xsyscall+0x128 Process 70921 (sshd) thread 0xffff800020e40600 (207973) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff8292e770) #0 witness_lock+0x4c7 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4c7 sys/kern/subr_witness.c:1164 #1 pageflttrap+0x7f sys/arch/amd64/amd64/trap.c:180 #2 kerntrap+0xec sys/arch/amd64/amd64/trap.c:302 #3 alltraps_kern_meltdown+0x7b #4 pool_cache_get+0x1b1 pool_cache_item_magic_check sys/kern/subr_pool.c:1770 [inline] #4 pool_cache_get+0x1b1 sys/kern/subr_pool.c:1884 #5 pool_get+0x91 sys/kern/subr_pool.c:572 #6 m_gethdr+0x4c sys/kern/uipc_mbuf.c:283 #7 tcp_output+0x14e6 #8 tcp_usrreq+0xa55 #9 sosend+0x671 sys/kern/uipc_socket.c:555 #10 dofilewritev+0x1b6 sys/kern/sys_generic.c:365 #11 sys_write+0x83 sys/kern/sys_generic.c:285 #12 syscall+0x4a4 mi_syscall sys/sys/syscall_mi.h:102 [inline] #12 syscall+0x4a4 sys/arch/amd64/amd64/trap.c:570 #13 Xsyscall+0x128 exclusive rwlock netlock r = 0 (0xffffffff82709b58) #0 witness_lock+0x4c7 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4c7 sys/kern/subr_witness.c:1164 #1 solock+0x5a sys/kern/uipc_socket2.c:282 #2 sosend+0x559 sys/kern/uipc_socket.c:543 #3 dofilewritev+0x1b6 sys/kern/sys_generic.c:365 #4 sys_write+0x83 sys/kern/sys_generic.c:285 #5 syscall+0x4a4 mi_syscall sys/sys/syscall_mi.h:102 [inline] #5 syscall+0x4a4 sys/arch/amd64/amd64/trap.c:570 #6 Xsyscall+0x128 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 9499 6408K 6731K 78643K 10790 0 pcb 13 8K 8K 78643K 50 0 rtable 113 3K 3K 78643K 241 0 ifaddr 60 12K 13K 78643K 70 0 counters 43 33K 34K 78643K 47 0 ioctlops 0 0K 4K 78643K 1481 0 iov 0 0K 16K 78643K 14 0 mount 1 1K 1K 78643K 1 0 vnodes 1221 77K 77K 78643K 1311 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 5K 78643K 4 0 VM map 2 1K 1K 78643K 2 0 sem 12 1K 1K 78643K 26 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1824 197K 290K 78643K 13058 0 file desc 4 9K 25K 78643K 158 0 proc 61 63K 95K 78643K 455 0 subproc 32 2K 2K 78643K 34 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 21 0 in_multi 40 2K 2K 78643K 59 0 ether_multi 1 0K 0K 78643K 8 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 61 281K 281K 78643K 61 0 exec 0 0K 1K 78643K 210 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 7 26K 26K 78643K 7 0 UVM amap 120 38K 39K 78643K 1433 0 UVM aobj 7 2K 2K 78643K 7 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 0K 78643K 24 0 NDP 9 0K 0K 78643K 15 0 temp 86 3857K 3921K 78643K 4597 0 kqueue 3 4K 8K 78643K 8 0 SYN cache 2 16K 16K 78643K 2 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle arp 64 9 0 1 1 0 1 1 0 8 0 plcache 128 20 0 0 1 0 1 1 0 8 0 rtpcb 80 21 0 19 1 0 1 1 0 8 0 rtentry 112 52 0 4 2 0 2 2 0 8 0 unpcb 120 117 0 107 1 0 1 1 0 8 0 syncache 264 4 0 4 2 2 0 1 0 8 0 tcpqe 32 216 0 216 1 1 0 1 0 8 0 tcpcb 544 54 0 48 1 0 1 1 0 8 0 inpcb 296 158 0 151 4 2 2 2 0 8 1 nd6 48 6 0 0 1 0 1 1 0 8 0 pkpcb 40 2 0 2 1 1 0 1 0 8 0 pfosfp 40 846 0 423 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfstitem 24 28 0 8 1 0 1 1 0 8 0 pfstkey 112 28 0 8 1 0 1 1 0 8 0 pfstate 328 28 0 8 3 0 3 3 0 8 0 pfrule 1360 21 0 16 2 1 1 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 209 0 2 13 0 13 13 0 8 0 art_table 32 210 0 2 2 0 2 2 0 8 0 art_node 16 51 0 7 1 0 1 1 0 8 0 sysvmsgpl 40 5 0 2 1 0 1 1 0 8 0 semupl 112 1 0 1 1 1 0 1 0 8 0 semapl 112 24 0 14 1 0 1 1 0 8 0 shmpl 112 4 0 0 1 0 1 1 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 1616 0 210 89 0 89 89 0 8 0 ffsino 272 1616 0 210 96 1 95 95 0 8 0 nchpl 144 2070 0 460 60 0 60 60 0 8 0 uvmvnodes 72 1712 0 0 32 0 32 32 0 8 0 vnodes 208 1712 0 0 91 0 91 91 0 8 0 namei 1024 5729 0 5729 4 3 1 1 0 8 1 percpumem 16 34 0 2 1 0 1 1 0 8 0 vcpupl 1984 2 0 0 1 0 1 1 0 8 0 vmpool 560 4 0 2 2 1 1 1 0 8 0 scxspl 192 5911 0 5910 7 6 1 6 0 8 0 plimitpl 152 21 0 13 1 0 1 1 0 8 0 sigapl 424 393 0 343 6 0 6 6 0 8 0 futexpl 56 1953 0 1953 3 2 1 1 0 8 1 knotepl 112 67 0 48 1 0 1 1 0 8 0 kqueuepl 144 34 0 32 1 0 1 1 0 8 0 pipepl 304 95 0 85 3 2 1 2 0 8 0 fdescpl 496 358 0 343 3 0 3 3 0 8 0 filepl 152 2167 0 2067 6 1 5 5 0 8 1 lockfpl 104 25 0 24 1 0 1 1 0 8 0 lockfspl 48 8 0 7 1 0 1 1 0 8 0 sessionpl 112 18 0 7 1 0 1 1 0 8 0 pgrppl 48 22 0 11 1 0 1 1 0 8 0 ucredpl 96 403 0 394 1 0 1 1 0 8 0 zombiepl 144 343 0 342 2 1 1 1 0 8 0 processpl 992 393 0 342 8 1 7 7 0 8 0 procpl 624 681 0 621 7 1 6 6 0 8 1 sockpl 400 301 0 282 5 2 3 3 0 8 1 mcl64k 65536 6 0 0 1 0 1 1 0 8 0 mcl16k 16384 1 0 0 1 0 1 1 0 8 0 mcl12k 12288 5 0 0 1 0 1 1 0 8 0 mcl9k 9216 4 0 0 1 0 1 1 0 8 0 mcl8k 8192 2 0 0 1 0 1 1 0 8 0 mcl4k 4096 6 0 0 1 0 1 1 0 8 0 mcl2k 2048 162 0 0 20 0 20 20 0 8 0 mtagpl 96 12 0 0 1 0 1 1 0 8 0 mbufpl 256 203 0 0 13 0 13 13 0 8 0 bufpl 280 3606 0 133 249 0 249 249 0 8 0 anonpl 16 52368 0 35672 85 15 70 82 0 124 2 amapchunkpl 152 1743 0 1594 9 2 7 9 0 158 0 amappl16 192 1782 0 880 56 9 47 55 0 8 1 amappl15 184 64 0 62 1 0 1 1 0 8 0 amappl14 176 24 0 20 1 0 1 1 0 8 0 amappl13 168 29 0 26 1 0 1 1 0 8 0 amappl12 160 8 0 8 1 1 0 1 0 8 0 amappl11 152 54 0 39 1 0 1 1 0 8 0 amappl10 144 80 0 73 1 0 1 1 0 8 0 amappl9 136 286 0 284 1 0 1 1 0 8 0 amappl8 128 341 0 313 1 0 1 1 0 8 0 amappl7 120 125 0 113 1 0 1 1 0 8 0 amappl6 112 25 0 21 2 1 1 1 0 8 0 amappl5 104 471 0 455 1 0 1 1 0 8 0 amappl4 96 486 0 455 1 0 1 1 0 8 0 amappl3 88 107 0 101 1 0 1 1 0 8 0 amappl2 80 1971 0 1910 2 0 2 2 0 8 0 amappl1 72 17665 0 17243 23 13 10 18 0 8 0 amappl 80 915 0 869 2 0 2 2 0 84 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 64 6 0 0 1 0 1 1 0 8 0 uaddrrnd 24 362 0 345 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 362 0 345 1 0 1 1 0 8 0 vmmpekpl 168 6778 0 6740 2 0 2 2 0 8 0 vmmpepl 168 51341 0 49383 125 28 97 116 0 357 4 vmsppl 368 361 0 345 2 0 2 2 0 8 0 pdppl 4096 731 0 692 8 2 6 6 0 8 0 pvpl 32 167090 0 147249 189 18 171 189 0 265 8 pmappl 232 361 0 345 3 1 2 2 0 8 0 extentpl 40 53 0 36 1 0 1 1 0 8 0 phpool 112 270 0 12 8 0 8 8 0 8 0 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+0x1a: addq $0x8,%rsp ddb{0}> trace x86_ipi_db(ffffffff82766ff0) at x86_ipi_db+0x1a sys/arch/amd64/amd64/db_interface.c:352 x86_ipi_handler() at x86_ipi_handler+0xc6 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 __sanitizer_cov_trace_cmp4(85b89,85b8a) at __sanitizer_cov_trace_cmp4+0x1e kd_curproc sys/dev/kcov.c:397 [inline] __sanitizer_cov_trace_cmp4(85b89,85b8a) at __sanitizer_cov_trace_cmp4+0x1e trace_cmp sys/dev/kcov.c:104 [inline] __sanitizer_cov_trace_cmp4(85b89,85b8a) at __sanitizer_cov_trace_cmp4+0x1e sys/dev/kcov.c:135 __mp_lock(ffffffff8292e568) at __mp_lock+0x139 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff8292e568) at __mp_lock+0x139 sys/kern/kern_lock.c:147 intr_handler(ffff800020d91020,ffff80000004a400) at intr_handler+0x5e sys/arch/amd64/amd64/intr.c:532 Xintr_ioapic_edge17_untramp() at Xintr_ioapic_edge17_untramp+0x19f acpicpu_idle() at acpicpu_idle+0x331 sys/dev/acpi/acpicpu.c:1187 sched_idle(ffffffff82766ff0) at sched_idle+0x3f7 sys/kern/kern_sched.c:178 end trace frame: 0x0, count: -9 ddb{0}> machine ddbcpu 1 Stopped at pool_cache_get+0x1b1: movq 0x10(%r14),%r13 ddb{1}> trace pool_cache_get(ffffffff828c79e0) at pool_cache_get+0x1b1 pool_cache_item_magic_check sys/kern/subr_pool.c:1770 [inline] pool_cache_get(ffffffff828c79e0) at pool_cache_get+0x1b1 sys/kern/subr_pool.c:1884 pool_get(ffffffff828c79e0,2) at pool_get+0x91 sys/kern/subr_pool.c:572 m_gethdr(2,2) at m_gethdr+0x4c sys/kern/uipc_mbuf.c:283 tcp_output(ffff800000ada320) at tcp_output+0x14e6 tcp_usrreq(fffffd806eadf968,9,fffffd806d1df600,0,0,ffff800020e40600) at tcp_usrreq+0xa55 sosend(fffffd806eadf968,0,ffff800020e044e8,0,0,80) at sosend+0x671 sys/kern/uipc_socket.c:555 dofilewritev(ffff800020e40600,4,ffff800020e044e8,0,ffff800020e045d0) at dofilewritev+0x1b6 sys/kern/sys_generic.c:365 sys_write(ffff800020e40600,ffff800020e04580,ffff800020e045d0) at sys_write+0x83 sys/kern/sys_generic.c:285 syscall(ffff800020e04650) at syscall+0x4a4 mi_syscall sys/sys/syscall_mi.h:102 [inline] syscall(ffff800020e04650) at syscall+0x4a4 sys/arch/amd64/amd64/trap.c:570 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffd8090, count: -10