uvm_fault(0xffffffff8354dee0, 0x7f837bd22c80, 0, 2) -> e kernel: page fault trap, code=2 Stopped at pmap_page_remove+0x45d: xchgq %rax,0(%r14,%rcx,1) TID PID UID PRFLAGS PFLAGS CPU COMMAND 358697 62430 0 0x14000 0x200 0 zerothread *129857 22424 0 0x14000 0x200 1 reaper pmap_page_remove(fffffd8008727360) at pmap_page_remove+0x45d _atomic_swap_64 machine/atomic.h:117 [inline] pmap_page_remove(fffffd8008727360) at pmap_page_remove+0x45d sys/arch/amd64/amd64/pmap.c:2014 uvm_anfree_list(fffffd80768ce9c0,0) at uvm_anfree_list+0xd6 amap_wipeout(fffffd806fc24750) at amap_wipeout+0x248 sys/uvm/uvm_amap.c:502 uvm_unmap_detach(ffff800029fe4f30,0) at uvm_unmap_detach+0x8a sys/uvm/uvm_map.c:1353 uvm_map_teardown(fffffd806bef36e0) at uvm_map_teardown+0x35e sys/uvm/uvm_map.c:2518 uvmspace_free(fffffd806bef36e0) at uvmspace_free+0xcd sys/uvm/uvm_map.c:3422 reaper(ffff800029fd8cb0) at reaper+0x246 sys/kern/kern_exit.c:477 end trace frame: 0x0, count: 8 https://www.openbsd.org/ddb.html describes the minimum info required in bug reports. Insufficient info makes it difficult to find and fix bugs. ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic *cpu1: uvm_fault(0xffffffff8354dee0, 0x7f837bd22c80, 0, 2) -> e ddb{1}> trace pmap_page_remove(fffffd8008727360) at pmap_page_remove+0x45d _atomic_swap_64 machine/atomic.h:117 [inline] pmap_page_remove(fffffd8008727360) at pmap_page_remove+0x45d sys/arch/amd64/amd64/pmap.c:2014 uvm_anfree_list(fffffd80768ce9c0,0) at uvm_anfree_list+0xd6 amap_wipeout(fffffd806fc24750) at amap_wipeout+0x248 sys/uvm/uvm_amap.c:502 uvm_unmap_detach(ffff800029fe4f30,0) at uvm_unmap_detach+0x8a sys/uvm/uvm_map.c:1353 uvm_map_teardown(fffffd806bef36e0) at uvm_map_teardown+0x35e sys/uvm/uvm_map.c:2518 uvmspace_free(fffffd806bef36e0) at uvmspace_free+0xcd sys/uvm/uvm_map.c:3422 reaper(ffff800029fd8cb0) at reaper+0x246 sys/kern/kern_exit.c:477 end trace frame: 0x0, count: -7 ddb{1}> show registers rdi 0 rsi 0 rbp 0xffff800029fe4e30 rbx 0xfffffd80558a7868 rdx 0 rcx 0x7f8000000000 rax 0 r8 0x6f73feb7000 r9 0 r10 0x47b428fe0f3c4198 r11 0xe4f95c824e364927 r12 0 r13 0x800000007f7c7000 r14 0x37bd22c80 r15 0xfffffd80087273c8 rip 0xffffffff829e9cdd pmap_page_remove+0x45d cs 0x8 rflags 0x10246 __ALIGN_SIZE+0xf246 rsp 0xffff800029fe4db0 ss 0x10 pmap_page_remove+0x45d: xchgq %rax,0(%r14,%rcx,1) ddb{1}> show proc PROC (reaper) tid=129857 pid=22424 tcnt=1 stat=onproc flags process=14000 proc=200 runpri=4, usrpri=83, slppri=4, nice=20 wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0 forw=0xffffffffffffffff, list=0xffff800029fd9448,0xffff800029fd82a0 process=0xffff800029fea448 user=0xffff800029fe0000, vmspace=0xffffffff8354dee0 estcpu=33, cpticks=0, pctcpu=21.55, user=0, sys=21970, intr=141 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 51682 211192 11046 0 3 0x10008a sigsusp sh 11046 6094 7341 0 3 0x82 wait syz-executor 81180 332356 7341 0 3 0x82 piperd syz-executor 92326 11132 7341 0 3 0x82 piperd syz-executor 41625 346310 7341 0 3 0x82 piperd syz-executor 25704 390483 7341 0 3 0x82 piperd syz-executor 32864 235765 7341 0 3 0x82 piperd syz-executor 57148 179436 7341 0 3 0x82 piperd syz-executor 59529 113105 1 0 3 0x100083 ttyin getty 86844 10349 0 0 3 0x14200 bored sosplice 7341 111064 15839 0 3 0x82 wait syz-executor 15839 39864 1432 0 3 0x10008a sigsusp ksh 1432 413271 9405 0 3 0x98 kqread sshd-session 9405 456715 54360 0 3 0x92 kqread sshd-session 54360 48100 1 0 3 0x88 kqread sshd 81911 373492 53422 74 3 0x1100092 bpf pflogd 53422 471845 1 0 3 0x80 sbwait pflogd 71867 161291 16217 73 3 0x1100010 biowait syslogd 16217 415507 1 0 3 0x100082 sbwait syslogd 68875 322498 1 0 3 0x100080 kqread resolvd 86276 257268 70860 77 3 0x100092 kqread dhcpleased 89949 327278 70860 77 3 0x100092 kqread dhcpleased 70860 81995 1 0 3 0x80 kqread dhcpleased 42902 222446 0 0 3 0x14200 bored smr 62430 358697 0 0 7 0x14200 zerothread 54053 80043 0 0 3 0x14200 aiodoned aiodoned 75621 467542 0 0 3 0x14200 syncer update 35663 431401 0 0 3 0x14200 cleaner cleaner *22424 129857 0 0 7 0x14200 reaper 83418 12130 0 0 3 0x14200 pgdaemon pagedaemon 14316 435561 0 0 3 0x14200 bored viomb 80135 144110 0 0 3 0x40014200 acpi0 acpi0 20999 405446 0 0 3 0x40014200 idle1 97428 290425 0 0 3 0x14200 bored softnet3 26529 304485 0 0 3 0x14200 bored softnet2 31929 7380 0 0 3 0x14200 bored softnet1 12525 161336 0 0 3 0x14200 bored softnet0 62349 369138 0 0 3 0x14200 bored systqmp 2188 464774 0 0 3 0x14200 bored systq 23272 101058 0 0 3 0x14200 tmoslp softclockmp 14957 119695 0 0 3 0x40014200 tmoslp softclock 67017 303963 0 0 3 0x40014200 idle0 1 25490 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{1}> show all locks CPU 1: exclusive mutex &pmap->pm_mtx r = 0 (0xfffffd806bd41bc0) #0 witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5bb sys/kern/subr_witness.c:1155 #1 mtx_enter_try+0x178 #2 mtx_enter+0x60 sys/kern/kern_lock.c:239 #3 pmap_page_remove+0xcd rcr3 machine/cpufunc.h:139 [inline] #3 pmap_page_remove+0xcd pmap_map_ptes sys/arch/amd64/amd64/pmap.c:437 [inline] #3 pmap_page_remove+0xcd sys/arch/amd64/amd64/pmap.c:1974 #4 uvm_anfree_list+0xd6 #5 amap_wipeout+0x248 sys/uvm/uvm_amap.c:502 #6 uvm_unmap_detach+0x8a sys/uvm/uvm_map.c:1353 #7 uvm_map_teardown+0x35e sys/uvm/uvm_map.c:2518 #8 uvmspace_free+0xcd sys/uvm/uvm_map.c:3422 #9 reaper+0x246 sys/kern/kern_exit.c:477 #10 proc_trampoline+0x10 Process 71867 (syslogd) thread 0xffff8000ffffd6d8 (161291) Process 22424 (reaper) thread 0xffff800029fd8cb0 (129857) ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10224 11211K 11539K 166960K 15489 0 pcb 17 18K 20K 166960K 885 0 rtable 172 7K 10K 166960K 3730 0 pf 35 17K 25K 166960K 362 0 ifaddr 36 7K 10K 166960K 518 0 ifgroup 52 2K 2K 166960K 560 0 sysctl 4 1K 1K 166960K 8 0 counters 62 36K 37K 166960K 346 0 ioctlops 0 0K 4K 166960K 1876 0 iov 0 0K 32K 166960K 519 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1588 100K 100K 166960K 5570 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 1K 13K 166960K 60 0 VM map 2 1K 1K 166960K 2 0 sem 12 0K 1K 166960K 244 0 dirhash 15 2K 3K 166960K 63 0 ACPI 1690 195K 286K 166960K 12468 0 file desc 10 33K 93K 166960K 5128 0 sigio 0 0K 0K 166960K 199 0 proc 72 91K 128K 166960K 3631 0 subproc 91 5K 7K 166960K 1417 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 624 0 in_multi 77 5K 7K 166960K 1331 0 ether_multi 1 0K 0K 166960K 34 0 mrt 1 0K 0K 166960K 10 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 247 1102K 1102K 166960K 247 0 exec 0 0K 1K 166960K 2147 0 pfkey data 0 0K 4K 166960K 7 0 tdb 3 0K 0K 166960K 3 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 179 71K 92K 166960K 43898 0 UVM aobj 131 8K 8K 166960K 145 0 pinsyscall 35 70K 104K 166960K 8824 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 1K 166960K 265 0 NDP 11 0K 2K 166960K 379 0 temp 82 6824K 7066K 166960K 217500 0 kqueue 13 20K 33K 166960K 629 0 SYN cache 2 16K 16K 166960K 2 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 24 0 0 1 0 1 1 0 8 0 rtpcb 120 510 0 507 2 1 1 2 0 8 0 rtentry 112 1333 0 1261 6 3 3 4 0 8 0 unpcb 144 3768 0 3751 45 44 1 9 0 8 0 syncache 336 16 0 16 7 7 0 1 0 8 0 tcpqe 32 4 0 4 3 3 0 1 0 8 0 tcpcb 808 1630 0 1626 43 41 2 8 0 8 1 arp 120 239 0 226 1 0 1 1 0 8 0 inpcb 336 5774 0 5767 81 79 2 13 0 8 1 nd6 136 361 0 343 1 0 1 1 0 8 0 pkpcb 40 22 0 22 10 10 0 1 0 8 0 kcovpl 48 109 0 102 1 0 1 1 0 8 0 ppxss 1168 33 0 33 11 11 0 1 0 8 0 pffrag 232 44 0 39 1 0 1 1 0 482 0 pffrnode 88 38 0 33 1 0 1 1 0 8 0 pffrent 40 70 0 65 1 0 1 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfrktable 1344 3 0 1 1 0 1 1 0 8 0 pfanchor 1288 3 0 0 1 0 1 1 0 8 0 pftag 88 1 0 0 1 0 1 1 0 8 0 pfstitem 24 499 0 409 1 0 1 1 0 8 0 pfstkey 128 499 0 409 4 0 4 4 0 8 0 pfstate 376 499 0 409 13 1 12 12 0 8 0 pfrule 1344 62 0 22 5 1 4 4 0 8 0 art_heap8 4096 4 0 0 4 0 4 4 0 8 0 art_heap4 256 5379 0 5014 46 16 30 32 0 8 0 art_table 32 5383 0 5014 5 1 4 5 0 8 0 art_node 16 1320 0 1257 1 0 1 1 0 8 0 sysvmsgpl 40 21 0 12 1 0 1 1 0 8 0 semapl 112 237 0 227 1 0 1 1 0 8 0 shmpl 112 142 0 14 4 0 4 4 0 8 0 dirhash 1024 52 0 33 3 0 3 3 0 8 0 dino2pl 256 8419 0 6721 107 0 107 107 0 8 0 ffsino 272 8419 0 6721 114 0 114 114 0 8 0 nchpl 144 13813 0 12006 68 0 68 68 0 8 0 uvmvnodes 80 7306 0 0 150 0 150 150 0 8 0 vnodes 216 7306 0 0 406 0 406 406 0 8 0 namei 1024 57943 0 57943 10 9 1 2 0 8 1 percpumem 16 187 0 142 1 0 1 1 0 8 0 pfiaddrpl 120 1 0 0 1 0 1 1 0 8 0 kstatmem 264 306 0 284 6 4 2 3 0 8 0 acpiwqpl 32 2 0 2 1 0 1 1 1 8 1 scsiplug 72 4 0 4 4 4 0 1 0 8 0 scxspl 216 97702 0 97701 27 26 1 8 1 8 0 plimitpl 152 1471 0 1454 1 0 1 1 0 8 0 sigapl 424 5277 0 5232 10 3 7 9 0 8 0 futexpl 64 56571 0 56571 9 9 0 1 0 8 0 knotepl 120 802 0 0 25 0 25 25 0 8 0 kqueuepl 216 1384 0 1375 22 21 1 5 0 8 0 pipepl 320 1070 0 1043 28 25 3 8 0 8 0 fdescpl 496 5229 0 5205 8 4 4 5 0 8 0 filepl 152 35376 0 35155 91 79 12 20 0 8 2 lockfpl 104 1925 0 1923 4 3 1 2 0 8 0 lockfspl 48 720 0 718 1 0 1 1 0 8 0 sessionpl 144 140 0 131 1 0 1 1 0 8 0 pgrppl 48 295 0 278 1 0 1 1 0 8 0 ucredpl 104 5795 0 5782 1 0 1 1 0 8 0 zombiepl 144 5234 0 5232 2 1 1 1 0 8 0 processpl 1160 5277 0 5232 6 1 5 6 0 8 0 procpl 648 11271 0 11226 9 3 6 8 0 8 0 srpgc 96 17 0 17 6 6 0 1 0 8 0 sosppl 168 28 0 28 11 11 0 1 0 8 0 sockpl 664 10252 0 10225 121 115 6 23 0 8 3 mcl64k 65536 18 0 0 3 0 3 3 0 8 0 mcl16k 16384 9 0 0 2 0 2 2 0 8 0 mcl12k 12288 3 0 0 1 0 1 1 0 8 0 mcl9k 9216 4 0 0 1 0 1 1 0 8 0 mcl8k 8192 16 0 0 2 0 2 2 0 8 0 mcl4k 4096 177 0 0 18 1 17 18 0 8 0 mcl2k2 2112 2 0 0 1 0 1 1 0 8 0 mcl2k 2048 98 0 0 7 2 5 5 0 8 0 mtagpl 96 14 0 0 1 0 1 1 0 8 0 mbufpl 256 8750 0 0 546 0 546 546 0 8 0 bufpl 280 24204 0 17770 523 10 513 523 0 8 0 anonpl 24 697499 0 693536 173 99 74 85 0 185 24 amapchunkpl 152 142553 0 142196 96 61 35 44 0 158 12 amappl16 200 13340 0 13321 159 150 9 26 0 8 5 amappl15 192 11 0 11 2 2 0 1 0 8 0 amappl14 184 336 0 324 1 0 1 1 0 8 0 amappl13 176 5 0 5 2 2 0 1 0 8 0 amappl12 168 7440 0 7415 4 2 2 3 0 8 0 amappl11 160 58 0 43 1 0 1 1 0 8 0 amappl10 152 32 0 32 1 1 0 1 0 8 0 amappl9 144 146 0 146 1 1 0 1 0 8 0 amappl8 136 39 0 36 1 0 1 1 0 8 0 amappl7 128 328 0 314 1 0 1 1 0 8 0 amappl6 120 1132 0 1128 1 0 1 1 0 8 0 amappl5 112 543 0 530 1 0 1 1 0 8 0 amappl4 104 671 0 651 1 0 1 1 0 8 0 amappl3 96 29014 0 28943 4 0 4 4 0 8 0 amappl2 88 2207 0 2134 2 0 2 2 0 8 0 amappl1 80 31393 0 30844 19 5 14 16 0 8 0 amappl 88 42392 0 42274 5 0 5 5 0 92 0 dma8192 8192 1 0 1 1 1 0 1 0 8 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 2 0 1 1 0 1 1 0 8 0 dma512 512 1 0 1 1 1 0 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 7 0 7 2 2 0 1 0 8 0 dma32 32 8 0 8 2 2 0 1 0 8 0 dma16 16 19 0 18 1 0 1 1 0 8 0 aobjpl 72 144 0 14 3 0 3 3 0 8 0 uaddrrnd 24 5229 0 5204 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 5229 0 5204 1 0 1 1 0 8 0 vmmpekpl 168 41830 0 41772 4 1 3 4 0 8 0 vmmpepl 168 319509 0 317995 177 84 93 105 0 357 9 vmsppl 440 5228 0 5203 6 2 4 5 0 8 0 rwobjpl 56 90908 0 82589 136 14 122 123 0 8 0 pdppl 4096 10465 0 10406 232 161 71 85 0 8 12 pvpl 32 45534 0 0 364 0 364 364 0 265 0 pmappl 248 5228 0 5203 3 0 3 3 0 8 0 extentpl 40 55 0 38 1 0 1 1 0 8 0 phpool 112 1241 0 331 27 0 27 27 0 8 0 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+0x27: addq $0x8,%rsp x86_ipi_db(ffffffff83420ff0) at x86_ipi_db+0x27 sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xd9 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 __mp_lock(ffffffff8361b028) at __mp_lock+0x192 __mp_lock_spin sys/kern/kern_lock.c:113 [inline] __mp_lock(ffffffff8361b028) at __mp_lock+0x192 sys/kern/kern_lock.c:144 intr_handler(ffff80002a003ba0,ffff800000079f80) at intr_handler+0xe1 sys/arch/amd64/amd64/intr.c:553 Xintr_ioapic_edge23_untramp() at Xintr_ioapic_edge23_untramp+0x18f uvm_pmr_pnaddr(fffffd8004313000,fffffd800816e680,ffff80002a003cc0,ffff80002a003cc8) at uvm_pmr_pnaddr+0xdd sys/uvm/uvm_pmemrange.c:315 uvm_pmr_insert_addr(fffffd8004313000,fffffd800816e680,0) at uvm_pmr_insert_addr+0x91 sys/uvm/uvm_pmemrange.c:416 uvm_pmr_remove_1strange(ffff80002a003de0,0,0,0) at uvm_pmr_remove_1strange+0x869 uvm_pmr_insert sys/uvm/uvm_pmemrange.c:479 [inline] uvm_pmr_remove_1strange(ffff80002a003de0,0,0,0) at uvm_pmr_remove_1strange+0x869 sys/uvm/uvm_pmemrange.c:697 uvm_pagezero_thread(ffff800029fd87a0) at uvm_pagezero_thread+0x22e sys/uvm/uvm_pmemrange.c:2208 end trace frame: 0x0, count: 5 ddb{0}> trace x86_ipi_db(ffffffff83420ff0) at x86_ipi_db+0x27 sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xd9 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 __mp_lock(ffffffff8361b028) at __mp_lock+0x192 __mp_lock_spin sys/kern/kern_lock.c:113 [inline] __mp_lock(ffffffff8361b028) at __mp_lock+0x192 sys/kern/kern_lock.c:144 intr_handler(ffff80002a003ba0,ffff800000079f80) at intr_handler+0xe1 sys/arch/amd64/amd64/intr.c:553 Xintr_ioapic_edge23_untramp() at Xintr_ioapic_edge23_untramp+0x18f uvm_pmr_pnaddr(fffffd8004313000,fffffd800816e680,ffff80002a003cc0,ffff80002a003cc8) at uvm_pmr_pnaddr+0xdd sys/uvm/uvm_pmemrange.c:315 uvm_pmr_insert_addr(fffffd8004313000,fffffd800816e680,0) at uvm_pmr_insert_addr+0x91 sys/uvm/uvm_pmemrange.c:416 uvm_pmr_remove_1strange(ffff80002a003de0,0,0,0) at uvm_pmr_remove_1strange+0x869 uvm_pmr_insert sys/uvm/uvm_pmemrange.c:479 [inline] uvm_pmr_remove_1strange(ffff80002a003de0,0,0,0) at uvm_pmr_remove_1strange+0x869 sys/uvm/uvm_pmemrange.c:697 uvm_pagezero_thread(ffff800029fd87a0) at uvm_pagezero_thread+0x22e sys/uvm/uvm_pmemrange.c:2208 end trace frame: 0x0, count: -10 ddb{0}> machine ddbcpu 1 Stopped at pmap_page_remove+0x45d: xchgq %rax,0(%r14,%rcx,1) pmap_page_remove(fffffd8008727360) at pmap_page_remove+0x45d _atomic_swap_64 machine/atomic.h:117 [inline] pmap_page_remove(fffffd8008727360) at pmap_page_remove+0x45d sys/arch/amd64/amd64/pmap.c:2014 uvm_anfree_list(fffffd80768ce9c0,0) at uvm_anfree_list+0xd6 amap_wipeout(fffffd806fc24750) at amap_wipeout+0x248 sys/uvm/uvm_amap.c:502 uvm_unmap_detach(ffff800029fe4f30,0) at uvm_unmap_detach+0x8a sys/uvm/uvm_map.c:1353 uvm_map_teardown(fffffd806bef36e0) at uvm_map_teardown+0x35e sys/uvm/uvm_map.c:2518 uvmspace_free(fffffd806bef36e0) at uvmspace_free+0xcd sys/uvm/uvm_map.c:3422 reaper(ffff800029fd8cb0) at reaper+0x246 sys/kern/kern_exit.c:477 end trace frame: 0x0, count: 8 ddb{1}> trace pmap_page_remove(fffffd8008727360) at pmap_page_remove+0x45d _atomic_swap_64 machine/atomic.h:117 [inline] pmap_page_remove(fffffd8008727360) at pmap_page_remove+0x45d sys/arch/amd64/amd64/pmap.c:2014 uvm_anfree_list(fffffd80768ce9c0,0) at uvm_anfree_list+0xd6 amap_wipeout(fffffd806fc24750) at amap_wipeout+0x248 sys/uvm/uvm_amap.c:502 uvm_unmap_detach(ffff800029fe4f30,0) at uvm_unmap_detach+0x8a sys/uvm/uvm_map.c:1353 uvm_map_teardown(fffffd806bef36e0) at uvm_map_teardown+0x35e sys/uvm/uvm_map.c:2518 uvmspace_free(fffffd806bef36e0) at uvmspace_free+0xcd sys/uvm/uvm_map.c:3422 reaper(ffff800029fd8cb0) at reaper+0x246 sys/kern/kern_exit.c:477 end trace frame: 0x0, count: -7