witness: lock order reversal: 1st 0xffffffff8290fe50 netlock (netlock) 2nd 0xfffffd8071f665f0 vmmaplk (&map->lock) lock order "&map->lock"(rwlock) -> "netlock"(rwlock) first seen at: #0 rw_enter_write+0x5b sys/kern/kern_rwlock.c:128 #1 udp_sysctl+0x8a sys/netinet/udp_usrreq.c:1269 #2 sys_sysctl+0x209 sys/kern/kern_sysctl.c:249 #3 syscall+0x489 mi_syscall sys/sys/syscall_mi.h:102 [inline] #3 syscall+0x489 sys/arch/amd64/amd64/trap.c:585 #4 Xsyscall+0x128 lock order "netlock"(rwlock) -> "&map->lock"(rwlock) first seen at: #0 rw_enter_read+0x66 sys/kern/kern_rwlock.c:112 #1 uvmfault_lookup+0xd9 sys/uvm/uvm_fault.c:1758 #2 uvm_fault_check+0x3a sys/uvm/uvm_fault.c:674 #3 uvm_fault+0x102 sys/uvm/uvm_fault.c:602 #4 kpageflttrap+0x209 #5 kerntrap+0xef sys/arch/amd64/amd64/trap.c:318 #6 alltraps_kern_meltdown+0x7b #7 copyout+0x53 #8 ifioctl_get+0x2dd #9 soo_ioctl+0x26c #10 sys_ioctl+0x4a2 #11 syscall+0x489 mi_syscall sys/sys/syscall_mi.h:102 [inline] #11 syscall+0x489 sys/arch/amd64/amd64/trap.c:585 #12 Xsyscall+0x128 Stopped at db_enter+0x18: addq $0x8,%rsp ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic the kernel did not panic ddb{1}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:440 witness_checkorder(fffffd8071f665f0,1,0) at witness_checkorder+0x10b7 witness_debugger sys/kern/subr_witness.c:2502 [inline] witness_checkorder(fffffd8071f665f0,1,0) at witness_checkorder+0x10b7 sys/kern/subr_witness.c:1105 rw_enter_read(fffffd8071f665e0) at rw_enter_read+0x66 sys/kern/kern_rwlock.c:112 uvmfault_lookup(ffff800029505d00,0) at uvmfault_lookup+0xd9 sys/uvm/uvm_fault.c:1758 uvm_fault_check(ffff800029505d00,ffff800029505d38,ffff800029505d60) at uvm_fault_check+0x3a sys/uvm/uvm_fault.c:674 uvm_fault(fffffd8071f665d8,20000000,0,2) at uvm_fault+0x102 sys/uvm/uvm_fault.c:602 kpageflttrap(ffff800029505e90,20000300) at kpageflttrap+0x209 kerntrap(ffff800029505e90) at kerntrap+0xef sys/arch/amd64/amd64/trap.c:318 alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b copyout() at copyout+0x53 ifioctl_get(c0106924,ffff800029506140) at ifioctl_get+0x2dd soo_ioctl(fffffd80676b1398,c0106924,ffff800029506140,ffff80002af4b7a8) at soo_ioctl+0x26c sys_ioctl(ffff80002af4b7a8,ffff800029506258,ffff8000295062b0) at sys_ioctl+0x4a2 syscall(ffff800029506320) at syscall+0x489 mi_syscall sys/sys/syscall_mi.h:102 [inline] syscall(ffff800029506320) at syscall+0x489 sys/arch/amd64/amd64/trap.c:585 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x9a7d3f26690, count: -15 ddb{1}> show registers rdi 0x3 rsi 0xffffffff8294abb0 __sancov_gen_cov_switch_values.134 rbp 0xffff8000295059f0 rbx 0x3 rdx 0 rcx 0 rax 0xffff80002af4b7a8 r8 0xffff800029505960 r9 0x8080808080808080 r10 0xea2311b10bf7e914 r11 0x4c63481144736c19 r12 0xffffffff82ab0010 w_lodata+0x526a0 r13 0 r14 0xffffffff82aa2af0 w_lodata+0x45180 r15 0xfffffd8002f62d80 rip 0xffffffff8138d658 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff8000295059e0 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb{1}> show proc PROC (syz-executor.2) pid=283854 stat=onproc flags process=10 proc=4000000 pri=32, usrpri=83, nice=20 forw=0xffffffffffffffff, list=0xffff80002af4b508,0xffff80002af4bcf8 process=0xffff8000ffff0018 user=0xffff800029501000, vmspace=0xfffffd8071f665d8 estcpu=36, cpticks=0, pctcpu=0.0 user=0, sys=0, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 24084 217558 80545 32767 7 0x10 syz-executor.1 62918 344634 64985 32767 2 0x10 syz-executor.2 *62918 283854 64985 32767 7 0x4000010 syz-executor.2 4977 321057 6238 32767 2 0x10 syz-executor.0 4977 68814 6238 32767 3 0x4000090 fsleep syz-executor.0 4977 27714 6238 32767 3 0x4000090 fsleep syz-executor.0 10527 229164 61635 32767 2 0x10 syz-executor.3 10527 489973 61635 32767 2 0x4000010 syz-executor.3 50334 226088 9604 32767 2 0x10 syz-executor.4 50334 511388 9604 32767 3 0x4000090 fsleep syz-executor.4 89851 438581 92311 32767 3 0x90 nanoslp syz-executor.5 89851 95698 92311 32767 3 0x4000090 ttyin syz-executor.5 89851 152867 92311 32767 3 0x4000090 fsleep syz-executor.5 92311 377278 81708 32767 3 0x90 nanoslp syz-executor.5 81708 464658 12317 0 3 0x82 wait syz-executor.5 62248 115222 3524 32767 3 0x90 nanoslp syz-executor.6 3524 318619 12317 0 3 0x82 wait syz-executor.6 67119 444305 0 0 3 0x14200 bored sosplice 89346 315043 36651 32767 2 0x10 syz-executor.7 36651 510170 12317 0 3 0x82 wait syz-executor.7 61635 464596 26100 32767 3 0x90 nanoslp syz-executor.3 9604 306773 79573 32767 3 0x90 nanoslp syz-executor.4 26100 360766 12317 0 3 0x82 wait syz-executor.3 79573 182872 12317 0 3 0x82 wait syz-executor.4 80545 496358 63968 32767 3 0x90 nanoslp syz-executor.1 64985 366715 44593 32767 3 0x90 nanoslp syz-executor.2 44593 16821 12317 0 3 0x82 wait syz-executor.2 63968 48193 12317 0 3 0x82 wait syz-executor.1 6238 61783 41254 32767 3 0x90 nanoslp syz-executor.0 41254 65676 12317 0 3 0x82 wait syz-executor.0 12317 287499 95340 0 3 0x82 thrsleep syz-fuzzer 12317 219687 95340 0 3 0x4000082 nanoslp syz-fuzzer 12317 58984 95340 0 3 0x4000082 kqread syz-fuzzer 12317 148424 95340 0 3 0x4000082 thrsleep syz-fuzzer 12317 358698 95340 0 3 0x4000082 thrsleep syz-fuzzer 12317 239662 95340 0 3 0x4000082 thrsleep syz-fuzzer 12317 198701 95340 0 3 0x4000082 thrsleep syz-fuzzer 12317 505984 95340 0 3 0x4000082 thrsleep syz-fuzzer 95340 177374 33298 0 3 0x10008a sigsusp ksh 33298 12358 72507 0 3 0x9a kqread sshd 59387 416654 1 0 3 0x100083 ttyin getty 72507 304204 1 0 3 0x88 kqread sshd 51298 131487 24738 73 3 0x1100090 kqread syslogd 24738 109208 1 0 3 0x100082 netio syslogd 67368 382762 1 0 3 0x100080 kqread resolvd 53740 37392 82795 77 3 0x100092 kqread dhcpleased 20644 149227 82795 77 3 0x100092 kqread dhcpleased 82795 299920 1 0 3 0x80 kqread dhcpleased 73876 237027 0 0 3 0x14200 bored smr 87779 141209 0 0 2 0x14200 zerothread 71834 111318 0 0 3 0x14200 aiodoned aiodoned 34894 452144 0 0 3 0x14200 syncer update 72995 360325 0 0 3 0x14200 cleaner cleaner 88840 354963 0 0 3 0x14200 reaper reaper 25022 262403 0 0 3 0x14200 pgdaemon pagedaemon 34936 19633 0 0 3 0x14200 bored viomb 63465 461373 0 0 3 0x40014200 acpi0 acpi0 70196 307501 0 0 3 0x40014200 idle1 13690 317821 0 0 3 0x14200 bored softnet 79458 246483 0 0 3 0x14200 bored systqmp 88620 273465 0 0 3 0x14200 bored systq 60177 63029 0 0 3 0x40014200 bored softclock 12068 411239 0 0 3 0x40014200 idle0 1 245975 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{1}> show all locks Process 62918 (syz-executor.2) thread 0xffff80002af4b7a8 (283854) shared rwlock netlock r = 0 (0xffffffff8290fe50) #0 witness_lock+0x44d #1 ifioctl_get+0x2d5 sys/net/if.c:2319 #2 soo_ioctl+0x26c #3 sys_ioctl+0x4a2 #4 syscall+0x489 mi_syscall sys/sys/syscall_mi.h:102 [inline] #4 syscall+0x489 sys/arch/amd64/amd64/trap.c:585 #5 Xsyscall+0x128 exclusive kernel_lock &kernel_lock r = 0 (0xffffffff829de988) #0 witness_lock+0x44d #1 soo_ioctl+0x25a sys/kern/sys_socket.c:136 #2 sys_ioctl+0x4a2 #3 syscall+0x489 mi_syscall sys/sys/syscall_mi.h:102 [inline] #3 syscall+0x489 sys/arch/amd64/amd64/trap.c:585 #4 Xsyscall+0x128 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10194 6410K 6419K 78643K 11303 0 pcb 13 12K 14K 78643K 17 0 rtable 260 7K 7K 78643K 774 0 ifaddr 81 17K 17K 78643K 125 0 sysctl 3 1K 1K 78643K 3 0 counters 56 35K 35K 78643K 68 0 ioctlops 0 0K 2K 78643K 60 0 iov 1 16K 16K 78643K 591 0 mount 1 1K 1K 78643K 1 0 log 0 0K 0K 78643K 4 0 vnodes 1271 79K 79K 78643K 1908 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 5K 78643K 41 0 VM map 2 1K 1K 78643K 2 0 sem 12 0K 0K 78643K 1233 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1697 195K 286K 78643K 12548 0 file desc 24 89K 121K 78643K 6512 0 sigio 2 0K 0K 78643K 7745 0 proc 56 74K 111K 78643K 1842 0 subproc 104 6K 6K 78643K 182 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 438 0 in_multi 99 6K 6K 78643K 202 0 ether_multi 1 0K 0K 78643K 22 0 mrt 2 0K 0K 78643K 2 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 181 811K 811K 78643K 181 0 exec 0 0K 2K 78643K 1196 0 tdb 3 0K 0K 78643K 3 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 7 26K 26K 78643K 7 0 UVM amap 383 88K 100K 78643K 84360 0 UVM aobj 131 8K 8K 78643K 131 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 0K 78643K 133 0 NDP 11 0K 2K 78643K 45 0 temp 125 4707K 4771K 78643K 18619 0 kqueue 12 18K 26K 78643K 459 0 SYN cache 2 16K 16K 78643K 2 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 22 0 0 1 0 1 1 0 8 0 rtpcb 120 247 0 244 3 2 1 2 0 8 0 rtentry 112 198 0 77 4 0 4 4 0 8 0 unpcb 136 16204 0 16186 45 37 8 12 0 8 6 syncache 296 46 0 46 11 11 0 1 0 8 0 tcpqe 32 9 0 9 4 4 0 1 0 8 0 tcpcb 736 2349 0 2342 43 41 2 14 0 8 1 arp 120 31 0 13 1 0 1 1 0 8 0 ipq 40 4 0 4 2 2 0 1 0 8 0 ipqe 40 12 0 12 2 2 0 1 0 8 0 inpcb 304 4481 0 4471 46 44 2 11 0 8 1 ip6q 72 40 0 40 3 3 0 1 0 8 0 ip6af 40 80 0 80 3 3 0 1 0 8 0 nd6 48 62 0 27 1 0 1 1 0 8 0 kcovpl 48 14 0 6 1 0 1 1 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 836 0 310 36 3 33 33 0 8 0 art_table 32 837 0 310 5 0 5 5 0 8 0 art_node 16 197 0 86 1 0 1 1 0 8 0 sysvmsgpl 40 36 0 24 1 0 1 1 0 8 0 semapl 112 1229 0 1219 1 0 1 1 0 8 0 shmpl 112 128 0 0 4 0 4 4 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 9357 0 7916 91 0 91 91 0 8 0 ffsino 272 9357 0 7916 97 0 97 97 0 8 0 nchpl 144 18234 0 16608 62 0 62 62 0 8 0 uvmvnodes 80 5926 0 0 121 0 121 121 0 8 0 vnodes 224 5926 0 0 349 0 349 349 0 8 0 namei 1024 60505 0 60505 4 3 1 2 0 8 1 percpumem 16 46 0 6 1 0 1 1 0 8 0 scxspl 216 57363 0 57363 11 10 1 8 0 8 1 plimitpl 152 777 0 754 3 2 1 2 0 8 0 sigapl 424 6787 0 6736 7 1 6 7 0 8 0 futexpl 64 53409 0 53405 2 1 1 1 0 8 0 knotepl 120 352 0 0 10 0 10 10 0 8 0 kqueuepl 216 1442 0 1434 25 24 1 5 0 8 0 pipepl 336 1123 0 1095 27 21 6 8 0 8 3 fdescpl 496 6772 0 6737 7 2 5 6 0 8 0 filepl 152 44196 0 43950 68 51 17 24 0 8 6 lockfpl 104 1424 0 1422 1 0 1 1 0 8 0 lockfspl 48 396 0 394 1 0 1 1 0 8 0 sessionpl 144 29 0 13 1 0 1 1 0 8 0 pgrppl 48 106 0 90 1 0 1 1 0 8 0 ucredpl 96 4069 0 4051 1 0 1 1 0 8 0 zombiepl 144 6737 0 6736 1 0 1 1 0 8 0 processpl 1064 6787 0 6736 5 1 4 4 0 8 0 procpl 672 18729 0 18664 14 7 7 8 0 8 1 sosppl 168 73 0 73 6 6 0 1 0 8 0 sockpl 480 21036 0 21005 353 341 12 42 0 8 8 mcl64k 65536 43 0 0 4 1 3 3 0 8 0 mcl16k 16384 17 0 0 3 0 3 3 0 8 0 mcl12k 12288 33 0 0 2 0 2 2 0 8 0 mcl9k 9216 9 0 0 1 0 1 1 0 8 0 mcl8k 8192 25 0 0 4 1 3 3 0 8 0 mcl4k 4096 17 0 0 3 0 3 3 0 8 0 mcl2k2 2112 5 0 0 1 0 1 1 0 8 0 mcl2k 2048 344 0 0 41 9 32 41 0 8 0 mtagpl 96 3 0 0 1 0 1 1 0 8 0 mbufpl 256 475 0 0 24 0 24 24 0 8 0 bufpl 288 11789 0 5454 453 0 453 453 0 8 0 anonpl 24 1884635 0 1870266 219 107 112 140 0 186 11 amapchunkpl 152 215406 0 214607 79 45 34 49 0 158 2 amappl16 200 18060 0 17688 130 104 26 48 0 8 6 amappl15 192 2614 0 2604 1 0 1 1 0 8 0 amappl14 184 1101 0 1099 1 0 1 1 0 8 0 amappl13 176 127 0 123 1 0 1 1 0 8 0 amappl12 168 33 0 25 1 0 1 1 0 8 0 amappl11 160 885 0 872 1 0 1 1 0 8 0 amappl10 152 35 0 31 1 0 1 1 0 8 0 amappl9 144 1213 0 1209 1 0 1 1 0 8 0 amappl8 136 2781 0 2680 4 0 4 4 0 8 0 amappl7 128 2056 0 2042 1 0 1 1 0 8 0 amappl6 120 973 0 942 2 1 1 2 0 8 0 amappl5 112 5981 0 5957 1 0 1 1 0 8 0 amappl4 104 2672 0 2639 2 0 2 2 0 8 0 amappl3 96 531 0 519 1 0 1 1 0 8 0 amappl2 88 910 0 864 3 1 2 3 0 8 0 amappl1 80 122592 0 121952 25 11 14 18 0 8 0 amappl 88 83689 0 83417 7 0 7 7 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 130 0 0 3 0 3 3 0 8 0 uaddrrnd 24 6772 0 6737 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 6772 0 6737 1 0 1 1 0 8 0 vmmpekpl 168 52355 0 52315 3 0 3 3 0 8 0 vmmpepl 168 611913 0 609238 203 72 131 138 0 357 9 vmsppl 368 6771 0 6737 4 0 4 4 0 8 0 rwobjpl 56 153703 0 146286 117 9 108 109 0 8 1 pdppl 4096 13551 0 13474 201 120 81 93 0 8 4 pvpl 32 3184768 0 3164905 405 210 195 246 0 265 25 pmappl 248 6771 0 6737 4 1 3 3 0 8 0 extentpl 40 58 0 38 1 0 1 1 0 8 0 phpool 112 1042 0 201 25 0 25 25 0 8 0 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+0x1a: addq $0x8,%rsp ddb{0}> trace x86_ipi_db(ffffffff829baff0) at x86_ipi_db+0x1a sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 __mp_lock(ffffffff829de780) at __mp_lock+0x122 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff829de780) at __mp_lock+0x122 sys/kern/kern_lock.c:147 intr_handler(ffff80002e385450,ffff800000077500) at intr_handler+0x5e sys/arch/amd64/amd64/intr.c:532 Xintr_ioapic_edge16_untramp() at Xintr_ioapic_edge16_untramp+0x18f __mp_lock(ffffffff829de780) at __mp_lock+0x122 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff829de780) at __mp_lock+0x122 sys/kern/kern_lock.c:147 softintr_dispatch(0) at softintr_dispatch+0x4e sys/arch/amd64/amd64/softintr.c:88 Xsoftclock() at Xsoftclock+0x1f __sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc+0x2f kd_curproc sys/dev/kcov.c:578 [inline] __sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc+0x2f sys/dev/kcov.c:148 __mp_lock(ffffffff829de780) at __mp_lock+0x133 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff829de780) at __mp_lock+0x133 sys/kern/kern_lock.c:147 syscall(ffff80002e385790) at syscall+0x3ef mi_syscall sys/sys/syscall_mi.h:93 [inline] syscall(ffff80002e385790) at syscall+0x3ef sys/arch/amd64/amd64/trap.c:585 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffec3c0, count: -13 ddb{0}> machine ddbcpu 1 Stopped at db_enter+0x18: addq $0x8,%rsp ddb{1}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:440 witness_checkorder(fffffd8071f665f0,1,0) at witness_checkorder+0x10b7 witness_debugger sys/kern/subr_witness.c:2502 [inline] witness_checkorder(fffffd8071f665f0,1,0) at witness_checkorder+0x10b7 sys/kern/subr_witness.c:1105 rw_enter_read(fffffd8071f665e0) at rw_enter_read+0x66 sys/kern/kern_rwlock.c:112 uvmfault_lookup(ffff800029505d00,0) at uvmfault_lookup+0xd9 sys/uvm/uvm_fault.c:1758 uvm_fault_check(ffff800029505d00,ffff800029505d38,ffff800029505d60) at uvm_fault_check+0x3a sys/uvm/uvm_fault.c:674 uvm_fault(fffffd8071f665d8,20000000,0,2) at uvm_fault+0x102 sys/uvm/uvm_fault.c:602 kpageflttrap(ffff800029505e90,20000300) at kpageflttrap+0x209 kerntrap(ffff800029505e90) at kerntrap+0xef sys/arch/amd64/amd64/trap.c:318 alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b copyout() at copyout+0x53 ifioctl_get(c0106924,ffff800029506140) at ifioctl_get+0x2dd soo_ioctl(fffffd80676b1398,c0106924,ffff800029506140,ffff80002af4b7a8) at soo_ioctl+0x26c sys_ioctl(ffff80002af4b7a8,ffff800029506258,ffff8000295062b0) at sys_ioctl+0x4a2 syscall(ffff800029506320) at syscall+0x489 mi_syscall sys/sys/syscall_mi.h:102 [inline] syscall(ffff800029506320) at syscall+0x489 sys/arch/amd64/amd64/trap.c:585 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x9a7d3f26690, count: -15