witness: lock order reversal: 1st 0xffffffff8290fe50 netlock (netlock) 2nd 0xfffffd806777e030 vmmaplk (&map->lock) lock order "&map->lock"(rwlock) -> "netlock"(rwlock) first seen at: #0 rw_enter_write+0x5b sys/kern/kern_rwlock.c:128 #1 udp_sysctl+0x8a sys/netinet/udp_usrreq.c:1269 #2 sys_sysctl+0x209 sys/kern/kern_sysctl.c:249 #3 syscall+0x489 mi_syscall sys/sys/syscall_mi.h:102 [inline] #3 syscall+0x489 sys/arch/amd64/amd64/trap.c:585 #4 Xsyscall+0x128 lock order "netlock"(rwlock) -> "&map->lock"(rwlock) first seen at: #0 rw_enter_read+0x66 sys/kern/kern_rwlock.c:112 #1 uvmfault_lookup+0xd9 sys/uvm/uvm_fault.c:1758 #2 uvm_fault_check+0x3a sys/uvm/uvm_fault.c:674 #3 uvm_fault+0x102 sys/uvm/uvm_fault.c:602 #4 kpageflttrap+0x209 #5 kerntrap+0xef sys/arch/amd64/amd64/trap.c:318 #6 alltraps_kern_meltdown+0x7b #7 copyout+0x53 #8 ifioctl_get+0x2dd #9 soo_ioctl+0x26c #10 sys_ioctl+0x4a2 #11 syscall+0x489 mi_syscall sys/sys/syscall_mi.h:102 [inline] #11 syscall+0x489 sys/arch/amd64/amd64/trap.c:585 #12 Xsyscall+0x128 Stopped at db_enter+0x18: addq $0x8,%rsp ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic the kernel did not panic ddb{0}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:440 witness_checkorder(fffffd806777e030,1,0) at witness_checkorder+0x10b7 witness_debugger sys/kern/subr_witness.c:2502 [inline] witness_checkorder(fffffd806777e030,1,0) at witness_checkorder+0x10b7 sys/kern/subr_witness.c:1105 rw_enter_read(fffffd806777e020) at rw_enter_read+0x66 sys/kern/kern_rwlock.c:112 uvmfault_lookup(ffff800021201ad0,0) at uvmfault_lookup+0xd9 sys/uvm/uvm_fault.c:1758 uvm_fault_check(ffff800021201ad0,ffff800021201b08,ffff800021201b30) at uvm_fault_check+0x3a sys/uvm/uvm_fault.c:674 uvm_fault(fffffd806777e018,20000000,0,2) at uvm_fault+0x102 sys/uvm/uvm_fault.c:602 kpageflttrap(ffff800021201c60,20000300) at kpageflttrap+0x209 kerntrap(ffff800021201c60) at kerntrap+0xef sys/arch/amd64/amd64/trap.c:318 alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b copyout() at copyout+0x53 ifioctl_get(c0106924,ffff800021201f10) at ifioctl_get+0x2dd soo_ioctl(fffffd80087374e0,c0106924,ffff800021201f10,ffff800022c5fa48) at soo_ioctl+0x26c sys_ioctl(ffff800022c5fa48,ffff800021202028,ffff800021202080) at sys_ioctl+0x4a2 syscall(ffff8000212020f0) at syscall+0x489 mi_syscall sys/sys/syscall_mi.h:102 [inline] syscall(ffff8000212020f0) at syscall+0x489 sys/arch/amd64/amd64/trap.c:585 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xcb12ae5c840, count: -15 ddb{0}> show registers rdi 0x3 rsi 0xffffffff8294abb0 __sancov_gen_cov_switch_values.134 rbp 0xffff8000212017c0 rbx 0x3 rdx 0 rcx 0 rax 0xffff800022c5fa48 r8 0xffff800021201730 r9 0x8080808080808080 r10 0x93625ce3e8045b5b r11 0x6223bcf8c7a3bfc1 r12 0xffffffff82ab0170 w_lodata+0x52800 r13 0 r14 0xffffffff82aa3ac0 w_lodata+0x46150 r15 0xfffffd8002f62d80 rip 0xffffffff8138d658 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff8000212017b0 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb{0}> show proc PROC (syz-executor.6) pid=397463 stat=onproc flags process=10 proc=4000000 pri=32, usrpri=82, nice=20 forw=0xffffffffffffffff, list=0xffff8000ffff4fc0,0xffff800022c5ed38 process=0xffff8000fffee9b0 user=0xffff8000211fd000, vmspace=0xfffffd806777e018 estcpu=36, cpticks=0, pctcpu=0.0 user=0, sys=0, intr=0 ddb{0}> ps PID TID PPID UID S FLAGS WAIT COMMAND 43302 295928 24894 32767 2 0x10 syz-executor.1 43302 501117 24894 32767 7 0x4000010 syz-executor.1 58508 437363 33988 32767 2 0x10 syz-executor.7 89996 96266 14824 32767 2 0x10 syz-executor.0 89996 55433 14824 32767 2 0x4000010 syz-executor.0 17465 275347 97842 32767 2 0x10 syz-executor.6 *17465 397463 97842 32767 7 0x4000010 syz-executor.6 43973 228723 72570 32767 2 0x10 syz-executor.2 43973 402899 72570 32767 3 0x4000090 fsleep syz-executor.2 53854 284556 12131 32767 2 0x10 syz-executor.3 53854 262404 12131 32767 3 0x4000090 fsleep syz-executor.3 53854 481460 12131 32767 3 0x4000090 fsleep syz-executor.3 14824 342896 89367 32767 3 0x90 nanoslp syz-executor.0 89367 232724 37905 0 3 0x82 wait syz-executor.0 8379 184490 0 0 3 0x14200 bored sosplice 33988 291458 67710 32767 3 0x90 nanoslp syz-executor.7 67710 54038 37905 0 3 0x82 wait syz-executor.7 97358 496306 44756 32767 2 0x10 syz-executor.4 97842 110694 31822 32767 3 0x90 nanoslp syz-executor.6 12131 523621 37984 32767 3 0x90 nanoslp syz-executor.3 8442 114375 6585 32767 3 0x90 nanoslp syz-executor.5 31822 81141 37905 0 3 0x82 wait syz-executor.6 6585 454137 37905 0 3 0x82 wait syz-executor.5 44756 168998 37905 0 3 0x82 wait syz-executor.4 72570 34859 11104 32767 3 0x90 nanoslp syz-executor.2 37984 21797 37905 0 3 0x82 wait syz-executor.3 24894 74564 35878 32767 3 0x90 nanoslp syz-executor.1 11104 178108 37905 0 3 0x82 wait syz-executor.2 35878 453056 37905 0 3 0x82 wait syz-executor.1 37905 224906 52600 0 3 0x82 kqread syz-fuzzer 37905 33479 52600 0 3 0x4000082 thrsleep syz-fuzzer 37905 66896 52600 0 3 0x4000082 thrsleep syz-fuzzer 37905 518577 52600 0 3 0x4000082 thrsleep syz-fuzzer 37905 232016 52600 0 3 0x4000082 thrsleep syz-fuzzer 37905 144924 52600 0 3 0x4000082 thrsleep syz-fuzzer 37905 254656 52600 0 3 0x4000082 thrsleep syz-fuzzer 37905 242075 52600 0 3 0x4000082 thrsleep syz-fuzzer 37905 317609 52600 0 3 0x4000082 thrsleep syz-fuzzer 52600 503092 8470 0 3 0x10008a sigsusp ksh 8470 193307 41763 0 3 0x9a kqread sshd 88512 355145 1 0 3 0x100083 ttyin getty 41763 406593 1 0 3 0x88 kqread sshd 2110 282684 31814 73 3 0x1100090 kqread syslogd 31814 205603 1 0 3 0x100082 netio syslogd 98123 130978 1 0 3 0x100080 kqread resolvd 5690 499637 34925 77 3 0x100092 kqread dhcpleased 34372 271434 34925 77 3 0x100092 kqread dhcpleased 34925 91703 1 0 3 0x80 kqread dhcpleased 82685 273293 0 0 3 0x14200 bored smr 49572 57721 0 0 2 0x14200 zerothread 36187 498127 0 0 3 0x14200 aiodoned aiodoned 37315 301782 0 0 3 0x14200 syncer update 32170 20038 0 0 3 0x14200 cleaner cleaner 64025 149659 0 0 3 0x14200 reaper reaper 29398 231987 0 0 3 0x14200 pgdaemon pagedaemon 37264 84225 0 0 3 0x14200 bored viomb 70727 112935 0 0 3 0x40014200 acpi0 acpi0 25271 173574 0 0 3 0x40014200 idle1 34369 435291 0 0 3 0x14200 bored softnet 21660 479552 0 0 3 0x14200 bored systqmp 67684 84305 0 0 3 0x14200 bored systq 41180 369306 0 0 3 0x40014200 bored softclock 80205 291625 0 0 3 0x40014200 idle0 1 395599 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{0}> show all locks Process 17465 (syz-executor.6) thread 0xffff800022c5fa48 (397463) shared rwlock netlock r = 0 (0xffffffff8290fe50) #0 witness_lock+0x44d #1 ifioctl_get+0x2d5 sys/net/if.c:2319 #2 soo_ioctl+0x26c #3 sys_ioctl+0x4a2 #4 syscall+0x489 mi_syscall sys/sys/syscall_mi.h:102 [inline] #4 syscall+0x489 sys/arch/amd64/amd64/trap.c:585 #5 Xsyscall+0x128 exclusive kernel_lock &kernel_lock r = 0 (0xffffffff829de988) #0 witness_lock+0x44d #1 soo_ioctl+0x25a sys/kern/sys_socket.c:136 #2 sys_ioctl+0x4a2 #3 syscall+0x489 mi_syscall sys/sys/syscall_mi.h:102 [inline] #3 syscall+0x489 sys/arch/amd64/amd64/trap.c:585 #4 Xsyscall+0x128 ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10202 6411K 6419K 78643K 11306 0 pcb 13 10K 12K 78643K 15 0 rtable 252 7K 7K 78643K 543 0 ifaddr 81 16K 16K 78643K 110 0 counters 56 35K 35K 78643K 64 0 ioctlops 0 0K 2K 78643K 47 0 iov 0 0K 16K 78643K 644 0 mount 1 1K 1K 78643K 1 0 log 0 0K 0K 78643K 5 0 vnodes 1271 79K 79K 78643K 2881 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 9K 78643K 31 0 VM map 2 1K 1K 78643K 2 0 sem 12 0K 0K 78643K 664 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1697 195K 286K 78643K 12548 0 file desc 24 89K 129K 78643K 3336 0 sigio 0 0K 0K 78643K 112 0 proc 56 74K 111K 78643K 771 0 subproc 104 6K 6K 78643K 156 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 201 0 in_multi 99 6K 6K 78643K 206 0 ether_multi 1 0K 0K 78643K 29 0 mrt 1 0K 0K 78643K 1 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 229 1023K 1023K 78643K 229 0 exec 0 0K 2K 78643K 1121 0 tdb 3 0K 0K 78643K 3 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 7 26K 26K 78643K 7 0 UVM amap 349 87K 102K 78643K 43565 0 UVM aobj 131 4K 4K 78643K 131 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 2 0K 0K 78643K 848 0 NDP 11 0K 2K 78643K 39 0 temp 125 4707K 4771K 78643K 11970 0 kqueue 12 18K 24K 78643K 221 0 SYN cache 2 16K 16K 78643K 2 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 22 0 0 1 0 1 1 0 8 0 rtpcb 120 183 0 180 3 0 3 3 0 8 2 rtentry 112 164 0 47 4 0 4 4 0 8 0 unpcb 136 1885 0 1871 21 20 1 6 0 8 0 syncache 296 39 0 39 7 7 0 1 0 8 0 tcpqe 32 113 0 113 5 5 0 1 0 8 0 tcpcb 736 978 0 952 35 32 3 11 0 8 0 arp 120 27 0 9 1 0 1 1 0 8 0 ipq 40 3 0 3 3 3 0 1 0 8 0 ipqe 40 12 0 12 3 3 0 1 0 8 0 inpcb 304 2892 0 2876 47 43 4 14 0 8 2 rttmr 72 35 0 34 1 0 1 1 0 8 0 ip6q 72 5 0 3 2 1 1 1 0 8 0 ip6af 40 6 0 4 1 0 1 1 0 8 0 nd6 48 43 0 13 1 0 1 1 0 8 0 kcovpl 48 12 0 4 1 0 1 1 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 713 0 206 33 1 32 33 0 8 0 art_table 32 714 0 206 5 0 5 5 0 8 0 art_node 16 163 0 56 1 0 1 1 0 8 0 sysvmsgpl 40 4 0 4 1 1 0 1 0 8 0 semapl 112 662 0 652 1 0 1 1 0 8 0 shmpl 112 128 0 0 4 0 4 4 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 6627 0 5198 90 0 90 90 0 8 0 ffsino 272 6627 0 5198 96 0 96 96 0 8 0 nchpl 144 11038 0 9416 62 0 62 62 0 8 0 uvmvnodes 80 5926 0 0 121 0 121 121 0 8 0 vnodes 224 5926 0 0 349 0 349 349 0 8 0 namei 1024 39097 0 39097 5 4 1 2 0 8 1 percpumem 16 44 0 4 1 0 1 1 0 8 0 scxspl 216 37963 0 37963 24 22 2 8 0 8 2 plimitpl 152 794 0 771 6 5 1 2 0 8 0 sigapl 424 3601 0 3550 7 0 7 7 0 8 0 futexpl 64 26005 0 26002 3 2 1 1 0 8 0 knotepl 120 399 0 0 11 0 11 11 0 8 0 kqueuepl 216 613 0 605 9 8 1 5 0 8 0 pipepl 336 689 0 661 17 11 6 8 0 8 3 fdescpl 496 3586 0 3551 7 1 6 6 0 8 0 filepl 152 25186 0 24943 42 27 15 17 0 8 5 lockfpl 104 437 0 435 1 0 1 1 0 8 0 lockfspl 48 123 0 121 1 0 1 1 0 8 0 sessionpl 144 27 0 11 1 0 1 1 0 8 0 pgrppl 48 119 0 103 1 0 1 1 0 8 0 ucredpl 96 9007 0 8989 1 0 1 1 0 8 0 zombiepl 144 3551 0 3550 1 0 1 1 0 8 0 processpl 1064 3601 0 3550 5 1 4 4 0 8 0 procpl 672 9420 0 9355 14 7 7 8 0 8 1 sosppl 168 54 0 54 5 4 1 1 0 8 1 sockpl 480 4984 0 4951 110 98 12 24 0 8 7 mcl64k 65536 16 0 0 2 0 2 2 0 8 0 mcl16k 16384 12 0 0 2 0 2 2 0 8 0 mcl12k 12288 17 0 0 2 0 2 2 0 8 0 mcl9k 9216 9 0 0 1 0 1 1 0 8 0 mcl8k 8192 17 0 0 3 0 3 3 0 8 0 mcl4k 4096 15 0 0 2 0 2 2 0 8 0 mcl2k2 2112 4 0 0 1 0 1 1 0 8 0 mcl2k 2048 169 0 0 20 1 19 20 0 8 0 mtagpl 96 3 0 0 1 0 1 1 0 8 0 mbufpl 256 724 0 0 36 0 36 36 0 8 0 bufpl 288 9919 0 3584 453 0 453 453 0 8 0 anonpl 24 989326 0 975480 167 61 106 118 0 186 13 amapchunkpl 152 111003 0 110219 73 37 36 47 0 158 3 amappl16 200 9160 0 8789 69 41 28 32 0 8 8 amappl15 192 905 0 894 1 0 1 1 0 8 0 amappl14 184 423 0 416 1 0 1 1 0 8 0 amappl13 176 508 0 504 1 0 1 1 0 8 0 amappl12 168 429 0 423 1 0 1 1 0 8 0 amappl11 160 828 0 815 1 0 1 1 0 8 0 amappl10 152 318 0 312 1 0 1 1 0 8 0 amappl9 144 986 0 981 1 0 1 1 0 8 0 amappl8 136 727 0 647 3 0 3 3 0 8 0 amappl7 128 129 0 119 1 0 1 1 0 8 0 amappl6 120 719 0 695 2 1 1 2 0 8 0 amappl5 112 2603 0 2581 1 0 1 1 0 8 0 amappl4 104 2904 0 2867 2 0 2 2 0 8 0 amappl3 96 813 0 800 1 0 1 1 0 8 0 amappl2 88 1186 0 1142 3 1 2 3 0 8 0 amappl1 80 66916 0 66300 20 6 14 18 0 8 0 amappl 88 42971 0 42711 7 0 7 7 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 130 0 0 3 0 3 3 0 8 0 uaddrrnd 24 3586 0 3551 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 3586 0 3551 1 0 1 1 0 8 0 vmmpekpl 168 31153 0 31100 3 0 3 3 0 8 0 vmmpepl 168 327167 0 324557 157 30 127 131 0 357 7 vmsppl 368 3585 0 3551 5 1 4 5 0 8 0 rwobjpl 56 84883 0 77503 110 3 107 107 0 8 1 pdppl 4096 7179 0 7102 191 110 81 97 0 8 4 pvpl 32 1776315 0 1757283 334 146 188 254 0 265 20 pmappl 248 3585 0 3551 4 1 3 3 0 8 0 extentpl 40 58 0 38 1 0 1 1 0 8 0 phpool 112 997 0 162 24 0 24 24 0 8 0 ddb{0}> machine ddbcpu 0 Invalid cpu 0 ddb{0}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:440 witness_checkorder(fffffd806777e030,1,0) at witness_checkorder+0x10b7 witness_debugger sys/kern/subr_witness.c:2502 [inline] witness_checkorder(fffffd806777e030,1,0) at witness_checkorder+0x10b7 sys/kern/subr_witness.c:1105 rw_enter_read(fffffd806777e020) at rw_enter_read+0x66 sys/kern/kern_rwlock.c:112 uvmfault_lookup(ffff800021201ad0,0) at uvmfault_lookup+0xd9 sys/uvm/uvm_fault.c:1758 uvm_fault_check(ffff800021201ad0,ffff800021201b08,ffff800021201b30) at uvm_fault_check+0x3a sys/uvm/uvm_fault.c:674 uvm_fault(fffffd806777e018,20000000,0,2) at uvm_fault+0x102 sys/uvm/uvm_fault.c:602 kpageflttrap(ffff800021201c60,20000300) at kpageflttrap+0x209 kerntrap(ffff800021201c60) at kerntrap+0xef sys/arch/amd64/amd64/trap.c:318 alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b copyout() at copyout+0x53 ifioctl_get(c0106924,ffff800021201f10) at ifioctl_get+0x2dd soo_ioctl(fffffd80087374e0,c0106924,ffff800021201f10,ffff800022c5fa48) at soo_ioctl+0x26c sys_ioctl(ffff800022c5fa48,ffff800021202028,ffff800021202080) at sys_ioctl+0x4a2 syscall(ffff8000212020f0) at syscall+0x489 mi_syscall sys/sys/syscall_mi.h:102 [inline] syscall(ffff8000212020f0) at syscall+0x489 sys/arch/amd64/amd64/trap.c:585 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xcb12ae5c840, count: -15 ddb{0}> machine ddbcpu 1 Stopped at x86_ipi_db+0x1a: addq $0x8,%rsp ddb{1}> trace x86_ipi_db(ffff800020ce8ff0) at x86_ipi_db+0x1a sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 __sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc+0x16 kd_curproc sys/dev/kcov.c:574 [inline] __sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc+0x16 sys/dev/kcov.c:148 __mp_lock(ffffffff829de780) at __mp_lock+0x133 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff829de780) at __mp_lock+0x133 sys/kern/kern_lock.c:147 syscall(ffff80002c944780) at syscall+0x3ef mi_syscall sys/sys/syscall_mi.h:93 [inline] syscall(ffff80002c944780) at syscall+0x3ef sys/arch/amd64/amd64/trap.c:585 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7966c0f3ef0, count: -7