login: witness: lock order reversal: 1st 0xffffffff8290fe50 netlock (netlock) 2nd 0xfffffd80679cd5f0 vmmaplk (&map->lock) lock order "&map->lock"(rwlock) -> "netlock"(rwlock) first seen at: #0 rw_enter_write+0x5b sys/kern/kern_rwlock.c:128 #1 udp_sysctl+0x8a sys/netinet/udp_usrreq.c:1269 #2 sys_sysctl+0x209 sys/kern/kern_sysctl.c:249 #3 syscall+0x489 mi_syscall sys/sys/syscall_mi.h:102 [inline] #3 syscall+0x489 sys/arch/amd64/amd64/trap.c:585 #4 Xsyscall+0x128 lock order "netlock"(rwlock) -> "&map->lock"(rwlock) first seen at: #0 rw_enter_read+0x66 sys/kern/kern_rwlock.c:112 #1 uvmfault_lookup+0xd9 sys/uvm/uvm_fault.c:1758 #2 uvm_fault_check+0x3a sys/uvm/uvm_fault.c:674 #3 uvm_fault+0x102 sys/uvm/uvm_fault.c:602 #4 kpageflttrap+0x209 #5 kerntrap+0xef sys/arch/amd64/amd64/trap.c:318 #6 alltraps_kern_meltdown+0x7b #7 copyout+0x53 #8 ifioctl_get+0x2dd #9 soo_ioctl+0x26c #10 sys_ioctl+0x4a2 #11 syscall+0x489 mi_syscall sys/sys/syscall_mi.h:102 [inline] #11 syscall+0x489 sys/arch/amd64/amd64/trap.c:585 #12 Xsyscall+0x128 Stopped at db_enter+0x18: addq $0x8,%rsp ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic the kernel did not panic ddb{1}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:440 witness_checkorder(fffffd80679cd5f0,1,0) at witness_checkorder+0x10b7 witness_debugger sys/kern/subr_witness.c:2502 [inline] witness_checkorder(fffffd80679cd5f0,1,0) at witness_checkorder+0x10b7 sys/kern/subr_witness.c:1105 rw_enter_read(fffffd80679cd5e0) at rw_enter_read+0x66 sys/kern/kern_rwlock.c:112 uvmfault_lookup(ffff80002e3c36e0,0) at uvmfault_lookup+0xd9 sys/uvm/uvm_fault.c:1758 uvm_fault_check(ffff80002e3c36e0,ffff80002e3c3718,ffff80002e3c3740) at uvm_fault_check+0x3a sys/uvm/uvm_fault.c:674 uvm_fault(fffffd80679cd5d8,20000000,0,2) at uvm_fault+0x102 sys/uvm/uvm_fault.c:602 kpageflttrap(ffff80002e3c3870,20000300) at kpageflttrap+0x209 kerntrap(ffff80002e3c3870) at kerntrap+0xef sys/arch/amd64/amd64/trap.c:318 alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b copyout() at copyout+0x53 ifioctl_get(c0106924,ffff80002e3c3b20) at ifioctl_get+0x2dd soo_ioctl(fffffd80674fa9a0,c0106924,ffff80002e3c3b20,ffff8000ffff6fd0) at soo_ioctl+0x26c sys_ioctl(ffff8000ffff6fd0,ffff80002e3c3c38,ffff80002e3c3c90) at sys_ioctl+0x4a2 syscall(ffff80002e3c3d00) at syscall+0x489 mi_syscall sys/sys/syscall_mi.h:102 [inline] syscall(ffff80002e3c3d00) at syscall+0x489 sys/arch/amd64/amd64/trap.c:585 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x68a1be03220, count: -15 ddb{1}> show registers rdi 0x3 rsi 0xffffffff8294abb0 __sancov_gen_cov_switch_values.134 rbp 0xffff80002e3c33d0 rbx 0x3 rdx 0 rcx 0 rax 0xffff8000ffff6fd0 r8 0xffff80002e3c3340 r9 0x8080808080808080 r10 0x8a4f30d36cc15c25 r11 0x9a1d2ae555f975ab r12 0xffffffff82ab0380 w_lodata+0x52a10 r13 0 r14 0xffffffff82aa8660 w_lodata+0x4acf0 r15 0xfffffd8002f62d80 rip 0xffffffff8138d658 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff80002e3c33c0 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb{1}> show proc PROC (syz-executor.3) pid=329508 stat=onproc flags process=10 proc=4000000 pri=32, usrpri=76, nice=20 forw=0xffffffffffffffff, list=0xffff8000ffff7510,0xffff800022c4f520 process=0xffff80002954add8 user=0xffff80002e3be000, vmspace=0xfffffd80679cd5d8 estcpu=36, cpticks=0, pctcpu=0.0 user=0, sys=0, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 10153 210712 89604 32767 2 0x10 syz-executor.1 10153 214673 89604 32767 2 0x4000010 syz-executor.1 59205 97895 76008 32767 2 0x10 syz-executor.6 59205 46311 76008 32767 3 0x4000090 fsleep syz-executor.6 85884 435755 25032 32767 2 0x10 syz-executor.7 85884 84012 25032 32767 3 0x4000090 fsleep syz-executor.7 546 89441 44293 32767 7 0x10 syz-executor.3 546 425164 44293 32767 3 0x4000090 kqpoll syz-executor.3 * 546 329508 44293 32767 7 0x4000010 syz-executor.3 546 242830 44293 32767 2 0x4000010 syz-executor.3 546 437588 44293 32767 2 0x4000010 syz-executor.3 546 115103 44293 32767 2 0x4000010 syz-executor.3 546 310682 44293 32767 2 0x4000010 syz-executor.3 546 26561 44293 32767 2 0x4000010 syz-executor.3 69289 202716 78590 32767 2 0x10 syz-executor.4 72256 170038 48356 32767 3 0x90 nanoslp syz-executor.0 72256 37000 48356 32767 3 0x4000090 fsleep syz-executor.0 72256 450471 48356 32767 3 0x4000090 fsleep syz-executor.0 72256 343310 48356 32767 3 0x4000090 fsleep syz-executor.0 71072 419390 60599 32767 2 0x10 syz-executor.5 71072 68038 60599 32767 3 0x4000090 netio syz-executor.5 71072 69310 60599 32767 2 0x4000010 syz-executor.5 71072 133829 60599 32767 3 0x4000090 fsleep syz-executor.5 40520 246221 76663 32767 3 0x90 nanoslp syz-executor.2 40520 334624 76663 32767 3 0x4000090 fsleep syz-executor.2 40520 405663 76663 32767 2 0x4000010 syz-executor.2 40520 423217 76663 32767 3 0x4000090 fsleep syz-executor.2 29215 32522 0 0 3 0x14200 bored sosplice 78590 130126 60953 32767 3 0x90 nanoslp syz-executor.4 76008 477552 57020 32767 3 0x90 nanoslp syz-executor.6 60599 72802 43162 32767 3 0x90 nanoslp syz-executor.5 43162 222600 83135 0 3 0x82 wait syz-executor.5 25032 270280 77221 32767 3 0x90 nanoslp syz-executor.7 57020 157772 83135 0 3 0x82 wait syz-executor.6 77221 484554 83135 0 3 0x82 wait syz-executor.7 60953 452939 83135 0 3 0x82 wait syz-executor.4 76663 179768 17641 32767 3 0x90 nanoslp syz-executor.2 44293 513193 56155 32767 3 0x90 nanoslp syz-executor.3 56155 501892 83135 0 3 0x82 wait syz-executor.3 17641 26159 83135 0 3 0x82 wait syz-executor.2 89604 176326 95681 32767 3 0x90 nanoslp syz-executor.1 48356 424141 42426 32767 3 0x90 nanoslp syz-executor.0 95681 8486 83135 0 3 0x82 wait syz-executor.1 42426 486822 83135 0 3 0x82 wait syz-executor.0 83135 170943 92751 0 3 0x82 thrsleep syz-fuzzer 83135 168459 92751 0 3 0x4000082 thrsleep syz-fuzzer 83135 395433 92751 0 3 0x4000082 thrsleep syz-fuzzer 83135 436802 92751 0 3 0x4000082 thrsleep syz-fuzzer 83135 378088 92751 0 3 0x4000082 thrsleep syz-fuzzer 83135 307081 92751 0 3 0x4000082 thrsleep syz-fuzzer 83135 456134 92751 0 3 0x4000082 kqread syz-fuzzer 83135 112240 92751 0 3 0x4000082 thrsleep syz-fuzzer 83135 254205 92751 0 3 0x4000082 thrsleep syz-fuzzer 92751 123960 63181 0 3 0x10008a sigsusp ksh 63181 436023 68190 0 3 0x9a kqread sshd 42930 411868 1 0 3 0x100083 ttyin getty 68190 507896 1 0 3 0x88 kqread sshd 17803 460383 48652 73 3 0x1100090 kqread syslogd 48652 396496 1 0 3 0x100082 netio syslogd 29971 101752 1 0 3 0x100080 kqread resolvd 20181 258454 32533 77 3 0x100092 kqread dhcpleased 23734 152863 32533 77 3 0x100092 kqread dhcpleased 32533 13331 1 0 3 0x80 kqread dhcpleased 1549 313875 0 0 3 0x14200 bored smr 90704 77157 0 0 2 0x14200 zerothread 76870 336106 0 0 3 0x14200 aiodoned aiodoned 60873 378745 0 0 3 0x14200 syncer update 25694 120409 0 0 3 0x14200 cleaner cleaner 13020 120145 0 0 3 0x14200 reaper reaper 98645 120793 0 0 3 0x14200 pgdaemon pagedaemon 29770 91019 0 0 3 0x14200 bored viomb 20663 40988 0 0 3 0x40014200 acpi0 acpi0 36038 294731 0 0 3 0x40014200 idle1 46162 421587 0 0 3 0x14200 bored softnet 93474 177326 0 0 3 0x14200 bored systqmp 15191 306031 0 0 3 0x14200 bored systq 6300 241158 0 0 3 0x40014200 bored softclock 52771 103429 0 0 3 0x40014200 idle0 1 36178 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{1}> show all locks Process 546 (syz-executor.3) thread 0xffff8000ffff6fd0 (329508) shared rwlock netlock r = 0 (0xffffffff8290fe50) #0 witness_lock+0x44d #1 ifioctl_get+0x2d5 sys/net/if.c:2319 #2 soo_ioctl+0x26c #3 sys_ioctl+0x4a2 #4 syscall+0x489 mi_syscall sys/sys/syscall_mi.h:102 [inline] #4 syscall+0x489 sys/arch/amd64/amd64/trap.c:585 #5 Xsyscall+0x128 exclusive kernel_lock &kernel_lock r = 0 (0xffffffff829de988) #0 witness_lock+0x44d #1 soo_ioctl+0x25a sys/kern/sys_socket.c:136 #2 sys_ioctl+0x4a2 #3 syscall+0x489 mi_syscall sys/sys/syscall_mi.h:102 [inline] #3 syscall+0x489 sys/arch/amd64/amd64/trap.c:585 #4 Xsyscall+0x128 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10175 6407K 6419K 78643K 11265 0 pcb 13 8K 8K 78643K 13 0 rtable 246 7K 7K 78643K 365 0 ifaddr 81 16K 16K 78643K 82 0 counters 56 35K 35K 78643K 56 0 ioctlops 0 0K 2K 78643K 40 0 iov 1 8K 24K 78643K 51 0 mount 1 1K 1K 78643K 1 0 log 0 0K 0K 78643K 5 0 vnodes 1270 79K 79K 78643K 1386 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 5K 78643K 5 0 VM map 2 1K 1K 78643K 2 0 sem 12 0K 1K 78643K 89 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1697 195K 286K 78643K 12548 0 file desc 26 97K 117K 78643K 704 0 sigio 0 0K 0K 78643K 11 0 proc 56 74K 111K 78643K 498 0 subproc 104 6K 6K 78643K 104 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 2 0K 0K 78643K 92 0 in_multi 99 6K 6K 78643K 105 0 ether_multi 1 0K 0K 78643K 4 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 67 307K 307K 78643K 67 0 exec 0 0K 2K 78643K 700 0 tdb 3 0K 0K 78643K 3 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 7 26K 26K 78643K 7 0 UVM amap 360 84K 92K 78643K 10490 0 UVM aobj 31 2K 2K 78643K 31 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 0K 78643K 16 0 NDP 11 0K 2K 78643K 27 0 temp 125 4707K 4771K 78643K 5480 0 kqueue 14 22K 22K 78643K 59 0 SYN cache 2 16K 16K 78643K 2 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 22 0 0 1 0 1 1 0 8 0 rtpcb 120 47 0 44 1 0 1 1 0 8 0 rtentry 112 115 0 1 4 0 4 4 0 8 0 unpcb 136 228 0 210 2 1 1 2 0 8 0 syncache 296 11 0 11 3 3 0 1 0 8 0 tcpqe 32 56 0 56 2 2 0 1 0 8 0 tcpcb 736 153 0 143 7 5 2 7 0 8 0 arp 120 18 0 0 1 0 1 1 0 8 0 ipq 40 1 0 0 1 0 1 1 0 8 0 ipqe 40 4 0 3 1 0 1 1 0 8 0 inpcb 304 1001 0 923 12 5 7 7 0 8 0 nd6 48 28 0 0 1 0 1 1 0 8 0 kcovpl 48 8 0 0 1 0 1 1 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 486 0 0 31 0 31 31 0 8 0 art_table 32 487 0 0 4 0 4 4 0 8 0 art_node 16 114 0 10 1 0 1 1 0 8 0 sysvmsgpl 40 46 0 6 1 0 1 1 0 8 0 semupl 112 2 0 2 1 1 0 1 0 8 0 semapl 112 83 0 73 1 0 1 1 0 8 0 shmpl 112 28 0 0 1 0 1 1 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 2189 0 760 90 0 90 90 0 8 0 ffsino 272 2189 0 760 96 0 96 96 0 8 0 nchpl 144 3191 0 1526 62 0 62 62 0 8 0 uvmvnodes 80 2302 0 0 47 0 47 47 0 8 0 vnodes 224 2302 0 0 136 0 136 136 0 8 0 namei 1024 9924 0 9924 2 1 1 2 0 8 1 percpumem 16 40 0 0 1 0 1 1 0 8 0 scxspl 216 9128 0 9128 9 8 1 8 0 8 1 plimitpl 152 49 0 27 1 0 1 1 0 8 0 sigapl 424 982 0 930 7 0 7 7 0 8 0 futexpl 64 4912 0 4904 1 0 1 1 0 8 0 knotepl 120 131 0 0 4 0 4 4 0 8 0 kqueuepl 216 114 0 103 3 2 1 3 0 8 0 pipepl 336 227 0 199 8 2 6 8 0 8 3 fdescpl 496 967 0 930 7 1 6 6 0 8 0 filepl 152 4663 0 4356 20 7 13 15 0 8 1 lockfpl 104 30 0 28 1 0 1 1 0 8 0 lockfspl 48 16 0 14 1 0 1 1 0 8 0 sessionpl 144 23 0 7 1 0 1 1 0 8 0 pgrppl 48 23 0 7 1 0 1 1 0 8 0 ucredpl 96 643 0 625 1 0 1 1 0 8 0 zombiepl 144 930 0 930 1 0 1 1 0 8 1 processpl 1064 982 0 930 5 1 4 4 0 8 0 procpl 672 2216 0 2137 8 0 8 8 0 8 1 sosppl 168 4 0 4 1 0 1 1 0 8 1 sockpl 480 1288 0 1189 20 7 13 13 0 8 0 mcl64k 65536 9 0 0 2 0 2 2 0 8 0 mcl16k 16384 7 0 0 1 0 1 1 0 8 0 mcl12k 12288 7 0 0 1 0 1 1 0 8 0 mcl9k 9216 3 0 0 1 0 1 1 0 8 0 mcl8k 8192 8 0 0 1 0 1 1 0 8 0 mcl4k 4096 6 0 0 1 0 1 1 0 8 0 mcl2k2 2112 3 0 0 1 0 1 1 0 8 0 mcl2k 2048 173 0 0 18 0 18 18 0 8 0 mtagpl 96 3 0 0 1 0 1 1 0 8 0 mbufpl 256 337 0 0 21 0 21 21 0 8 0 bufpl 288 4173 0 135 289 0 289 289 0 8 0 anonpl 24 226946 0 212573 98 2 96 96 0 186 5 amapchunkpl 152 24902 0 24005 39 2 37 37 0 158 2 amappl16 200 1749 0 1365 22 1 21 21 0 8 0 amappl15 192 88 0 82 1 0 1 1 0 8 0 amappl14 184 14 0 11 1 0 1 1 0 8 0 amappl13 176 149 0 144 1 0 1 1 0 8 0 amappl12 168 10 0 10 1 1 0 1 0 8 0 amappl11 160 204 0 187 1 0 1 1 0 8 0 amappl10 152 124 0 117 1 0 1 1 0 8 0 amappl9 144 548 0 544 1 0 1 1 0 8 0 amappl8 136 780 0 719 3 0 3 3 0 8 0 amappl7 128 336 0 317 1 0 1 1 0 8 0 amappl6 120 282 0 258 2 1 1 2 0 8 0 amappl5 112 874 0 846 1 0 1 1 0 8 0 amappl4 104 962 0 935 2 1 1 2 0 8 0 amappl3 96 209 0 197 1 0 1 1 0 8 0 amappl2 88 606 0 561 3 1 2 3 0 8 0 amappl1 80 20378 0 19713 18 4 14 18 0 8 0 amappl 88 10051 0 9756 7 0 7 7 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 30 0 0 1 0 1 1 0 8 0 uaddrrnd 24 967 0 930 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 967 0 930 1 0 1 1 0 8 0 vmmpekpl 168 12304 0 12259 3 0 3 3 0 8 0 vmmpepl 168 90538 0 87729 146 20 126 146 0 357 3 vmsppl 368 966 0 930 4 0 4 4 0 8 0 rwobjpl 56 25042 0 21337 54 1 53 53 0 8 0 pdppl 4096 1941 0 1860 111 30 81 91 0 8 0 pvpl 32 500263 0 480448 259 25 234 259 0 265 71 pmappl 248 966 0 930 4 1 3 3 0 8 0 extentpl 40 58 0 38 1 0 1 1 0 8 0 phpool 112 689 0 40 19 0 19 19 0 8 0 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+0x1a: addq $0x8,%rsp ddb{0}> trace x86_ipi_db(ffffffff829baff0) at x86_ipi_db+0x1a sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 __mp_lock(ffffffff829de780) at __mp_lock+0x122 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff829de780) at __mp_lock+0x122 sys/kern/kern_lock.c:147 intr_handler(ffff800027ad6e30,ffff800000077500) at intr_handler+0x5e sys/arch/amd64/amd64/intr.c:532 Xintr_ioapic_edge16_untramp() at Xintr_ioapic_edge16_untramp+0x18f __mp_lock(ffffffff829de780) at __mp_lock+0x122 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff829de780) at __mp_lock+0x122 sys/kern/kern_lock.c:147 softintr_dispatch(0) at softintr_dispatch+0x4e sys/arch/amd64/amd64/softintr.c:88 Xsoftclock() at Xsoftclock+0x1f __mp_lock(ffffffff829de780) at __mp_lock+0x122 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff829de780) at __mp_lock+0x122 sys/kern/kern_lock.c:147 syscall(ffff800027ad7160) at syscall+0x3ef mi_syscall sys/sys/syscall_mi.h:93 [inline] syscall(ffff800027ad7160) at syscall+0x3ef sys/arch/amd64/amd64/trap.c:585 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffe6510, count: -12 ddb{0}> machine ddbcpu 1 Stopped at db_enter+0x18: addq $0x8,%rsp ddb{1}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:440 witness_checkorder(fffffd80679cd5f0,1,0) at witness_checkorder+0x10b7 witness_debugger sys/kern/subr_witness.c:2502 [inline] witness_checkorder(fffffd80679cd5f0,1,0) at witness_checkorder+0x10b7 sys/kern/subr_witness.c:1105 rw_enter_read(fffffd80679cd5e0) at rw_enter_read+0x66 sys/kern/kern_rwlock.c:112 uvmfault_lookup(ffff80002e3c36e0,0) at uvmfault_lookup+0xd9 sys/uvm/uvm_fault.c:1758 uvm_fault_check(ffff80002e3c36e0,ffff80002e3c3718,ffff80002e3c3740) at uvm_fault_check+0x3a sys/uvm/uvm_fault.c:674 uvm_fault(fffffd80679cd5d8,20000000,0,2) at uvm_fault+0x102 sys/uvm/uvm_fault.c:602 kpageflttrap(ffff80002e3c3870,20000300) at kpageflttrap+0x209 kerntrap(ffff80002e3c3870) at kerntrap+0xef sys/arch/amd64/amd64/trap.c:318 alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b copyout() at copyout+0x53 ifioctl_get(c0106924,ffff80002e3c3b20) at ifioctl_get+0x2dd soo_ioctl(fffffd80674fa9a0,c0106924,ffff80002e3c3b20,ffff8000ffff6fd0) at soo_ioctl+0x26c sys_ioctl(ffff8000ffff6fd0,ffff80002e3c3c38,ffff80002e3c3c90) at sys_ioctl+0x4a2 syscall(ffff80002e3c3d00) at syscall+0x489 mi_syscall sys/sys/syscall_mi.h:102 [inline] syscall(ffff80002e3c3d00) at syscall+0x489 sys/arch/amd64/amd64/trap.c:585 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x68a1be03220, count: -15