login: witness: lock order reversal: 1st 0xffffffff8290fe50 netlock (netlock) 2nd 0xfffffd8068e08bb0 vmmaplk (&map->lock) lock order "&map->lock"(rwlock) -> "netlock"(rwlock) first seen at: #0 rw_enter_write+0x5b sys/kern/kern_rwlock.c:128 #1 udp_sysctl+0x8a sys/netinet/udp_usrreq.c:1269 #2 sys_sysctl+0x209 sys/kern/kern_sysctl.c:249 #3 syscall+0x489 mi_syscall sys/sys/syscall_mi.h:102 [inline] #3 syscall+0x489 sys/arch/amd64/amd64/trap.c:585 #4 Xsyscall+0x128 lock order "netlock"(rwlock) -> "&map->lock"(rwlock) first seen at: #0 rw_enter_read+0x66 sys/kern/kern_rwlock.c:112 #1 uvmfault_lookup+0xd9 sys/uvm/uvm_fault.c:1758 #2 uvm_fault_check+0x3a sys/uvm/uvm_fault.c:674 #3 uvm_fault+0x102 sys/uvm/uvm_fault.c:602 #4 kpageflttrap+0x209 #5 kerntrap+0xef sys/arch/amd64/amd64/trap.c:318 #6 alltraps_kern_meltdown+0x7b #7 copyout+0x53 #8 soo_ioctl+0x26c #9 sys_ioctl+0x4a2 #10 syscall+0x489 mi_syscall sys/sys/syscall_mi.h:102 [inline] #10 syscall+0x489 sys/arch/amd64/amd64/trap.c:585 #11 Xsyscall+0x128 Stopped at db_enter+0x18: addq $0x8,%rsp ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic the kernel did not panic ddb{1}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:440 witness_checkorder(fffffd8068e08bb0,1,0) at witness_checkorder+0x10b7 witness_debugger sys/kern/subr_witness.c:2502 [inline] witness_checkorder(fffffd8068e08bb0,1,0) at witness_checkorder+0x10b7 sys/kern/subr_witness.c:1105 rw_enter_read(fffffd8068e08ba0) at rw_enter_read+0x66 sys/kern/kern_rwlock.c:112 uvmfault_lookup(ffff80002c95d610,0) at uvmfault_lookup+0xd9 sys/uvm/uvm_fault.c:1758 uvm_fault_check(ffff80002c95d610,ffff80002c95d648,ffff80002c95d670) at uvm_fault_check+0x3a sys/uvm/uvm_fault.c:674 uvm_fault(fffffd8068e08b98,20ff9000,0,2) at uvm_fault+0x102 sys/uvm/uvm_fault.c:602 kpageflttrap(ffff80002c95d7a0,20ff9000) at kpageflttrap+0x209 kerntrap(ffff80002c95d7a0) at kerntrap+0xef sys/arch/amd64/amd64/trap.c:318 alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b copyout() at copyout+0x53 soo_ioctl(fffffd80759ad448,c028698d,ffff80002c95d9c0,ffff80002385b270) at soo_ioctl+0x26c sys_ioctl(ffff80002385b270,ffff80002c95dad8,ffff80002c95db30) at sys_ioctl+0x4a2 syscall(ffff80002c95dba0) at syscall+0x489 mi_syscall sys/sys/syscall_mi.h:102 [inline] syscall(ffff80002c95dba0) at syscall+0x489 sys/arch/amd64/amd64/trap.c:585 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xbb9a8d10490, count: -14 ddb{1}> show registers rdi 0x3 rsi 0x40000 acpi_pdirpa+0x2be68 rbp 0xffff80002c95d300 rbx 0x3 rdx 0xffff800000bc7100 rcx 0 rax 0x3ffff acpi_pdirpa+0x2be67 r8 0xffff80002c95d270 r9 0x8080808080808080 r10 0x9de35bf423531c3b r11 0x87f3a6ed5c376ed0 r12 0xffffffff82ab04e0 w_lodata+0x52b70 r13 0 r14 0xffffffff82aa8030 w_lodata+0x4a6c0 r15 0xfffffd8002f62d80 rip 0xffffffff8138d658 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff80002c95d2f0 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb{1}> show proc PROC (syz-executor.4) pid=370363 stat=onproc flags process=10 proc=4000000 pri=32, usrpri=76, nice=20 forw=0xffffffffffffffff, list=0xffff80002385ad30,0xffff80002385a2c0 process=0xffff800028d1d4e8 user=0xffff80002c958000, vmspace=0xfffffd8068e08b98 estcpu=36, cpticks=0, pctcpu=0.0 user=0, sys=0, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 7485 79362 603 32767 2 0x10 syz-executor.0 7485 53407 603 32767 3 0x4000090 fsleep syz-executor.0 88366 100426 1711 32767 2 0x10 syz-executor.3 88366 511 1711 32767 2 0x4000010 syz-executor.3 67595 33375 39940 32767 2 0x10 syz-executor.5 73514 132793 5043 32767 2 0x10 syz-executor.6 73514 523911 5043 32767 2 0x4000010 syz-executor.6 41508 45346 36173 32767 2 0x10 syz-executor.4 *41508 370363 36173 32767 7 0x4000010 syz-executor.4 14368 45421 9191 32767 2 0x10 syz-executor.7 14368 89497 9191 32767 3 0x4000090 ttyin syz-executor.7 60906 427559 0 0 3 0x14200 bored sosplice 9191 65218 12320 32767 2 0x10 syz-executor.7 39940 108461 93799 32767 3 0x90 nanoslp syz-executor.5 5043 239563 27818 32767 3 0x90 nanoslp syz-executor.6 27818 477612 53032 0 3 0x82 wait syz-executor.6 36173 421884 42840 32767 3 0x90 nanoslp syz-executor.4 93799 26443 53032 0 3 0x82 wait syz-executor.5 12320 444672 53032 0 3 0x82 wait syz-executor.7 42840 214032 53032 0 3 0x82 wait syz-executor.4 1711 352590 73668 32767 3 0x90 nanoslp syz-executor.3 73668 443037 53032 0 3 0x82 wait syz-executor.3 82192 369482 62901 32767 3 0x90 nanoslp syz-executor.2 40749 234084 49408 32767 3 0x90 nanoslp syz-executor.1 62901 373971 53032 0 3 0x82 wait syz-executor.2 603 310561 29618 32767 3 0x90 nanoslp syz-executor.0 49408 116433 53032 0 3 0x82 wait syz-executor.1 29618 301625 53032 0 3 0x82 wait syz-executor.0 53032 201895 54919 0 3 0x82 kqread syz-fuzzer 53032 214840 54919 0 3 0x4000082 thrsleep syz-fuzzer 53032 304394 54919 0 3 0x4000082 thrsleep syz-fuzzer 53032 106898 54919 0 3 0x4000082 thrsleep syz-fuzzer 53032 247590 54919 0 3 0x4000082 thrsleep syz-fuzzer 53032 334146 54919 0 3 0x4000082 thrsleep syz-fuzzer 53032 66068 54919 0 3 0x4000082 thrsleep syz-fuzzer 53032 140426 54919 0 3 0x4000082 thrsleep syz-fuzzer 54919 398684 19696 0 3 0x10008a sigsusp ksh 19696 304831 30794 0 3 0x9a kqread sshd 75558 48927 1 0 3 0x100083 ttyin getty 30794 520934 1 0 3 0x88 kqread sshd 47731 154870 54167 73 3 0x1100090 kqread syslogd 54167 132605 1 0 3 0x100082 netio syslogd 94688 92017 1 0 3 0x100080 kqread resolvd 24676 263148 4538 77 3 0x100092 kqread dhcpleased 48737 187184 4538 77 3 0x100092 kqread dhcpleased 4538 328716 1 0 3 0x80 kqread dhcpleased 71374 109184 0 0 3 0x14200 bored smr 49964 511708 0 0 2 0x14200 zerothread 57365 477421 0 0 3 0x14200 aiodoned aiodoned 17500 33767 0 0 3 0x14200 syncer update 89998 59106 0 0 3 0x14200 cleaner cleaner 50209 56852 0 0 7 0x14200 reaper 26972 381467 0 0 3 0x14200 pgdaemon pagedaemon 23761 96226 0 0 3 0x14200 bored viomb 99072 396448 0 0 3 0x40014200 acpi0 acpi0 43926 343359 0 0 3 0x40014200 idle1 9283 260446 0 0 3 0x14200 bored softnet 36457 33732 0 0 3 0x14200 bored systqmp 24699 231657 0 0 3 0x14200 bored systq 46497 255213 0 0 2 0x40014200 softclock 7401 275711 0 0 3 0x40014200 idle0 1 189577 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{1}> show all locks Process 41508 (syz-executor.4) thread 0xffff80002385b270 (370363) shared rwlock netlock r = 0 (0xffffffff8290fe50) #0 witness_lock+0x44d #1 ifioctl_get+0x81 if_getgrouplist sys/net/if.c:2911 [inline] #1 ifioctl_get+0x81 sys/net/if.c:2337 #2 soo_ioctl+0x26c #3 sys_ioctl+0x4a2 #4 syscall+0x489 mi_syscall sys/sys/syscall_mi.h:102 [inline] #4 syscall+0x489 sys/arch/amd64/amd64/trap.c:585 #5 Xsyscall+0x128 exclusive kernel_lock &kernel_lock r = 0 (0xffffffff829de988) #0 witness_lock+0x44d #1 soo_ioctl+0x25a sys/kern/sys_socket.c:136 #2 sys_ioctl+0x4a2 #3 syscall+0x489 mi_syscall sys/sys/syscall_mi.h:102 [inline] #3 syscall+0x489 sys/arch/amd64/amd64/trap.c:585 #4 Xsyscall+0x128 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10176 6407K 6419K 78643K 11266 0 pcb 13 8K 8K 78643K 13 0 rtable 242 7K 7K 78643K 384 0 ifaddr 81 16K 16K 78643K 82 0 counters 56 35K 35K 78643K 56 0 ioctlops 0 0K 2K 78643K 37 0 iov 0 0K 4K 78643K 7 0 mount 1 1K 1K 78643K 1 0 log 0 0K 0K 78643K 5 0 vnodes 1270 79K 79K 78643K 1307 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 1K 78643K 2 0 VM map 2 1K 1K 78643K 2 0 sem 12 0K 0K 78643K 205 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1697 195K 286K 78643K 12548 0 file desc 24 89K 121K 78643K 999 0 proc 56 74K 111K 78643K 483 0 subproc 104 6K 6K 78643K 104 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 115 0 in_multi 99 6K 7K 78643K 173 0 ether_multi 1 0K 0K 78643K 16 0 mrt 2 0K 0K 78643K 2 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 73 334K 334K 78643K 73 0 exec 0 0K 2K 78643K 613 0 tdb 3 0K 0K 78643K 3 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 7 26K 26K 78643K 7 0 UVM amap 296 79K 81K 78643K 13231 0 UVM aobj 3 2K 2K 78643K 3 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 0K 78643K 132 0 NDP 11 0K 2K 78643K 27 0 temp 125 4703K 4767K 78643K 5955 0 kqueue 12 18K 26K 78643K 111 0 SYN cache 2 16K 16K 78643K 2 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 22 0 0 1 0 1 1 0 8 0 rtpcb 120 116 0 113 1 0 1 1 0 8 0 rtentry 112 114 0 2 4 0 4 4 0 8 0 unpcb 136 405 0 391 2 1 1 2 0 8 0 syncache 296 16 0 16 2 1 1 1 0 8 1 tcpqe 32 42 0 42 2 1 1 1 0 8 1 tcpcb 736 412 0 403 11 3 8 8 0 8 5 arp 120 19 0 0 1 0 1 1 0 8 0 inpcb 304 685 0 672 3 1 2 2 0 8 0 nd6 48 26 0 1 1 0 1 1 0 8 0 kcovpl 48 8 0 0 1 0 1 1 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 457 0 1 29 0 29 29 0 8 0 art_table 32 458 0 1 4 0 4 4 0 8 0 art_node 16 113 0 11 1 0 1 1 0 8 0 sysvmsgpl 40 58 0 42 1 0 1 1 0 8 0 semupl 112 1 0 1 1 1 0 1 0 8 0 semapl 112 202 0 192 1 0 1 1 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 2473 0 1045 90 0 90 90 0 8 0 ffsino 272 2473 0 1045 96 0 96 96 0 8 0 nchpl 144 3942 0 2281 63 0 63 63 0 8 0 uvmvnodes 80 2586 0 0 53 0 53 53 0 8 0 vnodes 224 2586 0 0 153 0 153 153 0 8 0 namei 1024 10807 0 10807 2 1 1 2 0 8 1 percpumem 16 40 0 0 1 0 1 1 0 8 0 scxspl 216 12204 0 12204 9 8 1 8 0 8 1 plimitpl 152 92 0 68 2 0 2 2 0 8 1 sigapl 424 1276 0 1224 7 1 6 7 0 8 0 futexpl 64 8268 0 8267 1 0 1 1 0 8 0 knotepl 120 108 0 0 4 0 4 4 0 8 0 kqueuepl 216 187 0 179 1 0 1 1 0 8 0 pipepl 336 165 0 137 3 0 3 3 0 8 0 fdescpl 496 1261 0 1226 7 2 5 6 0 8 0 filepl 152 4563 0 4326 10 0 10 10 0 8 0 lockfpl 104 48 0 46 1 0 1 1 0 8 0 lockfspl 48 22 0 20 1 0 1 1 0 8 0 sessionpl 144 23 0 7 1 0 1 1 0 8 0 pgrppl 48 23 0 7 1 0 1 1 0 8 0 ucredpl 96 640 0 622 1 0 1 1 0 8 0 zombiepl 144 1226 0 1224 1 0 1 1 0 8 0 processpl 1064 1276 0 1224 5 1 4 4 0 8 0 procpl 672 2294 0 2229 6 0 6 6 0 8 0 sosppl 168 81 0 81 1 0 1 1 0 8 1 sockpl 480 1210 0 1181 8 3 5 6 0 8 0 mcl64k 65536 5 0 0 1 0 1 1 0 8 0 mcl16k 16384 5 0 0 1 0 1 1 0 8 0 mcl12k 12288 2 0 0 1 0 1 1 0 8 0 mcl9k 9216 2 0 0 1 0 1 1 0 8 0 mcl8k 8192 12 0 0 2 0 2 2 0 8 0 mcl4k 4096 14 0 0 2 0 2 2 0 8 0 mcl2k 2048 137 0 0 16 0 16 16 0 8 0 mtagpl 96 3 0 0 1 0 1 1 0 8 0 mbufpl 256 437 0 0 26 0 26 26 0 8 0 bufpl 288 5078 0 139 353 0 353 353 0 8 0 anonpl 24 293269 0 285439 80 7 73 77 0 186 19 amapchunkpl 152 31220 0 30599 40 6 34 38 0 158 7 amappl16 200 2099 0 1942 11 2 9 10 0 8 0 amappl15 192 302 0 297 1 0 1 1 0 8 0 amappl14 184 231 0 225 1 0 1 1 0 8 0 amappl13 176 208 0 203 1 0 1 1 0 8 0 amappl12 168 250 0 243 1 0 1 1 0 8 0 amappl11 160 160 0 147 1 0 1 1 0 8 0 amappl10 152 36 0 33 1 0 1 1 0 8 0 amappl9 144 473 0 470 1 0 1 1 0 8 0 amappl8 136 684 0 626 3 0 3 3 0 8 0 amappl7 128 220 0 208 1 0 1 1 0 8 0 amappl6 120 191 0 173 2 1 1 2 0 8 0 amappl5 112 738 0 718 1 0 1 1 0 8 0 amappl4 104 1030 0 998 2 0 2 2 0 8 0 amappl3 96 518 0 498 1 0 1 1 0 8 0 amappl2 88 786 0 739 3 1 2 3 0 8 0 amappl1 80 24449 0 23821 18 3 15 18 0 8 0 amappl 88 12783 0 12555 6 0 6 6 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 2 0 0 1 0 1 1 0 8 0 uaddrrnd 24 1261 0 1226 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 1261 0 1226 1 0 1 1 0 8 0 vmmpekpl 168 12175 0 12122 3 0 3 3 0 8 0 vmmpepl 168 112258 0 109892 114 2 112 112 0 357 2 vmsppl 368 1260 0 1225 4 0 4 4 0 8 0 rwobjpl 56 28997 0 25344 52 0 52 52 0 8 0 pdppl 4096 2529 0 2450 109 28 81 93 0 8 2 pvpl 32 650278 0 637387 261 24 237 260 0 265 122 pmappl 248 1260 0 1225 4 1 3 3 0 8 0 extentpl 40 58 0 38 1 0 1 1 0 8 0 phpool 112 752 0 34 21 0 21 21 0 8 0 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+0x1a: addq $0x8,%rsp ddb{0}> trace x86_ipi_db(ffffffff829baff0) at x86_ipi_db+0x1a sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 __mp_lock(ffffffff829de780) at __mp_lock+0x122 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff829de780) at __mp_lock+0x122 sys/kern/kern_lock.c:147 intr_handler(ffff800021134d00,ffff800000077500) at intr_handler+0x5e sys/arch/amd64/amd64/intr.c:532 Xintr_ioapic_edge16_untramp() at Xintr_ioapic_edge16_untramp+0x18f __sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc+0x2f kd_curproc sys/dev/kcov.c:578 [inline] __sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc+0x2f sys/dev/kcov.c:148 __mp_lock(ffffffff829de780) at __mp_lock+0x133 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff829de780) at __mp_lock+0x133 sys/kern/kern_lock.c:147 softintr_dispatch(0) at softintr_dispatch+0x4e sys/arch/amd64/amd64/softintr.c:88 Xsoftclock() at Xsoftclock+0x1f __mp_lock(ffffffff829de780) at __mp_lock+0x122 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff829de780) at __mp_lock+0x122 sys/kern/kern_lock.c:147 uvm_unmap_detach(ffff800021134fd0,1) at uvm_unmap_detach+0x113 sys/uvm/uvm_map.c:1615 uvm_map_teardown(fffffd8068e085d8) at uvm_map_teardown+0x262 sys/uvm/uvm_map.c:2789 uvmspace_free(fffffd8068e085d8) at uvmspace_free+0xa6 sys/uvm/uvm_map.c:3685 reaper(ffff8000210f9260) at reaper+0x18b sys/kern/kern_exit.c:457 end trace frame: 0x0, count: -15 ddb{0}> machine ddbcpu 1 Stopped at db_enter+0x18: addq $0x8,%rsp ddb{1}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:440 witness_checkorder(fffffd8068e08bb0,1,0) at witness_checkorder+0x10b7 witness_debugger sys/kern/subr_witness.c:2502 [inline] witness_checkorder(fffffd8068e08bb0,1,0) at witness_checkorder+0x10b7 sys/kern/subr_witness.c:1105 rw_enter_read(fffffd8068e08ba0) at rw_enter_read+0x66 sys/kern/kern_rwlock.c:112 uvmfault_lookup(ffff80002c95d610,0) at uvmfault_lookup+0xd9 sys/uvm/uvm_fault.c:1758 uvm_fault_check(ffff80002c95d610,ffff80002c95d648,ffff80002c95d670) at uvm_fault_check+0x3a sys/uvm/uvm_fault.c:674 uvm_fault(fffffd8068e08b98,20ff9000,0,2) at uvm_fault+0x102 sys/uvm/uvm_fault.c:602 kpageflttrap(ffff80002c95d7a0,20ff9000) at kpageflttrap+0x209 kerntrap(ffff80002c95d7a0) at kerntrap+0xef sys/arch/amd64/amd64/trap.c:318 alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b copyout() at copyout+0x53 soo_ioctl(fffffd80759ad448,c028698d,ffff80002c95d9c0,ffff80002385b270) at soo_ioctl+0x26c sys_ioctl(ffff80002385b270,ffff80002c95dad8,ffff80002c95db30) at sys_ioctl+0x4a2 syscall(ffff80002c95dba0) at syscall+0x489 mi_syscall sys/sys/syscall_mi.h:102 [inline] syscall(ffff80002c95dba0) at syscall+0x489 sys/arch/amd64/amd64/trap.c:585 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xbb9a8d10490, count: -14