witness: lock order reversal: 1st 0xffffffff827728c0 netlock (netlock) 2nd 0xfffffd806acd8b40 inode (&ip->i_lock) lock order "&ip->i_lock"(rrwlock) -> "netlock"(rwlock) first seen at: #0 witness_checkorder+0x65e witness_lock_order_add sys/kern/subr_witness.c:2463 [inline] #0 witness_checkorder+0x65e sys/kern/subr_witness.c:880 #1 rw_enter_write+0x5b sys/kern/kern_rwlock.c:127 #2 uvn_get+0xeb uvm_vnode_lock sys/uvm/uvm_vnode.c:1501 [inline] #2 uvn_get+0xeb sys/uvm/uvm_vnode.c:993 #3 uvm_fault+0xa41 sys/uvm/uvm_fault.c:1073 #4 kpageflttrap+0x202 sys/arch/amd64/amd64/trap.c:265 #5 kerntrap+0xef sys/arch/amd64/amd64/trap.c:321 #6 alltraps_kern_meltdown+0x7b #7 copyin+0x53 #8 sys_connect+0x9b sys/kern/uipc_syscalls.c:362 #9 syscall+0x4a1 mi_syscall sys/sys/syscall_mi.h:102 [inline] #9 syscall+0x4a1 sys/arch/amd64/amd64/trap.c:590 #10 Xsyscall+0x128 lock order "netlock"(rwlock) -> "&ip->i_lock"(rrwlock) first seen at: #0 witness_checkorder+0x65e witness_lock_order_add sys/kern/subr_witness.c:2463 [inline] #0 witness_checkorder+0x65e sys/kern/subr_witness.c:880 #1 rw_enter+0xd4 #2 rrw_enter+0x88 sys/kern/kern_rwlock.c:462 #3 VOP_LOCK+0x4b sys/kern/vfs_vops.c:614 #4 vn_lock+0x6c sys/kern/vfs_vnops.c:575 #5 vget+0x1c6 sys/kern/vfs_subr.c:671 #6 ktrwriteraw+0x138 sys/kern/kern_ktrace.c:659 #7 ktrstruct+0x169 ktrwrite2 sys/kern/kern_ktrace.c:627 [inline] #7 ktrstruct+0x169 sys/kern/kern_ktrace.c:311 #8 sys_connect+0x246 #9 syscall+0x4a1 mi_syscall sys/sys/syscall_mi.h:102 [inline] #9 syscall+0x4a1 sys/arch/amd64/amd64/trap.c:590 #10 Xsyscall+0x128 Stopped at db_enter+0x18: addq $0x8,%rsp ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic the kernel did not panic ddb{0}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 witness_checkorder(fffffd806acd8b40,9,0) at witness_checkorder+0xf5a witness_debugger sys/kern/subr_witness.c:2490 [inline] witness_checkorder(fffffd806acd8b40,9,0) at witness_checkorder+0xf5a sys/kern/subr_witness.c:1087 rw_enter(fffffd806acd8b30,1) at rw_enter+0xd4 rrw_enter(fffffd806acd8b30,1) at rrw_enter+0x88 sys/kern/kern_rwlock.c:462 VOP_LOCK(fffffd806e7c24e8,2001) at VOP_LOCK+0x4b sys/kern/vfs_vops.c:614 vn_lock(fffffd806e7c24e8,2001) at vn_lock+0x6c sys/kern/vfs_vnops.c:575 vget(fffffd806e7c24e8,2001) at vget+0x1c6 sys/kern/vfs_subr.c:671 ktrwriteraw(ffff800022344a68,fffffd806e7c24e8,fffffd807f7b7960,ffff800021cb9000,ffff800021cb8fe0) at ktrwriteraw+0x138 sys/kern/kern_ktrace.c:659 ktrstruct(ffff800022344a68,ffffffff823b392d,fffffd80658ad520,10) at ktrstruct+0x169 ktrwrite2 sys/kern/kern_ktrace.c:627 [inline] ktrstruct(ffff800022344a68,ffffffff823b392d,fffffd80658ad520,10) at ktrstruct+0x169 sys/kern/kern_ktrace.c:311 sys_connect(ffff800022344a68,ffff800021cb9148,ffff800021cb9190) at sys_connect+0x246 syscall(ffff800021cb9210) at syscall+0x4a1 mi_syscall sys/sys/syscall_mi.h:102 [inline] syscall(ffff800021cb9210) at syscall+0x4a1 sys/arch/amd64/amd64/trap.c:590 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x8bcadc15310, count: -12 ddb{0}> show registers rdi 0x3 rsi 0x40000 acpi_pdirpa+0x2be68 rbp 0xffff800021cb8c10 rbx 0x3 rdx 0xffff800000b08700 rcx 0x3 rax 0x3ffff acpi_pdirpa+0x2be67 r8 0xffffffff823582d3 witness_checkorder+0xf33 r9 0x5 r10 0xc179882558a0f4d3 r11 0x94631f3638571753 r12 0xffffffff828b9510 w_lodata+0x48300 r13 0 r14 0xffffffff828b7150 w_lodata+0x45f40 r15 0xfffffd8002ce5500 rip 0xffffffff81f33568 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff800021cb8c00 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb{0}> show proc PROC (syz-executor.0) pid=86478 stat=onproc flags process=0 proc=4000001 pri=32, usrpri=86, nice=20 forw=0xffffffffffffffff, list=0xffff8000223442b8,0xffffffff828de248 process=0xffff80002231dcf8 user=0xffff800021cb4000, vmspace=0xfffffd807effa8a0 estcpu=36, cpticks=0, pctcpu=0.0 user=0, sys=0, intr=0 ddb{0}> ps PID TID PPID UID S FLAGS WAIT COMMAND 58816 115772 87096 0 7 0x1 syz-executor.0 *58816 86478 87096 0 7 0x4000001 syz-executor.0 84518 85317 74238 0 3 0x82 nanosleep syz-executor.1 87191 136155 0 0 3 0x14280 nfsidl nfsio 97395 272114 0 0 3 0x14280 nfsidl nfsio 39157 9899 0 0 3 0x14280 nfsidl nfsio 26767 146103 0 0 3 0x14280 nfsidl nfsio 48754 472346 0 0 3 0x14280 nfsidl nfsio 63292 456247 0 0 3 0x14280 nfsidl nfsio 75182 29179 0 0 3 0x14280 nfsidl nfsio 10558 285441 0 0 3 0x14280 nfsidl nfsio 2101 514863 0 0 3 0x14280 nfsidl nfsio 14118 295753 0 0 3 0x14280 nfsidl nfsio 19479 102146 0 0 3 0x14280 nfsidl nfsio 11547 315935 0 0 3 0x14280 nfsidl nfsio 44284 471294 0 0 3 0x14280 nfsidl nfsio 14975 266998 0 0 3 0x14280 nfsidl nfsio 78243 378036 0 0 3 0x14280 nfsidl nfsio 41773 145496 0 0 3 0x14280 nfsidl nfsio 49080 485942 0 0 3 0x14280 nfsidl nfsio 60531 438534 0 0 3 0x14280 nfsidl nfsio 6245 154269 0 0 3 0x14280 nfsidl nfsio 4398 16565 0 0 3 0x14280 nfsidl nfsio 14947 34656 0 0 3 0x14200 bored sosplice 87096 262297 74238 0 3 0x82 nanosleep syz-executor.0 74238 465383 89434 0 3 0x82 thrsleep syz-fuzzer 74238 227742 89434 0 3 0x4000082 thrsleep syz-fuzzer 74238 264868 89434 0 3 0x4000082 thrsleep syz-fuzzer 74238 397402 89434 0 3 0x4000082 thrsleep syz-fuzzer 74238 161286 89434 0 3 0x4000082 thrsleep syz-fuzzer 74238 41247 89434 0 3 0x4000082 thrsleep syz-fuzzer 74238 423219 89434 0 3 0x4000082 kqread syz-fuzzer 74238 347530 89434 0 3 0x4000082 thrsleep syz-fuzzer 89434 245378 52097 0 3 0x10008a pause ksh 52097 371825 94000 0 3 0x92 select sshd 79458 470353 1 0 3 0x100083 ttyin getty 94000 451753 1 0 3 0x80 select sshd 33079 105436 83912 74 3 0x100092 bpf pflogd 83912 297715 1 0 3 0x80 netio pflogd 84226 299918 70593 73 3 0x100090 kqread syslogd 70593 99033 1 0 3 0x100082 netio syslogd 31850 97008 1 77 3 0x100090 poll dhclient 33214 90274 1 0 3 0x80 poll dhclient 70531 440002 0 0 3 0x14200 bored smr 33021 253817 0 0 3 0x14200 pgzero zerothread 37975 245424 0 0 3 0x14200 aiodoned aiodoned 32669 299263 0 0 3 0x14200 syncer update 19157 477657 0 0 3 0x14200 cleaner cleaner 9822 96208 0 0 3 0x14200 reaper reaper 21176 373496 0 0 3 0x14200 pgdaemon pagedaemon 46569 79612 0 0 3 0x14200 bored crynlk 87164 108758 0 0 3 0x14200 bored crypto 92335 264279 0 0 3 0x14200 bored viomb 7853 318396 0 0 3 0x40014200 acpi0 acpi0 79097 124086 0 0 3 0x40014200 idle1 72555 505551 0 0 3 0x14200 bored softnet 81155 26779 0 0 3 0x14200 bored systqmp 26747 367331 0 0 3 0x14200 bored systq 26527 277618 0 0 3 0x40014200 bored softclock 39420 369004 0 0 3 0x40014200 idle0 1 179521 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{0}> show all locks Process 58816 (syz-executor.0) thread 0xffff800022344a68 (86478) exclusive rwlock netlock r = 0 (0xffffffff827728c0) #0 witness_lock+0x4b0 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4b0 sys/kern/subr_witness.c:1164 #1 solock+0x5a sys/kern/uipc_socket2.c:282 #2 sys_connect+0x6a sys/kern/uipc_syscalls.c:357 #3 syscall+0x4a1 mi_syscall sys/sys/syscall_mi.h:102 [inline] #3 syscall+0x4a1 sys/arch/amd64/amd64/trap.c:590 #4 Xsyscall+0x128 exclusive kernel_lock &kernel_lock r = 1 (0xffffffff828db810) #0 witness_lock+0x4b0 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4b0 sys/kern/subr_witness.c:1164 #1 syscall+0x3fd mi_syscall sys/sys/syscall_mi.h:93 [inline] #1 syscall+0x3fd sys/arch/amd64/amd64/trap.c:590 #2 Xsyscall+0x128 ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 9515 6428K 7071K 78643K 13570 0 pcb 13 8K 8K 78643K 102 0 rtable 78 2K 3K 78643K 741 0 ifaddr 60 12K 13K 78643K 171 0 sysctl 2 0K 0K 78643K 2 0 counters 43 33K 34K 78643K 65 0 ioctlops 0 0K 4K 78643K 1684 0 iov 0 0K 12K 78643K 99 0 mount 1 1K 1K 78643K 1 0 vnodes 1223 77K 77K 78643K 2438 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 9K 78643K 28 0 VM map 2 1K 1K 78643K 2 0 sem 12 0K 0K 78643K 347 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1825 197K 290K 78643K 13109 0 file desc 5 13K 25K 78643K 3203 0 sigio 0 0K 0K 78643K 82 0 proc 64 63K 95K 78643K 682 0 subproc 32 2K 2K 78643K 68 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 427 0 in_multi 26 1K 2K 78643K 341 0 ether_multi 1 0K 0K 78643K 26 0 mrt 0 0K 0K 78643K 7 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 49 228K 228K 78643K 49 0 exec 0 0K 2K 78643K 583 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 7 26K 26K 78643K 7 0 UVM amap 139 25K 41K 78643K 8058 0 UVM aobj 45 2K 2K 78643K 45 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 0K 78643K 106 0 NDP 10 0K 0K 78643K 60 0 temp 123 3968K 4606K 78643K 12600 0 kqueue 3 4K 9K 78643K 111 0 SYN cache 2 16K 16K 78643K 2 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle arp 64 25 0 19 1 0 1 1 0 8 0 plcache 128 20 0 0 1 0 1 1 0 8 0 rtpcb 120 83 0 81 1 0 1 1 0 8 0 rtentry 112 169 0 139 2 0 2 2 0 8 0 unpcb 120 724 0 706 1 0 1 1 0 8 0 syncache 296 39 0 39 5 5 0 1 0 8 0 tcpqe 32 38 0 38 5 5 0 1 0 8 0 tcpcb 736 616 0 610 13 11 2 5 0 8 0 inpcb 296 1820 0 1810 5 3 2 2 0 8 1 nd6 48 43 0 39 1 0 1 1 0 8 0 pkpcb 40 20 0 20 3 2 1 1 0 8 1 kcovpl 48 4 0 2 1 0 1 1 0 8 0 pfstscr 40 62 0 62 1 1 0 1 0 8 0 pffrag 232 17 0 17 1 1 0 1 0 482 0 pffrnode 88 17 0 17 1 1 0 1 0 8 0 pffrent 40 765 0 765 1 1 0 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfrktable 1344 5 0 5 1 1 0 1 0 8 0 pfstitem 24 89 0 24 1 0 1 1 0 8 0 pfstkey 112 149 0 84 2 0 2 2 0 8 0 pfstate 328 118 0 53 6 0 6 6 0 8 0 pfrule 1360 47 0 42 2 1 1 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 634 0 502 12 2 10 12 0 8 1 art_table 32 635 0 502 2 0 2 2 0 8 0 art_node 16 168 0 145 1 0 1 1 0 8 0 sysvmsgpl 40 20 0 1 1 0 1 1 0 8 0 semupl 112 18 0 18 1 1 0 1 0 8 0 semapl 112 345 0 335 1 0 1 1 0 8 0 shmpl 112 42 0 0 2 0 2 2 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 5837 0 4430 91 2 89 89 0 8 0 ffsino 272 5837 0 4430 95 0 95 95 0 8 0 nchpl 144 10102 0 8509 60 0 60 60 0 8 0 uvmvnodes 72 5926 0 0 108 0 108 108 0 8 0 vnodes 208 5926 0 0 312 0 312 312 0 8 0 namei 1024 24640 0 24640 1 0 1 1 0 8 1 percpumem 16 43 0 11 1 0 1 1 0 8 0 scsiplug 72 4 0 4 1 1 0 1 0 8 0 scxspl 216 27683 0 27683 10 9 1 8 0 8 1 plimitpl 152 98 0 90 1 0 1 1 0 8 0 sigapl 424 3435 0 3382 7 0 7 7 0 8 1 futexpl 56 25630 0 25630 1 0 1 1 0 8 1 knotepl 112 257 0 237 1 0 1 1 0 8 0 kqueuepl 152 1516 0 1514 1 0 1 1 0 8 0 pipepl 304 270 0 259 6 4 2 2 0 8 1 fdescpl 496 3398 0 3382 3 0 3 3 0 8 0 filepl 152 12627 0 12523 6 1 5 6 0 8 1 lockfpl 104 654 0 652 1 0 1 1 0 8 0 lockfspl 48 229 0 227 1 0 1 1 0 8 0 sessionpl 144 20 0 9 1 0 1 1 0 8 0 pgrppl 48 29 0 18 1 0 1 1 0 8 0 ucredpl 96 1559 0 1550 1 0 1 1 0 8 0 zombiepl 144 3382 0 3381 1 0 1 1 0 8 0 processpl 1056 3435 0 3381 4 0 4 4 0 8 0 procpl 656 7124 0 7062 6 0 6 6 0 8 0 sosppl 168 52 0 51 4 3 1 1 0 8 0 sockpl 400 2653 0 2623 6 2 4 5 0 8 0 mcl64k 65536 20 0 0 3 0 3 3 0 8 0 mcl16k 16384 13 0 0 2 0 2 2 0 8 0 mcl12k 12288 22 0 0 2 0 2 2 0 8 0 mcl9k 9216 15 0 0 2 0 2 2 0 8 0 mcl8k 8192 5 0 0 1 0 1 1 0 8 0 mcl4k 4096 18 0 0 3 0 3 3 0 8 0 mcl2k 2048 203 0 0 18 0 18 18 0 8 0 mtagpl 96 316 0 0 8 0 8 8 0 8 0 mbufpl 256 1208 0 0 71 0 71 71 0 8 0 bufpl 280 10021 0 3772 447 0 447 447 0 8 0 anonpl 16 366816 0 361942 53 25 28 50 0 124 0 amapchunkpl 152 17823 0 17687 24 17 7 16 0 158 1 amappl16 192 12703 0 12534 27 18 9 27 0 8 0 amappl15 184 1437 0 1434 1 0 1 1 0 8 0 amappl14 176 26 0 20 1 0 1 1 0 8 0 amappl13 168 57 0 56 1 0 1 1 0 8 0 amappl12 160 15 0 8 2 1 1 1 0 8 0 amappl11 152 61 0 46 1 0 1 1 0 8 0 amappl10 144 133 0 129 1 0 1 1 0 8 0 amappl9 136 1668 0 1666 1 0 1 1 0 8 0 amappl8 128 269 0 200 3 0 3 3 0 8 0 amappl7 120 1857 0 1847 1 0 1 1 0 8 0 amappl6 112 220 0 208 1 0 1 1 0 8 0 amappl5 104 4116 0 4099 1 0 1 1 0 8 0 amappl4 96 329 0 299 1 0 1 1 0 8 0 amappl3 88 233 0 224 1 0 1 1 0 8 0 amappl2 80 23088 0 23018 3 1 2 3 0 8 0 amappl1 72 106784 0 106294 24 13 11 19 0 8 0 amappl 80 7486 0 7436 2 0 2 2 0 84 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 64 44 0 0 1 0 1 1 0 8 0 uaddrrnd 24 3398 0 3382 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 3398 0 3382 1 0 1 1 0 8 0 vmmpekpl 168 18804 0 18769 2 0 2 2 0 8 0 vmmpepl 168 430479 0 429061 102 34 68 83 0 357 1 vmsppl 368 3397 0 3382 2 0 2 2 0 8 0 pdppl 4096 6803 0 6764 57 16 41 45 0 8 2 pvpl 32 1208044 0 1199880 186 94 92 130 0 265 17 pmappl 232 3397 0 3382 2 1 1 2 0 8 0 extentpl 40 57 0 39 1 0 1 1 0 8 0 phpool 112 384 0 36 11 0 11 11 0 8 0 ddb{0}> machine ddbcpu 0 Invalid cpu 0 ddb{0}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 witness_checkorder(fffffd806acd8b40,9,0) at witness_checkorder+0xf5a witness_debugger sys/kern/subr_witness.c:2490 [inline] witness_checkorder(fffffd806acd8b40,9,0) at witness_checkorder+0xf5a sys/kern/subr_witness.c:1087 rw_enter(fffffd806acd8b30,1) at rw_enter+0xd4 rrw_enter(fffffd806acd8b30,1) at rrw_enter+0x88 sys/kern/kern_rwlock.c:462 VOP_LOCK(fffffd806e7c24e8,2001) at VOP_LOCK+0x4b sys/kern/vfs_vops.c:614 vn_lock(fffffd806e7c24e8,2001) at vn_lock+0x6c sys/kern/vfs_vnops.c:575 vget(fffffd806e7c24e8,2001) at vget+0x1c6 sys/kern/vfs_subr.c:671 ktrwriteraw(ffff800022344a68,fffffd806e7c24e8,fffffd807f7b7960,ffff800021cb9000,ffff800021cb8fe0) at ktrwriteraw+0x138 sys/kern/kern_ktrace.c:659 ktrstruct(ffff800022344a68,ffffffff823b392d,fffffd80658ad520,10) at ktrstruct+0x169 ktrwrite2 sys/kern/kern_ktrace.c:627 [inline] ktrstruct(ffff800022344a68,ffffffff823b392d,fffffd80658ad520,10) at ktrstruct+0x169 sys/kern/kern_ktrace.c:311 sys_connect(ffff800022344a68,ffff800021cb9148,ffff800021cb9190) at sys_connect+0x246 syscall(ffff800021cb9210) at syscall+0x4a1 mi_syscall sys/sys/syscall_mi.h:102 [inline] syscall(ffff800021cb9210) at syscall+0x4a1 sys/arch/amd64/amd64/trap.c:590 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x8bcadc15310, count: -12 ddb{0}> machine ddbcpu 1 Stopped at x86_ipi_db+0x1a: addq $0x8,%rsp ddb{1}> trace x86_ipi_db(ffff800020d68ff0) at x86_ipi_db+0x1a sys/arch/amd64/amd64/db_interface.c:352 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 __sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc+0x37 kd_curproc sys/dev/kcov.c:570 [inline] __sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc+0x37 sys/dev/kcov.c:143 __mp_lock(ffffffff828db608) at __mp_lock+0x133 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff828db608) at __mp_lock+0x133 sys/kern/kern_lock.c:147 ktrstruct(ffff8000223442b8,ffffffff823bffb5,ffff8000222d3508,10) at ktrstruct+0xee sys_clock_gettime(ffff8000223442b8,ffff8000222d3570,ffff8000222d35c0) at sys_clock_gettime+0xfb sys/kern/kern_time.c:171 syscall(ffff8000222d3640) at syscall+0x4a1 mi_syscall sys/sys/syscall_mi.h:102 [inline] syscall(ffff8000222d3640) at syscall+0x4a1 sys/arch/amd64/amd64/trap.c:590 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7f7fffff1510, count: -9