witness: lock order reversal: 1st 0xffffffff82c4a4f0 pf_lock (pf_lock) 2nd 0xfffffd806e14c218 solock (&so->so_lock) lock order "&so->so_lock"(rwlock) -> "pf_lock"(rwlock) first seen at: #0 rw_enter_write+0x5c sys/kern/kern_rwlock.c:128 #1 pf_remove_divert_state+0xbf sys/net/pf.c:1833 #2 in_pcbdisconnect+0x2c sys/netinet/in_pcb.c:578 #3 udp_disconnect+0xf1 sys/netinet/udp_usrreq.c:1215 #4 soconnect+0xc1 pru_disconnect sys/sys/protosw.h:334 [inline] #4 soconnect+0xc1 sodisconnect sys/kern/uipc_socket.c:521 [inline] #4 soconnect+0xc1 sys/kern/uipc_socket.c:485 #5 sys_connect+0x207 sys/kern/uipc_syscalls.c:422 #6 syscall+0x50f mi_syscall sys/sys/syscall_mi.h:103 [inline] #6 syscall+0x50f sys/arch/amd64/amd64/trap.c:606 #7 Xsyscall+0x128 lock order "pf_lock"(rwlock) -> "&so->so_lock"(rwlock) first seen at: #0 rw_enter_write+0x5c sys/kern/kern_rwlock.c:128 #1 route_input+0x101 sys/net/rtsock.c:520 #2 rtm_send+0x1b2 sys/net/rtsock.c:1744 #3 rt_clone+0xd1 sys/net/route.c:279 #4 rtalloc+0x91 rt_match sys/net/route.c:248 [inline] #4 rtalloc+0x91 sys/net/route.c:377 #5 icmp_reflect+0x1bd sys/netinet/ip_icmp.c:740 #6 icmp_error+0x61 sys/netinet/ip_icmp.c:300 #7 pf_test_rule+0x98d sys/net/pf.c:4478 #8 pf_test+0x1d1c sys/net/pf.c:7678 #9 ip_input_if+0x131 sys/netinet/ip_input.c:417 #10 ipv4_input+0x4c sys/netinet/ip_input.c:291 #11 ether_input+0x58a #12 tun_dev_write+0x203 sys/net/if_tun.c:914 #13 spec_write+0xd9 sys/kern/spec_vnops.c:302 #14 VOP_WRITE+0xc3 sys/kern/vfs_vops.c:245 #15 vn_write+0x15b sys/kern/vfs_vnops.c:408 #16 dofilewritev+0x1a0 sys/kern/sys_generic.c:375 #17 sys_write+0x87 sys/kern/sys_generic.c:295 #18 syscall+0x50f mi_syscall sys/sys/syscall_mi.h:103 [inline] #18 syscall+0x50f sys/arch/amd64/amd64/trap.c:606 Stopped at db_enter+0x1c: addq $0x8,%rsp ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic the kernel did not panic ddb{1}> trace db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 witness_checkorder(fffffd806e14c218,9,0) at witness_checkorder+0x10b7 witness_debugger sys/kern/subr_witness.c:2510 [inline] witness_checkorder(fffffd806e14c218,9,0) at witness_checkorder+0x10b7 sys/kern/subr_witness.c:1110 rw_enter_write(fffffd806e14c208) at rw_enter_write+0x5c sys/kern/kern_rwlock.c:128 route_input(fffffd80681e3b00,0,2) at route_input+0x101 sys/net/rtsock.c:520 rtm_send(fffffd8066961a20,1,0,0) at rtm_send+0x1b2 sys/net/rtsock.c:1744 rt_clone(ffff80002f11bd40,ffff80002f11bdb8,0) at rt_clone+0xd1 sys/net/route.c:279 rtalloc(ffff80002f11bdb8,1,0) at rtalloc+0x91 rt_match sys/net/route.c:248 [inline] rtalloc(ffff80002f11bdb8,1,0) at rtalloc+0x91 sys/net/route.c:377 icmp_reflect(fffffd8075271400,0,0) at icmp_reflect+0x1bd sys/netinet/ip_icmp.c:740 icmp_error(fffffd80681e3b00,3,3,0,0) at icmp_error+0x61 sys/netinet/ip_icmp.c:300 pf_test_rule(ffff80002f11c078,ffff80002f11c170,ffff80002f11c180,ffff80002f11c160,ffff80002f11c140,fffffd80702d8cf0) at pf_test_rule+0x98d sys/net/pf.c:4478 pf_test(2,1,ffff800000cff800,ffff80002f11c2a8) at pf_test+0x1d1c sys/net/pf.c:7678 ip_input_if(ffff80002f11c2a8,ffff80002f11c2b4,4,0,ffff800000cff800) at ip_input_if+0x131 sys/netinet/ip_input.c:417 ipv4_input(ffff800000cff800,fffffd80702d8c00) at ipv4_input+0x4c sys/netinet/ip_input.c:291 ether_input(ffff800000cff800,fffffd80702d8c00) at ether_input+0x58a tun_dev_write(5d00,ffff80002f11c618,ffff80002a2d7000,2) at tun_dev_write+0x203 sys/net/if_tun.c:914 spec_write(ffff80002f11c470) at spec_write+0xd9 sys/kern/spec_vnops.c:302 VOP_WRITE(fffffd807061f798,ffff80002f11c618,11,fffffd807f7d67b8) at VOP_WRITE+0xc3 sys/kern/vfs_vops.c:245 vn_write(fffffd806bbdcad0,ffff80002f11c618,0) at vn_write+0x15b sys/kern/vfs_vnops.c:408 dofilewritev(ffff80002a1ed2c0,c8,ffff80002f11c618,0,ffff80002f11c700) at dofilewritev+0x1a0 sys/kern/sys_generic.c:375 sys_write(ffff80002a1ed2c0,ffff80002f11c6b0,ffff80002f11c700) at sys_write+0x87 sys/kern/sys_generic.c:295 syscall(ffff80002f11c780) at syscall+0x50f mi_syscall sys/sys/syscall_mi.h:103 [inline] syscall(ffff80002f11c780) at syscall+0x50f sys/arch/amd64/amd64/trap.c:606 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x626f4853ec0, count: -22 ddb{1}> show registers rdi 0x3 rsi 0x40000 acpi_pdirpa+0x2be63 rbp 0xffff80002f11b940 rbx 0x3 rdx 0xffff800000d12480 rcx 0xffff80002a1ed2c0 rax 0x3ffff acpi_pdirpa+0x2be62 r8 0xffff80002f11b8b0 r9 0x8080808080808080 r10 0xb882cb71c970f27 r11 0x90ab4f8f4947e72c r12 0xfffffd8003a01440 r13 0 r14 0xfffffd80039fb400 r15 0xfffffd80031d29c0 rip 0xffffffff8272870c db_enter+0x1c cs 0x8 rflags 0x246 rsp 0xffff80002f11b930 ss 0x10 db_enter+0x1c: addq $0x8,%rsp ddb{1}> show proc PROC (syz-executor.0) tid=370064 pid=21838 tcnt=2 stat=onproc flags process=0 proc=4000000 runpri=81, usrpri=81, slppri=17, nice=20 wchan=0x0, wmesg=, ps_single=0x0 forw=0xffffffffffffffff, list=0xffff80002f13aaa0,0xffffffff82d4ed50 process=0xffff800033f92a00 user=0xffff80002f117000, vmspace=0xfffffd80662ac208 estcpu=36, cpticks=0, pctcpu=0.0, user=0, sys=0, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 21838 194544 85896 0 7 0 syz-executor.0 *21838 370064 85896 0 7 0x4000000 syz-executor.0 12199 85185 6966 0 3 0x80 nanoslp syz-executor.6 12199 290063 6966 0 3 0x4000080 fsleep syz-executor.6 12199 221262 6966 0 3 0x4000080 fsleep syz-executor.6 41683 345189 32602 0 3 0x80 nanoslp syz-executor.4 41683 57292 32602 0 3 0x4000080 kqread syz-executor.4 41683 3243 32602 0 3 0x4000080 kqpoll syz-executor.4 41683 107022 32602 0 3 0x4000080 fsleep syz-executor.4 27546 434287 47992 0 3 0x82 nanoslp syz-executor.5 16425 217941 47992 0 3 0x82 nanoslp syz-executor.7 31966 109636 47992 0 3 0x82 nanoslp syz-executor.3 25774 60283 1 0 3 0x100083 ttyin getty 32602 97749 47992 0 3 0x82 nanoslp syz-executor.4 41427 281296 0 0 3 0x14280 nfsidl nfsio 48909 419201 0 0 3 0x14280 nfsidl nfsio 64301 211968 47992 0 3 0x82 nanoslp syz-executor.1 91250 408640 0 0 3 0x14280 nfsidl nfsio 69468 158913 0 0 3 0x14280 nfsidl nfsio 72339 514091 0 0 3 0x14280 nfsidl nfsio 74242 388666 0 0 3 0x14280 nfsidl nfsio 93130 450659 0 0 3 0x14280 nfsidl nfsio 68886 397923 0 0 3 0x14280 nfsidl nfsio 81278 511382 0 0 3 0x14280 nfsidl nfsio 27066 321831 0 0 3 0x14280 nfsidl nfsio 42877 181732 0 0 3 0x14280 nfsidl nfsio 92763 345380 0 0 3 0x14280 nfsidl nfsio 83117 260862 0 0 3 0x14280 nfsidl nfsio 19294 75944 0 0 3 0x14280 nfsidl nfsio 77072 204184 0 0 3 0x14280 nfsidl nfsio 59133 113769 0 0 3 0x14280 nfsidl nfsio 79234 311044 0 0 3 0x14280 nfsidl nfsio 6805 268610 0 0 3 0x14280 nfsidl nfsio 78286 233695 0 0 3 0x14280 nfsidl nfsio 76158 510693 0 0 3 0x14280 nfsidl nfsio 52229 388302 0 0 3 0x14200 acct acct 96213 266535 0 0 3 0x14200 bored sosplice 6966 165925 47992 0 3 0x82 nanoslp syz-executor.6 13528 259699 70485 0 3 0x100082 netio arp 70485 132727 1 0 3 0x10008a sigsusp sh 4555 96654 47992 0 3 0x82 nanoslp syz-executor.2 85896 81555 47992 0 3 0x82 nanoslp syz-executor.0 47992 494828 3173 0 3 0x2000082 thrsleep syz-fuzzer 47992 425083 3173 0 3 0x6000082 thrsleep syz-fuzzer 47992 455334 3173 0 3 0x6000082 wait syz-fuzzer 47992 444149 3173 0 3 0x6000082 wait syz-fuzzer 47992 372371 3173 0 3 0x6000082 wait syz-fuzzer 47992 270120 3173 0 3 0x6000082 wait syz-fuzzer 47992 12341 3173 0 3 0x6000082 wait syz-fuzzer 47992 487265 3173 0 3 0x6000082 wait syz-fuzzer 47992 215488 3173 0 3 0x6000082 thrsleep syz-fuzzer 47992 168443 3173 0 3 0x6000082 thrsleep syz-fuzzer 47992 12561 3173 0 3 0x6000082 kqread syz-fuzzer 47992 398855 3173 0 3 0x6000082 thrsleep syz-fuzzer 47992 12454 3173 0 3 0x6000082 wait syz-fuzzer 47992 107809 3173 0 3 0x6000082 thrsleep syz-fuzzer 47992 289101 3173 0 3 0x6000082 wait syz-fuzzer 3173 43565 5013 0 3 0x10008a sigsusp ksh 5013 331113 40814 0 3 0x9a kqread sshd 40814 415180 1 0 3 0x88 kqread sshd 96187 377059 93822 74 3 0x1100092 bpf pflogd 93822 478158 1 0 3 0x80 netio pflogd 2783 160330 36151 73 3 0x1100090 kqread syslogd 36151 143865 1 0 3 0x100082 netio syslogd 63483 520960 1 0 3 0x100080 kqread resolvd 66187 107259 48959 77 3 0x100092 kqread dhcpleased 77854 92295 48959 77 3 0x100092 kqread dhcpleased 48959 86248 1 0 3 0x80 kqread dhcpleased 32996 379577 0 0 3 0x14200 bored smr 52687 26535 0 0 2 0x14200 zerothread 22774 299698 0 0 3 0x14200 aiodoned aiodoned 1850 234861 0 0 3 0x14200 syncer update 48090 194586 0 0 3 0x14200 cleaner cleaner 80767 92876 0 0 3 0x14200 reaper reaper 91198 110804 0 0 3 0x14200 pgdaemon pagedaemon 49341 369790 0 0 3 0x14200 bored viomb 37332 475975 0 0 3 0x40014200 acpi0 acpi0 13906 294945 0 0 3 0x40014200 idle1 70248 79182 0 0 3 0x14200 bored softnet3 45458 52462 0 0 3 0x14200 bored softnet2 84942 497568 0 0 3 0x14200 bored softnet1 66815 305597 0 0 3 0x14200 bored softnet0 81926 113518 0 0 3 0x14200 bored systqmp 91964 445586 0 0 3 0x14200 bored systq 10813 293936 0 0 3 0x14200 tmoslp softclockmp 55971 7448 0 0 3 0x40014200 tmoslp softclock 92722 184205 0 0 3 0x40014200 idle0 1 160726 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{1}> show all locks Process 21838 (syz-executor.0) thread 0xffff80002a1ed2c0 (370064) exclusive rwlock pf_lock r = 0 (0xffffffff82c4a4f0) #0 witness_lock+0x447 #1 pf_test+0x1cfd #2 ip_input_if+0x131 sys/netinet/ip_input.c:417 #3 ipv4_input+0x4c sys/netinet/ip_input.c:291 #4 ether_input+0x58a #5 tun_dev_write+0x203 sys/net/if_tun.c:914 #6 spec_write+0xd9 sys/kern/spec_vnops.c:302 #7 VOP_WRITE+0xc3 sys/kern/vfs_vops.c:245 #8 vn_write+0x15b sys/kern/vfs_vnops.c:408 #9 dofilewritev+0x1a0 sys/kern/sys_generic.c:375 #10 sys_write+0x87 sys/kern/sys_generic.c:295 #11 syscall+0x50f mi_syscall sys/sys/syscall_mi.h:103 [inline] #11 syscall+0x50f sys/arch/amd64/amd64/trap.c:606 #12 Xsyscall+0x128 exclusive rwlock netlock r = 0 (0xffffffff82c19210) #0 witness_lock+0x447 #1 tun_dev_write+0x1f8 sys/net/if_tun.c:913 #2 spec_write+0xd9 sys/kern/spec_vnops.c:302 #3 VOP_WRITE+0xc3 sys/kern/vfs_vops.c:245 #4 vn_write+0x15b sys/kern/vfs_vnops.c:408 #5 dofilewritev+0x1a0 sys/kern/sys_generic.c:375 #6 sys_write+0x87 sys/kern/sys_generic.c:295 #7 syscall+0x50f mi_syscall sys/sys/syscall_mi.h:103 [inline] #7 syscall+0x50f sys/arch/amd64/amd64/trap.c:606 #8 Xsyscall+0x128 exclusive kernel_lock &kernel_lock r = 0 (0xffffffff82c7c1a0) #0 witness_lock+0x447 #1 vn_write+0x46 sys/kern/vfs_vnops.c:393 #2 dofilewritev+0x1a0 sys/kern/sys_generic.c:375 #3 sys_write+0x87 sys/kern/sys_generic.c:295 #4 syscall+0x50f mi_syscall sys/sys/syscall_mi.h:103 [inline] #4 syscall+0x50f sys/arch/amd64/amd64/trap.c:606 #5 Xsyscall+0x128 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10230 6494K 6941K 166960K 15128 0 pcb 15 22K 26K 166960K 398 0 rtable 180 13K 15K 166960K 846 0 pf 33 9K 10K 166960K 81 0 ifaddr 36 13K 15K 166960K 95 0 ifgroup 56 2K 2K 166960K 119 0 sysctl 3 0K 2K 166960K 13 0 counters 64 36K 37K 166960K 100 0 ioctlops 0 0K 4K 166960K 1608 0 iov 0 0K 18K 166960K 531 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1363 85K 86K 166960K 3531 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 1K 5K 166960K 51 0 VM map 2 1K 1K 166960K 2 0 sem 12 0K 0K 166960K 850 0 dirhash 12 2K 2K 166960K 12 0 ACPI 1697 195K 286K 166960K 12548 0 file desc 16 57K 93K 166960K 3787 0 sigio 0 0K 0K 166960K 65 0 proc 69 91K 115K 166960K 917 0 subproc 117 7K 7K 166960K 239 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 1678 0 in_multi 66 4K 7K 166960K 231 0 ether_multi 1 0K 0K 166960K 9 0 mrt 1 0K 0K 166960K 9 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 217 970K 970K 166960K 217 0 exec 0 0K 1K 166960K 1059 0 pfkey data 0 0K 0K 166960K 4 0 tdb 3 0K 0K 166960K 3 0 pagedep 1 8K 8K 166960K 1 0 inodedep 1 32K 32K 166960K 1 0 newblk 1 0K 0K 166960K 1 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 406 253K 254K 166960K 39897 0 UVM aobj 131 5K 5K 166960K 137 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 0K 166960K 289 0 NDP 12 0K 0K 166960K 64 0 temp 75 5932K 6060K 166960K 28070 0 kqueue 14 22K 26K 166960K 276 0 SYN cache 2 16K 16K 166960K 2 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 24 0 0 1 0 1 1 0 8 0 rtpcb 120 264 0 260 5 4 1 5 0 8 0 rtentry 112 245 0 165 4 0 4 4 0 8 0 unpcb 144 4015 0 4000 33 30 3 6 0 8 2 syncache 312 26 0 26 5 5 0 1 0 8 0 tcpqe 32 558 0 558 6 6 0 1 0 8 0 tcpcb 808 3793 0 3773 55 46 9 22 0 8 6 arp 120 48 0 34 1 0 1 1 0 8 0 inpcb 368 7085 0 7055 63 54 9 29 0 8 5 nd6 136 57 0 41 1 0 1 1 0 8 0 pkpcb 40 46 0 46 2 2 0 1 0 8 0 kcovpl 48 18 0 9 1 0 1 1 0 8 0 ppxss 1256 5 0 5 2 2 0 1 0 8 0 pffrag 232 25 0 24 3 2 1 1 0 482 0 pffrnode 88 25 0 24 3 2 1 1 0 8 0 pffrent 40 56 0 55 3 2 1 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfstitem 24 139 0 112 1 0 1 1 0 8 0 pfstkey 128 139 0 112 3 1 2 2 0 8 0 pfstate 376 139 0 112 10 6 4 5 0 8 0 pfrule 1344 21 0 16 2 1 1 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 890 0 564 33 12 21 29 0 8 0 art_table 32 891 0 564 4 0 4 4 0 8 0 art_node 16 244 0 172 1 0 1 1 0 8 0 sysvmsgpl 40 15 0 9 1 0 1 1 0 8 0 semupl 112 6 0 6 2 2 0 1 0 8 0 semapl 112 847 0 837 1 0 1 1 0 8 0 shmpl 112 134 0 6 4 0 4 4 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 6856 0 5400 92 0 92 92 0 8 0 ffsino 272 6856 0 5400 98 0 98 98 0 8 0 nchpl 144 12099 0 10446 64 0 64 64 0 8 0 uvmvnodes 80 5926 0 0 121 0 121 121 0 8 0 vnodes 216 5926 0 0 330 0 330 330 0 8 0 namei 1024 40571 0 40571 3 2 1 2 0 8 1 percpumem 16 64 0 18 1 0 1 1 0 8 0 vcpupl 2048 17 0 0 3 0 3 3 0 8 0 vmpool 696 59 0 42 2 0 2 2 0 8 0 kstatmem 264 60 0 36 3 1 2 3 0 8 0 scxspl 216 38516 0 38516 12 11 1 8 1 8 1 plimitpl 152 658 0 641 1 0 1 1 0 8 0 sigapl 424 4185 0 4112 11 2 9 9 0 8 0 futexpl 64 33729 0 33726 3 2 1 1 0 8 0 knotepl 120 203 0 0 6 0 6 6 0 8 0 kqueuepl 216 821 0 811 17 16 1 5 0 8 0 pipepl 320 1496 0 1465 35 32 3 9 0 8 0 fdescpl 496 4065 0 4036 5 0 5 5 0 8 0 filepl 152 31679 0 31411 59 42 17 26 0 8 5 lockfpl 104 1087 0 1084 2 1 1 2 0 8 0 lockfspl 48 479 0 476 1 0 1 1 0 8 0 sessionpl 144 35 0 17 1 0 1 1 0 8 0 pgrppl 48 56 0 38 1 0 1 1 0 8 0 ucredpl 104 5525 0 5513 1 0 1 1 0 8 0 zombiepl 144 4117 0 4112 2 1 1 1 0 8 0 processpl 1072 4185 0 4112 5 0 5 5 0 8 0 procpl 680 10994 0 10901 14 5 9 10 0 8 0 srpgc 96 20 0 20 9 9 0 1 0 8 0 sosppl 168 33 0 31 6 5 1 1 0 8 0 sockpl 488 11413 0 11367 208 193 15 51 0 8 8 mcl64k 65536 17 0 0 3 0 3 3 0 8 0 mcl16k 16384 13 0 0 2 0 2 2 0 8 0 mcl12k 12288 17 0 0 2 0 2 2 0 8 0 mcl9k 9216 11 0 0 1 0 1 1 0 8 0 mcl8k 8192 17 0 0 3 0 3 3 0 8 0 mcl4k 4096 29 0 0 4 1 3 3 0 8 0 mcl2k2 2112 38 0 0 3 0 3 3 0 8 0 mcl2k 2048 432 0 0 49 4 45 49 0 8 0 mtagpl 96 232 0 0 6 0 6 6 0 8 0 mbufpl 256 1117 0 0 59 0 59 59 0 8 0 bufpl 288 10141 0 3816 452 0 452 452 0 8 0 anonpl 24 570065 0 554926 187 78 109 169 0 186 0 amapchunkpl 152 133611 0 132666 86 44 42 72 0 158 0 amappl16 200 12885 0 12481 64 37 27 35 0 8 1 amappl15 192 13 0 13 1 1 0 1 0 8 0 amappl14 184 189 0 173 2 1 1 2 0 8 0 amappl13 176 85 0 83 1 0 1 1 0 8 0 amappl12 168 4855 0 4825 3 1 2 2 0 8 0 amappl11 160 59 0 44 1 0 1 1 0 8 0 amappl10 152 45 0 31 1 0 1 1 0 8 0 amappl9 144 188 0 187 1 0 1 1 0 8 0 amappl8 136 385 0 290 4 0 4 4 0 8 0 amappl7 128 226 0 197 2 0 2 2 0 8 0 amappl6 120 425 0 413 1 0 1 1 0 8 0 amappl5 112 199 0 188 1 0 1 1 0 8 0 amappl4 104 579 0 547 2 1 1 2 0 8 0 amappl3 96 24907 0 24816 7 4 3 4 0 8 0 amappl2 88 4657 0 4577 3 1 2 3 0 8 0 amappl1 80 23493 0 22916 23 10 13 23 0 8 0 amappl 88 39156 0 38895 7 0 7 7 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 136 0 6 3 0 3 3 0 8 0 uaddrrnd 24 4124 0 4078 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 4124 0 4078 1 0 1 1 0 8 0 vmmpekpl 168 40434 0 40369 5 1 4 4 0 8 0 vmmpepl 168 263334 0 260951 221 102 119 136 0 357 2 vmsppl 464 4123 0 4078 8 1 7 7 0 8 0 rwobjpl 56 76934 0 69277 114 3 111 112 0 8 0 pdppl 4096 8256 0 8173 299 208 91 91 0 8 8 pvpl 32 42995 0 0 348 1 347 347 0 265 0 pmappl 248 4123 0 4078 4 0 4 4 0 8 0 extentpl 40 56 0 38 1 0 1 1 0 8 0 phpool 112 1246 0 341 27 0 27 27 0 8 0 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+0x1e: addq $0x8,%rsp ddb{0}> trace x86_ipi_db(ffffffff82b6fff0) at x86_ipi_db+0x1e sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 __mp_lock(ffffffff82c7bf98) at __mp_lock+0x133 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff82c7bf98) at __mp_lock+0x133 sys/kern/kern_lock.c:147 intr_handler(ffff80002f156840,ffff800000077e80) at intr_handler+0x62 sys/arch/amd64/amd64/intr.c:532 Xintr_ioapic_edge16_untramp() at Xintr_ioapic_edge16_untramp+0x18f __mp_lock(ffffffff82c7bf98) at __mp_lock+0x12e __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff82c7bf98) at __mp_lock+0x12e sys/kern/kern_lock.c:147 softintr_dispatch(0) at softintr_dispatch+0x52 sys/arch/amd64/amd64/softintr.c:88 Xsoftclock() at Xsoftclock+0x27 end of kernel end trace frame: 0x7ea9eb14eda0, count: -9 ddb{0}> machine ddbcpu 1 Stopped at db_enter+0x1c: addq $0x8,%rsp ddb{1}> trace db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 witness_checkorder(fffffd806e14c218,9,0) at witness_checkorder+0x10b7 witness_debugger sys/kern/subr_witness.c:2510 [inline] witness_checkorder(fffffd806e14c218,9,0) at witness_checkorder+0x10b7 sys/kern/subr_witness.c:1110 rw_enter_write(fffffd806e14c208) at rw_enter_write+0x5c sys/kern/kern_rwlock.c:128 route_input(fffffd80681e3b00,0,2) at route_input+0x101 sys/net/rtsock.c:520 rtm_send(fffffd8066961a20,1,0,0) at rtm_send+0x1b2 sys/net/rtsock.c:1744 rt_clone(ffff80002f11bd40,ffff80002f11bdb8,0) at rt_clone+0xd1 sys/net/route.c:279 rtalloc(ffff80002f11bdb8,1,0) at rtalloc+0x91 rt_match sys/net/route.c:248 [inline] rtalloc(ffff80002f11bdb8,1,0) at rtalloc+0x91 sys/net/route.c:377 icmp_reflect(fffffd8075271400,0,0) at icmp_reflect+0x1bd sys/netinet/ip_icmp.c:740 icmp_error(fffffd80681e3b00,3,3,0,0) at icmp_error+0x61 sys/netinet/ip_icmp.c:300 pf_test_rule(ffff80002f11c078,ffff80002f11c170,ffff80002f11c180,ffff80002f11c160,ffff80002f11c140,fffffd80702d8cf0) at pf_test_rule+0x98d sys/net/pf.c:4478 pf_test(2,1,ffff800000cff800,ffff80002f11c2a8) at pf_test+0x1d1c sys/net/pf.c:7678 ip_input_if(ffff80002f11c2a8,ffff80002f11c2b4,4,0,ffff800000cff800) at ip_input_if+0x131 sys/netinet/ip_input.c:417 ipv4_input(ffff800000cff800,fffffd80702d8c00) at ipv4_input+0x4c sys/netinet/ip_input.c:291 ether_input(ffff800000cff800,fffffd80702d8c00) at ether_input+0x58a tun_dev_write(5d00,ffff80002f11c618,ffff80002a2d7000,2) at tun_dev_write+0x203 sys/net/if_tun.c:914 spec_write(ffff80002f11c470) at spec_write+0xd9 sys/kern/spec_vnops.c:302 VOP_WRITE(fffffd807061f798,ffff80002f11c618,11,fffffd807f7d67b8) at VOP_WRITE+0xc3 sys/kern/vfs_vops.c:245 vn_write(fffffd806bbdcad0,ffff80002f11c618,0) at vn_write+0x15b sys/kern/vfs_vnops.c:408 dofilewritev(ffff80002a1ed2c0,c8,ffff80002f11c618,0,ffff80002f11c700) at dofilewritev+0x1a0 sys/kern/sys_generic.c:375 sys_write(ffff80002a1ed2c0,ffff80002f11c6b0,ffff80002f11c700) at sys_write+0x87 sys/kern/sys_generic.c:295 syscall(ffff80002f11c780) at syscall+0x50f mi_syscall sys/sys/syscall_mi.h:103 [inline] syscall(ffff80002f11c780) at syscall+0x50f sys/arch/amd64/amd64/trap.c:606 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x626f4853ec0, count: -22