witness: lock order reversal: 1st 0xffffffff82b890e0 pf_lock (pf_lock) 2nd 0xfffffd806eeb85d8 solock (&so->so_lock) lock order "&so->so_lock"(rwlock) -> "pf_lock"(rwlock) first seen at: #0 rw_enter_write+0x5c sys/kern/kern_rwlock.c:128 #1 pf_remove_divert_state+0xbf sys/net/pf.c:1833 #2 in_pcbdisconnect+0x2c sys/netinet/in_pcb.c:578 #3 udp_disconnect+0xf1 sys/netinet/udp_usrreq.c:1189 #4 soconnect+0xc1 pru_disconnect sys/sys/protosw.h:334 [inline] #4 soconnect+0xc1 sodisconnect sys/kern/uipc_socket.c:521 [inline] #4 soconnect+0xc1 sys/kern/uipc_socket.c:485 #5 sys_connect+0x207 sys/kern/uipc_syscalls.c:422 #6 syscall+0x50f mi_syscall sys/sys/syscall_mi.h:103 [inline] #6 syscall+0x50f sys/arch/amd64/amd64/trap.c:606 #7 Xsyscall+0x128 lock order "pf_lock"(rwlock) -> "&so->so_lock"(rwlock) first seen at: #0 rw_enter_write+0x5c sys/kern/kern_rwlock.c:128 #1 route_input+0x101 sys/net/rtsock.c:520 #2 rtm_send+0x1b2 sys/net/rtsock.c:1744 #3 rt_clone+0xd1 sys/net/route.c:279 #4 rtalloc+0x91 rt_match sys/net/route.c:248 [inline] #4 rtalloc+0x91 sys/net/route.c:377 #5 icmp_reflect+0x1bd sys/netinet/ip_icmp.c:740 #6 icmp_error+0x61 sys/netinet/ip_icmp.c:300 #7 pf_test_rule+0x98d sys/net/pf.c:4476 #8 pf_test+0x1d1c sys/net/pf.c:7676 #9 ip_input_if+0x131 sys/netinet/ip_input.c:417 #10 ipv4_input+0x4c sys/netinet/ip_input.c:291 #11 ether_input+0x58a #12 tun_dev_write+0x203 sys/net/if_tun.c:914 #13 spec_write+0xd9 sys/kern/spec_vnops.c:302 #14 VOP_WRITE+0xc3 sys/kern/vfs_vops.c:245 #15 vn_write+0x15b sys/kern/vfs_vnops.c:408 #16 dofilewritev+0x1a0 sys/kern/sys_generic.c:375 #17 sys_write+0x87 sys/kern/sys_generic.c:295 #18 syscall+0x50f mi_syscall sys/sys/syscall_mi.h:103 [inline] #18 syscall+0x50f sys/arch/amd64/amd64/trap.c:606 Stopped at db_enter+0x1c: addq $0x8,%rsp ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic the kernel did not panic ddb{0}> trace db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 witness_checkorder(fffffd806eeb85d8,9,0) at witness_checkorder+0x10b7 witness_debugger sys/kern/subr_witness.c:2510 [inline] witness_checkorder(fffffd806eeb85d8,9,0) at witness_checkorder+0x10b7 sys/kern/subr_witness.c:1110 rw_enter_write(fffffd806eeb85c8) at rw_enter_write+0x5c sys/kern/kern_rwlock.c:128 route_input(fffffd80625ae700,0,2) at route_input+0x101 sys/net/rtsock.c:520 rtm_send(fffffd80798c8390,1,0,0) at rtm_send+0x1b2 sys/net/rtsock.c:1744 rt_clone(ffff800036f9c8f0,ffff800036f9c968,0) at rt_clone+0xd1 sys/net/route.c:279 rtalloc(ffff800036f9c968,1,0) at rtalloc+0x91 rt_match sys/net/route.c:248 [inline] rtalloc(ffff800036f9c968,1,0) at rtalloc+0x91 sys/net/route.c:377 icmp_reflect(fffffd8067889b00,0,0) at icmp_reflect+0x1bd sys/netinet/ip_icmp.c:740 icmp_error(fffffd80625ae700,3,3,0,0) at icmp_error+0x61 sys/netinet/ip_icmp.c:300 pf_test_rule(ffff800036f9cc28,ffff800036f9cd20,ffff800036f9cd30,ffff800036f9cd10,ffff800036f9ccf0,fffffd807618eaf0) at pf_test_rule+0x98d sys/net/pf.c:4476 pf_test(2,1,ffff800000e1f800,ffff800036f9ce58) at pf_test+0x1d1c sys/net/pf.c:7676 ip_input_if(ffff800036f9ce58,ffff800036f9ce64,4,0,ffff800000e1f800) at ip_input_if+0x131 sys/netinet/ip_input.c:417 ipv4_input(ffff800000e1f800,fffffd807618ea00) at ipv4_input+0x4c sys/netinet/ip_input.c:291 ether_input(ffff800000e1f800,fffffd807618ea00) at ether_input+0x58a tun_dev_write(5d02,ffff800036f9d1c8,ffff80002cf05000,2) at tun_dev_write+0x203 sys/net/if_tun.c:914 spec_write(ffff800036f9d020) at spec_write+0xd9 sys/kern/spec_vnops.c:302 VOP_WRITE(fffffd807a2d9bd0,ffff800036f9d1c8,11,fffffd807f7d7208) at VOP_WRITE+0xc3 sys/kern/vfs_vops.c:245 vn_write(fffffd80789687c0,ffff800036f9d1c8,0) at vn_write+0x15b sys/kern/vfs_vnops.c:408 dofilewritev(ffff80002a2682b8,c8,ffff800036f9d1c8,0,ffff800036f9d2b0) at dofilewritev+0x1a0 sys/kern/sys_generic.c:375 sys_write(ffff80002a2682b8,ffff800036f9d260,ffff800036f9d2b0) at sys_write+0x87 sys/kern/sys_generic.c:295 syscall(ffff800036f9d330) at syscall+0x50f mi_syscall sys/sys/syscall_mi.h:103 [inline] syscall(ffff800036f9d330) at syscall+0x50f sys/arch/amd64/amd64/trap.c:606 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xc97ad866630, count: -22 ddb{0}> show registers rdi 0x3 rsi 0x40000 acpi_pdirpa+0x2be63 rbp 0xffff800036f9c4f0 rbx 0x3 rdx 0xffff800000dd2e40 rcx 0xffff80002a2682b8 rax 0x3ffff acpi_pdirpa+0x2be62 r8 0xffff800036f9c460 r9 0x8080808080808080 r10 0x13662adde966bfbe r11 0x14cb0bd5dade6431 r12 0xfffffd8003a06280 r13 0 r14 0xfffffd80039faba0 r15 0xfffffd80031d59c0 rip 0xffffffff812c3c8c db_enter+0x1c cs 0x8 rflags 0x246 rsp 0xffff800036f9c4e0 ss 0x10 db_enter+0x1c: addq $0x8,%rsp ddb{0}> show proc PROC (syz-executor.2) tid=457324 pid=24148 tcnt=2 stat=onproc flags process=0 proc=4000000 runpri=82, usrpri=82, slppri=17, nice=20 wchan=0x0, wmesg=, ps_single=0x0 forw=0xffffffffffffffff, list=0xffff80002a267020,0xffffffff82d7e898 process=0xffff8000ffff0448 user=0xffff800036f98000, vmspace=0xfffffd80736061d0 estcpu=36, cpticks=0, pctcpu=0.0, user=0, sys=0, intr=0 ddb{0}> ps PID TID PPID UID S FLAGS WAIT COMMAND 24148 129748 74018 0 2 0 syz-executor.2 *24148 457324 74018 0 7 0x4000000 syz-executor.2 80095 146294 39629 0 2 0 syz-executor.1 88642 100173 65783 0 2 0 syz-executor.4 66509 403045 55609 0 2 0 syz-executor.3 20676 243251 53181 0 2 0 syz-executor.6 20676 238010 53181 0 2 0x4000000 syz-executor.6 9589 176881 3581 0 2 0 syz-executor.0 9589 333386 3581 0 3 0x4000000 clonelk syz-executor.0 9589 273957 3581 0 3 0x4000080 pipewr syz-executor.0 51686 495874 0 0 3 0x14280 nfsidl nfsio 49308 487152 0 0 3 0x14280 nfsidl nfsio 6259 126905 0 0 3 0x14280 nfsidl nfsio 48360 96016 0 0 3 0x14280 nfsidl nfsio 95835 419690 0 0 3 0x14280 nfsidl nfsio 19004 238140 0 0 3 0x14280 nfsidl nfsio 9826 229851 0 0 3 0x14280 nfsidl nfsio 94414 75710 0 0 3 0x14280 nfsidl nfsio 10335 110637 0 0 3 0x14280 nfsidl nfsio 33765 430505 0 0 3 0x14280 nfsidl nfsio 16724 403746 0 0 3 0x14280 nfsidl nfsio 5726 378404 0 0 3 0x14280 nfsidl nfsio 40149 324578 0 0 3 0x14280 nfsidl nfsio 86130 349583 0 0 3 0x14280 nfsidl nfsio 9293 408549 0 0 3 0x14280 nfsidl nfsio 90735 204101 0 0 3 0x14280 nfsidl nfsio 63048 151399 0 0 3 0x14280 nfsidl nfsio 59428 365879 0 0 3 0x14280 nfsidl nfsio 55513 55843 0 0 3 0x14280 nfsidl nfsio 33585 410317 0 0 3 0x14280 nfsidl nfsio 74293 356903 74567 0 2 0x2 syz-executor.5 53181 305122 74567 0 3 0x82 nanoslp syz-executor.6 37508 153558 0 0 3 0x14200 acct acct 39629 146329 74567 0 2 0x2 syz-executor.1 55609 216950 74567 0 2 0x2 syz-executor.3 74018 23907 74567 0 3 0x82 nanoslp syz-executor.2 65783 395407 74567 0 3 0x82 nanoslp syz-executor.4 3581 56179 74567 0 3 0x82 nanoslp syz-executor.0 88615 258671 1 0 3 0x100083 ttyopn getty 44780 178583 0 0 3 0x14200 bored sosplice 74567 500746 25374 0 3 0x2000082 thrsleep syz-fuzzer 74567 520031 25374 0 3 0x6000082 nanoslp syz-fuzzer 74567 435564 25374 0 3 0x6000082 wait syz-fuzzer 74567 344439 25374 0 3 0x6000082 wait syz-fuzzer 74567 442456 25374 0 3 0x6000082 kqread syz-fuzzer 74567 319927 25374 0 3 0x6000082 wait syz-fuzzer 74567 191298 25374 0 3 0x6000082 wait syz-fuzzer 74567 412962 25374 0 3 0x6000082 thrsleep syz-fuzzer 74567 466587 25374 0 3 0x6000082 wait syz-fuzzer 74567 22479 25374 0 3 0x6000082 wait syz-fuzzer 74567 243382 25374 0 3 0x6000082 thrsleep syz-fuzzer 74567 435468 25374 0 3 0x6000082 thrsleep syz-fuzzer 74567 123230 25374 0 3 0x6000082 wait syz-fuzzer 74567 400815 25374 0 3 0x6000082 thrsleep syz-fuzzer 74567 135978 25374 0 3 0x6000082 wait syz-fuzzer 74567 125356 25374 0 3 0x6000082 thrsleep syz-fuzzer 25374 115640 5612 0 3 0x10008a sigsusp ksh 5612 42995 23894 0 3 0x9a kqread sshd 23894 212713 1 0 3 0x88 kqread sshd 49757 492473 99467 74 3 0x1100092 bpf pflogd 99467 394458 1 0 3 0x80 netio pflogd 51548 67706 47355 73 3 0x1100090 kqread syslogd 47355 284881 1 0 3 0x100082 netio syslogd 76509 406994 1 0 3 0x100080 kqread resolvd 60281 445492 46754 77 3 0x100092 kqread dhcpleased 2548 423920 46754 77 3 0x100092 kqread dhcpleased 46754 464106 1 0 3 0x80 kqread dhcpleased 59886 178410 0 0 3 0x14200 bored smr 9197 179351 0 0 2 0x14200 zerothread 79611 32843 0 0 3 0x14200 aiodoned aiodoned 87734 41256 0 0 3 0x14200 syncer update 4909 209909 0 0 3 0x14200 cleaner cleaner 97593 285395 0 0 3 0x14200 reaper reaper 18945 217869 0 0 3 0x14200 pgdaemon pagedaemon 90738 462965 0 0 3 0x14200 bored viomb 90414 121174 0 0 3 0x40014200 acpi0 acpi0 75181 61076 0 0 7 0x40014200 idle1 11355 79624 0 0 3 0x14200 bored softnet3 34572 355027 0 0 3 0x14200 bored softnet2 69885 76360 0 0 3 0x14200 bored softnet1 43175 66712 0 0 3 0x14200 bored softnet0 33102 90693 0 0 3 0x14200 bored systqmp 14957 16926 0 0 3 0x14200 bored systq 80848 290292 0 0 3 0x14200 tmoslp softclockmp 53391 520168 0 0 3 0x40014200 tmoslp softclock 19550 113185 0 0 3 0x40014200 idle0 1 370213 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{0}> show all locks Process 24148 (syz-executor.2) thread 0xffff80002a2682b8 (457324) exclusive rwlock pf_lock r = 0 (0xffffffff82b890e0) #0 witness_lock+0x447 #1 pf_test+0x1cfd #2 ip_input_if+0x131 sys/netinet/ip_input.c:417 #3 ipv4_input+0x4c sys/netinet/ip_input.c:291 #4 ether_input+0x58a #5 tun_dev_write+0x203 sys/net/if_tun.c:914 #6 spec_write+0xd9 sys/kern/spec_vnops.c:302 #7 VOP_WRITE+0xc3 sys/kern/vfs_vops.c:245 #8 vn_write+0x15b sys/kern/vfs_vnops.c:408 #9 dofilewritev+0x1a0 sys/kern/sys_generic.c:375 #10 sys_write+0x87 sys/kern/sys_generic.c:295 #11 syscall+0x50f mi_syscall sys/sys/syscall_mi.h:103 [inline] #11 syscall+0x50f sys/arch/amd64/amd64/trap.c:606 #12 Xsyscall+0x128 exclusive rwlock netlock r = 0 (0xffffffff82bacb10) #0 witness_lock+0x447 #1 tun_dev_write+0x1f8 sys/net/if_tun.c:913 #2 spec_write+0xd9 sys/kern/spec_vnops.c:302 #3 VOP_WRITE+0xc3 sys/kern/vfs_vops.c:245 #4 vn_write+0x15b sys/kern/vfs_vnops.c:408 #5 dofilewritev+0x1a0 sys/kern/sys_generic.c:375 #6 sys_write+0x87 sys/kern/sys_generic.c:295 #7 syscall+0x50f mi_syscall sys/sys/syscall_mi.h:103 [inline] #7 syscall+0x50f sys/arch/amd64/amd64/trap.c:606 #8 Xsyscall+0x128 exclusive kernel_lock &kernel_lock r = 0 (0xffffffff82d7d398) #0 witness_lock+0x447 #1 vn_write+0x46 sys/kern/vfs_vnops.c:393 #2 dofilewritev+0x1a0 sys/kern/sys_generic.c:375 #3 sys_write+0x87 sys/kern/sys_generic.c:295 #4 syscall+0x50f mi_syscall sys/sys/syscall_mi.h:103 [inline] #4 syscall+0x50f sys/arch/amd64/amd64/trap.c:606 #5 Xsyscall+0x128 ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10244 6592K 10660K 166960K 55459 0 pcb 13 16K 18K 166960K 1310 0 rtable 236 15K 15K 166960K 3070 0 pf 37 10K 11K 166960K 412 0 ifaddr 44 16K 18K 166960K 379 0 ifgroup 64 2K 2K 166960K 670 0 sysctl 3 0K 0K 166960K 3 0 counters 68 36K 37K 166960K 420 0 ioctlops 0 0K 4K 166960K 2231 0 iov 0 0K 30K 166960K 2436 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1447 90K 91K 166960K 14451 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 1K 9K 166960K 249 0 VM map 2 1K 1K 166960K 2 0 sem 10 1K 1K 166960K 16 0 dirhash 12 2K 2K 166960K 87 0 ACPI 1697 195K 286K 166960K 12548 0 file desc 16 57K 89K 166960K 22525 0 sigio 0 0K 0K 166960K 5771 0 proc 70 91K 128K 166960K 2589 0 subproc 104 6K 7K 166960K 736 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 1139 0 in_multi 88 6K 7K 166960K 852 0 ether_multi 1 0K 0K 166960K 22 0 mrt 1 0K 0K 166960K 10 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 241 1076K 1076K 166960K 241 0 exec 0 0K 1K 166960K 2908 0 pfkey data 0 0K 0K 166960K 20 0 tdb 3 0K 0K 166960K 3 0 pagedep 1 8K 8K 166960K 1 0 inodedep 1 32K 32K 166960K 1 0 newblk 1 0K 0K 166960K 1 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 533 773K 784K 166960K 220386 0 UVM aobj 131 4K 4K 166960K 131 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 0K 166960K 727 0 NDP 14 0K 1K 166960K 302 0 temp 74 5932K 137008K 166960K 213340 0 kqueue 12 18K 28K 166960K 1303 0 SYN cache 2 16K 16K 166960K 2 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 24 0 0 1 0 1 1 0 8 0 rtpcb 120 1046 0 1043 13 12 1 5 0 8 0 rtentry 112 884 0 776 4 0 4 4 0 8 0 unpcb 144 12133 0 12117 149 147 2 8 0 8 1 syncache 312 237 0 237 45 44 1 1 0 8 1 sackhl 24 3 0 3 3 3 0 1 0 8 0 tcpqe 32 317 0 317 35 34 1 1 0 8 1 tcpcb 808 5648 0 5629 156 153 3 13 0 8 0 arp 120 183 0 165 1 0 1 1 0 8 0 ipq 40 7 0 7 2 2 0 1 0 8 0 ipqe 40 21 0 21 2 2 0 1 0 8 0 inpcb 368 13431 0 13407 190 187 3 17 0 8 0 nd6 136 193 0 165 2 0 2 2 0 8 0 pkpcb 40 134 0 134 14 14 0 1 0 8 0 kcovpl 48 56 0 48 1 0 1 1 0 8 0 ppxss 1256 55 0 55 17 16 1 1 0 8 1 pffrag 232 93 0 93 3 3 0 1 0 482 0 pffrnode 88 93 0 93 3 3 0 1 0 8 0 pffrent 40 590 0 590 5 5 0 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfstitem 24 708 0 699 1 0 1 1 0 8 0 pfstkey 128 708 0 699 3 2 1 2 0 8 0 pfstate 376 708 0 699 17 14 3 6 0 8 0 pfrule 1344 21 0 16 2 1 1 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 2784 0 2347 60 30 30 30 0 8 0 art_table 32 2785 0 2347 4 0 4 4 0 8 0 art_node 16 833 0 735 1 0 1 1 0 8 0 sysvmsgpl 40 6 0 6 2 2 0 1 0 8 0 semupl 112 3 0 3 1 1 0 1 0 8 0 semapl 112 8 0 0 1 0 1 1 0 8 0 shmpl 112 128 0 0 4 0 4 4 0 8 0 dirhash 1024 67 0 50 3 0 3 3 0 8 0 dino2pl 256 30632 0 29080 98 0 98 98 0 8 0 ffsino 272 30632 0 29080 104 0 104 104 0 8 0 nchpl 144 60081 0 58429 63 0 63 63 0 8 0 uvmvnodes 80 7030 0 0 144 0 144 144 0 8 0 vnodes 216 7030 0 0 391 0 391 391 0 8 0 namei 1024 196763 0 196763 8 7 1 2 0 8 1 percpumem 16 224 0 176 1 0 1 1 0 8 0 vcpupl 2048 245 0 0 31 0 31 31 0 8 0 vmpool 696 354 0 109 27 4 23 23 0 8 0 kstatmem 264 380 0 352 14 11 3 3 0 8 0 scxspl 216 172290 0 172290 27 26 1 8 1 8 1 plimitpl 152 2312 0 2296 1 0 1 1 0 8 0 sigapl 424 23050 0 22981 20 12 8 9 0 8 0 futexpl 64 197455 0 197455 3 2 1 1 0 8 1 knotepl 120 1158 0 0 18 0 18 18 0 8 0 kqueuepl 216 2941 0 2933 52 51 1 5 0 8 0 pipepl 320 27535 0 27507 250 242 8 14 0 8 5 fdescpl 496 22737 0 22708 27 23 4 5 0 8 0 filepl 152 165148 0 164904 242 226 16 25 0 8 5 lockfpl 104 11657 0 11655 18 17 1 2 0 8 0 lockfspl 48 5050 0 5048 1 0 1 1 0 8 0 sessionpl 144 73 0 56 1 0 1 1 0 8 0 pgrppl 48 294 0 277 1 0 1 1 0 8 0 ucredpl 104 16365 0 16344 1 0 1 1 0 8 0 zombiepl 144 22982 0 22981 4 3 1 1 0 8 0 processpl 1072 23050 0 22981 5 0 5 5 0 8 0 procpl 680 62054 0 61966 12 3 9 9 0 8 0 srpgc 96 72 0 72 24 24 0 1 0 8 0 sosppl 168 235 0 232 16 15 1 1 0 8 0 sockpl 488 26840 0 26795 504 492 12 41 0 8 5 mcl64k 65536 31 0 0 4 1 3 3 0 8 0 mcl16k 16384 33 0 0 3 0 3 3 0 8 0 mcl12k 12288 27 0 0 2 0 2 2 0 8 0 mcl9k 9216 17 0 0 2 0 2 2 0 8 0 mcl8k 8192 32 0 0 3 0 3 3 0 8 0 mcl4k 4096 54 0 0 5 2 3 3 0 8 0 mcl2k2 2112 14 0 0 1 0 1 1 0 8 0 mcl2k 2048 555 0 0 32 10 22 31 0 8 0 mtagpl 96 1406 0 0 31 0 31 31 0 8 0 mbufpl 256 2419 0 0 128 0 128 128 0 8 0 bufpl 288 34127 0 27043 507 0 507 507 0 8 0 anonpl 24 2079412 0 2064186 229 113 116 120 0 186 0 amapchunkpl 152 695086 0 694243 106 63 43 49 0 158 0 amappl16 200 37345 0 36886 136 106 30 39 0 8 0 amappl15 192 59 0 59 4 4 0 1 0 8 0 amappl14 184 337 0 318 3 2 1 2 0 8 0 amappl13 176 95 0 93 1 0 1 1 0 8 0 amappl12 168 24175 0 24135 14 11 3 3 0 8 0 amappl11 160 105 0 91 1 0 1 1 0 8 0 amappl10 152 93 0 74 1 0 1 1 0 8 0 amappl9 144 307 0 307 24 24 0 1 0 8 0 amappl8 136 1068 0 878 7 0 7 7 0 8 0 amappl7 128 376 0 348 2 0 2 2 0 8 0 amappl6 120 1081 0 1059 1 0 1 1 0 8 0 amappl5 112 562 0 551 1 0 1 1 0 8 0 amappl4 104 996 0 961 4 2 2 2 0 8 0 amappl3 96 136070 0 135991 3 0 3 3 0 8 0 amappl2 88 24330 0 24252 3 1 2 3 0 8 0 amappl1 80 92111 0 91567 23 10 13 23 0 8 0 amappl 88 218818 0 218533 8 0 8 8 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 130 0 0 3 0 3 3 0 8 0 uaddrrnd 24 23092 0 22818 2 0 2 2 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 23092 0 22818 2 0 2 2 0 8 0 vmmpekpl 168 244372 0 244250 6 0 6 6 0 8 0 vmmpepl 168 1338456 0 1335657 538 392 146 160 0 357 0 vmsppl 464 23091 0 22818 36 1 35 35 0 8 0 rwobjpl 56 317123 0 308220 136 10 126 126 0 8 0 pdppl 4096 46191 0 45881 1137 823 314 314 0 8 4 pvpl 32 38084 0 0 308 0 308 308 0 265 0 pmappl 248 23091 0 22818 19 1 18 18 0 8 0 extentpl 40 56 0 38 1 0 1 1 0 8 0 phpool 112 2547 0 1278 37 0 37 37 0 8 0 ddb{0}> machine ddbcpu 0 Invalid cpu 0 ddb{0}> trace db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 witness_checkorder(fffffd806eeb85d8,9,0) at witness_checkorder+0x10b7 witness_debugger sys/kern/subr_witness.c:2510 [inline] witness_checkorder(fffffd806eeb85d8,9,0) at witness_checkorder+0x10b7 sys/kern/subr_witness.c:1110 rw_enter_write(fffffd806eeb85c8) at rw_enter_write+0x5c sys/kern/kern_rwlock.c:128 route_input(fffffd80625ae700,0,2) at route_input+0x101 sys/net/rtsock.c:520 rtm_send(fffffd80798c8390,1,0,0) at rtm_send+0x1b2 sys/net/rtsock.c:1744 rt_clone(ffff800036f9c8f0,ffff800036f9c968,0) at rt_clone+0xd1 sys/net/route.c:279 rtalloc(ffff800036f9c968,1,0) at rtalloc+0x91 rt_match sys/net/route.c:248 [inline] rtalloc(ffff800036f9c968,1,0) at rtalloc+0x91 sys/net/route.c:377 icmp_reflect(fffffd8067889b00,0,0) at icmp_reflect+0x1bd sys/netinet/ip_icmp.c:740 icmp_error(fffffd80625ae700,3,3,0,0) at icmp_error+0x61 sys/netinet/ip_icmp.c:300 pf_test_rule(ffff800036f9cc28,ffff800036f9cd20,ffff800036f9cd30,ffff800036f9cd10,ffff800036f9ccf0,fffffd807618eaf0) at pf_test_rule+0x98d sys/net/pf.c:4476 pf_test(2,1,ffff800000e1f800,ffff800036f9ce58) at pf_test+0x1d1c sys/net/pf.c:7676 ip_input_if(ffff800036f9ce58,ffff800036f9ce64,4,0,ffff800000e1f800) at ip_input_if+0x131 sys/netinet/ip_input.c:417 ipv4_input(ffff800000e1f800,fffffd807618ea00) at ipv4_input+0x4c sys/netinet/ip_input.c:291 ether_input(ffff800000e1f800,fffffd807618ea00) at ether_input+0x58a tun_dev_write(5d02,ffff800036f9d1c8,ffff80002cf05000,2) at tun_dev_write+0x203 sys/net/if_tun.c:914 spec_write(ffff800036f9d020) at spec_write+0xd9 sys/kern/spec_vnops.c:302 VOP_WRITE(fffffd807a2d9bd0,ffff800036f9d1c8,11,fffffd807f7d7208) at VOP_WRITE+0xc3 sys/kern/vfs_vops.c:245 vn_write(fffffd80789687c0,ffff800036f9d1c8,0) at vn_write+0x15b sys/kern/vfs_vnops.c:408 dofilewritev(ffff80002a2682b8,c8,ffff800036f9d1c8,0,ffff800036f9d2b0) at dofilewritev+0x1a0 sys/kern/sys_generic.c:375 sys_write(ffff80002a2682b8,ffff800036f9d260,ffff800036f9d2b0) at sys_write+0x87 sys/kern/sys_generic.c:295 syscall(ffff800036f9d330) at syscall+0x50f mi_syscall sys/sys/syscall_mi.h:103 [inline] syscall(ffff800036f9d330) at syscall+0x50f sys/arch/amd64/amd64/trap.c:606 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xc97ad866630, count: -22 ddb{0}> machine ddbcpu 1 Stopped at x86_ipi_db+0x1e: addq $0x8,%rsp ddb{1}> trace x86_ipi_db(ffff800029d2bff0) at x86_ipi_db+0x1e sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 acpicpu_idle() at acpicpu_idle+0x312 sys/dev/acpi/acpicpu.c:1206 sched_idle(ffff800029d2bff0) at sched_idle+0x41e sys/kern/kern_sched.c:191 end trace frame: 0x0, count: -5