witness: lock order reversal: 1st 0xffffffff82bcab90 pf_lock (pf_lock) 2nd 0xfffffd806d74a218 solock (&so->so_lock) lock order "&so->so_lock"(rwlock) -> "pf_lock"(rwlock) first seen at: #0 rw_enter_write+0x5c sys/kern/kern_rwlock.c:128 #1 pf_remove_divert_state+0xbf sys/net/pf.c:1833 #2 in_pcbdisconnect+0x2c sys/netinet/in_pcb.c:578 #3 udp_disconnect+0xf1 sys/netinet/udp_usrreq.c:1189 #4 soconnect+0xc1 pru_disconnect sys/sys/protosw.h:334 [inline] #4 soconnect+0xc1 sodisconnect sys/kern/uipc_socket.c:521 [inline] #4 soconnect+0xc1 sys/kern/uipc_socket.c:485 #5 sys_connect+0x207 sys/kern/uipc_syscalls.c:422 #6 syscall+0x50f mi_syscall sys/sys/syscall_mi.h:103 [inline] #6 syscall+0x50f sys/arch/amd64/amd64/trap.c:606 #7 Xsyscall+0x128 lock order "pf_lock"(rwlock) -> "&so->so_lock"(rwlock) first seen at: #0 rw_enter_write+0x5c sys/kern/kern_rwlock.c:128 #1 route_input+0x101 sys/net/rtsock.c:520 #2 rtm_send+0x1b2 sys/net/rtsock.c:1744 #3 rt_clone+0xd1 sys/net/route.c:279 #4 rtalloc+0x91 rt_match sys/net/route.c:248 [inline] #4 rtalloc+0x91 sys/net/route.c:377 #5 icmp_reflect+0x1bd sys/netinet/ip_icmp.c:740 #6 icmp_error+0x61 sys/netinet/ip_icmp.c:300 #7 pf_test_rule+0x98d sys/net/pf.c:4476 #8 pf_test+0x1d1c sys/net/pf.c:7676 #9 ip_input_if+0x131 sys/netinet/ip_input.c:417 #10 ipv4_input+0x4c sys/netinet/ip_input.c:291 #11 ether_input+0x58a #12 tun_dev_write+0x203 sys/net/if_tun.c:914 #13 spec_write+0xd9 sys/kern/spec_vnops.c:302 #14 VOP_WRITE+0xc3 sys/kern/vfs_vops.c:245 #15 vn_write+0x15b sys/kern/vfs_vnops.c:408 #16 dofilewritev+0x1a0 sys/kern/sys_generic.c:375 #17 sys_write+0x87 sys/kern/sys_generic.c:295 #18 syscall+0x50f mi_syscall sys/sys/syscall_mi.h:103 [inline] #18 syscall+0x50f sys/arch/amd64/amd64/trap.c:606 Stopped at db_enter+0x1c: addq $0x8,%rsp ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic the kernel did not panic ddb{1}> trace db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 witness_checkorder(fffffd806d74a218,9,0) at witness_checkorder+0x10b7 witness_debugger sys/kern/subr_witness.c:2510 [inline] witness_checkorder(fffffd806d74a218,9,0) at witness_checkorder+0x10b7 sys/kern/subr_witness.c:1110 rw_enter_write(fffffd806d74a208) at rw_enter_write+0x5c sys/kern/kern_rwlock.c:128 route_input(fffffd806d469c00,0,2) at route_input+0x101 sys/net/rtsock.c:520 rtm_send(fffffd8068360478,1,0,0) at rtm_send+0x1b2 sys/net/rtsock.c:1744 rt_clone(ffff80002e3057b0,ffff80002e305828,0) at rt_clone+0xd1 sys/net/route.c:279 rtalloc(ffff80002e305828,1,0) at rtalloc+0x91 rt_match sys/net/route.c:248 [inline] rtalloc(ffff80002e305828,1,0) at rtalloc+0x91 sys/net/route.c:377 icmp_reflect(fffffd806d47f700,0,0) at icmp_reflect+0x1bd sys/netinet/ip_icmp.c:740 icmp_error(fffffd806d469c00,3,3,0,0) at icmp_error+0x61 sys/netinet/ip_icmp.c:300 pf_test_rule(ffff80002e305ae8,ffff80002e305be0,ffff80002e305bf0,ffff80002e305bd0,ffff80002e305bb0,fffffd80645c19f0) at pf_test_rule+0x98d sys/net/pf.c:4476 pf_test(2,1,ffff800000d45000,ffff80002e305d18) at pf_test+0x1d1c sys/net/pf.c:7676 ip_input_if(ffff80002e305d18,ffff80002e305d24,4,0,ffff800000d45000) at ip_input_if+0x131 sys/netinet/ip_input.c:417 ipv4_input(ffff800000d45000,fffffd80645c1900) at ipv4_input+0x4c sys/netinet/ip_input.c:291 ether_input(ffff800000d45000,fffffd80645c1900) at ether_input+0x58a tun_dev_write(5d03,ffff80002e306088,ffff80002f159000,2) at tun_dev_write+0x203 sys/net/if_tun.c:914 spec_write(ffff80002e305ee0) at spec_write+0xd9 sys/kern/spec_vnops.c:302 VOP_WRITE(fffffd806d3fed88,ffff80002e306088,11,fffffd807f7d6548) at VOP_WRITE+0xc3 sys/kern/vfs_vops.c:245 vn_write(fffffd80649e9d30,ffff80002e306088,0) at vn_write+0x15b sys/kern/vfs_vnops.c:408 dofilewritev(ffff80002a1e5aa8,c8,ffff80002e306088,0,ffff80002e306170) at dofilewritev+0x1a0 sys/kern/sys_generic.c:375 sys_write(ffff80002a1e5aa8,ffff80002e306120,ffff80002e306170) at sys_write+0x87 sys/kern/sys_generic.c:295 syscall(ffff80002e3061f0) at syscall+0x50f mi_syscall sys/sys/syscall_mi.h:103 [inline] syscall(ffff80002e3061f0) at syscall+0x50f sys/arch/amd64/amd64/trap.c:606 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xa2ecd224270, count: -22 ddb{1}> show registers rdi 0x3 rsi 0x40000 acpi_pdirpa+0x2be63 rbp 0xffff80002e3053b0 rbx 0x3 rdx 0xffff800000d25900 rcx 0xffff80002a1e5aa8 rax 0x3ffff acpi_pdirpa+0x2be62 r8 0xffff80002e305320 r9 0x8080808080808080 r10 0xfaa9bb39bf454dc r11 0x3de4abc484161b09 r12 0xfffffd8003a06e60 r13 0 r14 0xfffffd80039fe2d0 r15 0xfffffd80031d69c0 rip 0xffffffff81825f8c db_enter+0x1c cs 0x8 rflags 0x246 rsp 0xffff80002e3053a0 ss 0x10 db_enter+0x1c: addq $0x8,%rsp ddb{1}> show proc PROC (syz-executor.3) tid=312794 pid=90897 tcnt=2 stat=onproc flags process=0 proc=4000000 runpri=32, usrpri=84, slppri=32, nice=20 wchan=0x0, wmesg=, ps_single=0x0 forw=0xffffffffffffffff, list=0xffff80002a1e52b0,0xffff80002a263810 process=0xffff8000fffed0e0 user=0xffff80002e301000, vmspace=0xfffffd80649f4ce8 estcpu=36, cpticks=0, pctcpu=0.0, user=0, sys=0, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 28935 299058 12919 0 2 0 syz-executor.6 90897 110778 34199 0 2 0 syz-executor.3 *90897 312794 34199 0 7 0x4000000 syz-executor.3 34132 223988 92655 0 7 0 syz-executor.5 34132 465212 92655 0 2 0x4000000 syz-executor.5 34132 179128 92655 0 3 0x4000080 fsleep syz-executor.5 45555 495999 88890 0 2 0 syz-executor.0 45555 242799 88890 0 3 0x4000000 biowait syz-executor.0 18776 499646 10492 0 2 0 syz-executor.2 18776 273082 10492 0 3 0x4000080 fsleep syz-executor.2 18776 474479 10492 0 3 0x4000080 pipewr syz-executor.2 12919 61343 89086 0 3 0x82 nanoslp syz-executor.6 54752 494563 89086 0 3 0x2 biowait syz-executor.7 73175 287277 1 0 3 0x100083 ttyin getty 9032 175214 0 0 3 0x14280 nfsidl nfsio 31693 308862 0 0 3 0x14280 nfsidl nfsio 62516 188230 0 0 3 0x14280 nfsidl nfsio 86936 371691 0 0 3 0x14280 nfsidl nfsio 58073 349290 0 0 3 0x14280 nfsidl nfsio 50269 331367 0 0 3 0x14280 nfsidl nfsio 86696 39034 0 0 3 0x14280 nfsidl nfsio 50175 481132 0 0 3 0x14280 nfsidl nfsio 53812 21274 0 0 3 0x14280 nfsidl nfsio 49650 470715 0 0 3 0x14280 nfsidl nfsio 95038 402107 0 0 3 0x14280 nfsidl nfsio 6583 129325 0 0 3 0x14280 nfsidl nfsio 43073 223974 0 0 3 0x14280 nfsidl nfsio 77921 183725 0 0 3 0x14280 nfsidl nfsio 9457 497054 0 0 3 0x14280 nfsidl nfsio 16640 483553 0 0 3 0x14280 nfsidl nfsio 43977 415581 0 0 3 0x14280 nfsidl nfsio 27086 379273 0 0 3 0x14280 nfsidl nfsio 61602 165637 0 0 3 0x14280 nfsidl nfsio 8828 462909 0 0 3 0x14280 nfsidl nfsio 92655 385188 89086 0 3 0x82 nanoslp syz-executor.5 10492 220653 89086 0 3 0x82 nanoslp syz-executor.2 17403 253958 89086 0 3 0x2 getblk syz-executor.1 30775 161591 89086 0 3 0x82 nanoslp syz-executor.4 67537 455423 0 0 3 0x14200 bored sosplice 44481 4966 65363 0 3 0x100082 netio arp 65363 404233 1 0 3 0x10008a sigsusp sh 34199 523662 89086 0 3 0x82 nanoslp syz-executor.3 88890 280827 89086 0 3 0x82 nanoslp syz-executor.0 89086 352533 70939 0 3 0x2000082 thrsleep syz-fuzzer 89086 327365 70939 0 3 0x6000082 nanoslp syz-fuzzer 89086 41999 70939 0 3 0x6000082 wait syz-fuzzer 89086 490203 70939 0 3 0x6000082 wait syz-fuzzer 89086 394194 70939 0 3 0x6000082 wait syz-fuzzer 89086 404758 70939 0 3 0x6000082 wait syz-fuzzer 89086 261339 70939 0 3 0x6000082 thrsleep syz-fuzzer 89086 283053 70939 0 3 0x6000082 wait syz-fuzzer 89086 11397 70939 0 3 0x6000082 thrsleep syz-fuzzer 89086 358601 70939 0 3 0x6000082 wait syz-fuzzer 89086 421158 70939 0 3 0x6000082 wait syz-fuzzer 89086 162708 70939 0 3 0x6000082 thrsleep syz-fuzzer 89086 168736 70939 0 3 0x6000082 wait syz-fuzzer 89086 424869 70939 0 3 0x6000082 kqread syz-fuzzer 89086 91377 70939 0 3 0x6000082 thrsleep syz-fuzzer 89086 473469 70939 0 3 0x6000082 thrsleep syz-fuzzer 70939 348711 76093 0 3 0x10008a sigsusp ksh 76093 300683 31368 0 3 0x9a kqread sshd 31368 308675 1 0 3 0x88 kqread sshd 98629 47205 4140 74 3 0x1100092 bpf pflogd 4140 435381 1 0 3 0x80 netio pflogd 22793 86336 65836 73 3 0x1100090 kqread syslogd 65836 109661 1 0 3 0x100082 netio syslogd 88482 451712 1 0 3 0x100080 kqread resolvd 34470 2449 42230 77 3 0x100092 kqread dhcpleased 49815 220351 42230 77 3 0x100092 kqread dhcpleased 42230 463255 1 0 3 0x80 kqread dhcpleased 48442 395495 0 0 3 0x14200 bored smr 30854 129770 0 0 2 0x14200 zerothread 82674 180170 0 0 3 0x14200 aiodoned aiodoned 17411 471850 0 0 3 0x14200 syncer update 34056 377192 0 0 3 0x14200 cleaner cleaner 42419 138126 0 0 3 0x14200 reaper reaper 92414 144115 0 0 3 0x14200 pgdaemon pagedaemon 87200 371545 0 0 3 0x14200 bored viomb 77729 493279 0 0 3 0x40014200 acpi0 acpi0 90768 206547 0 0 3 0x40014200 idle1 48105 150170 0 0 3 0x14200 bored softnet3 44633 446968 0 0 3 0x14200 bored softnet2 89732 160683 0 0 3 0x14200 bored softnet1 42304 422473 0 0 3 0x14200 bored softnet0 83854 389104 0 0 3 0x14200 bored systqmp 77215 306066 0 0 3 0x14200 bored systq 37541 172202 0 0 3 0x14200 tmoslp softclockmp 96001 69417 0 0 3 0x40014200 tmoslp softclock 96615 266121 0 0 3 0x40014200 idle0 1 227060 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{1}> show all locks Process 90897 (syz-executor.3) thread 0xffff80002a1e5aa8 (312794) exclusive rwlock pf_lock r = 0 (0xffffffff82bcab90) #0 witness_lock+0x447 #1 pf_test+0x1cfd #2 ip_input_if+0x131 sys/netinet/ip_input.c:417 #3 ipv4_input+0x4c sys/netinet/ip_input.c:291 #4 ether_input+0x58a #5 tun_dev_write+0x203 sys/net/if_tun.c:914 #6 spec_write+0xd9 sys/kern/spec_vnops.c:302 #7 VOP_WRITE+0xc3 sys/kern/vfs_vops.c:245 #8 vn_write+0x15b sys/kern/vfs_vnops.c:408 #9 dofilewritev+0x1a0 sys/kern/sys_generic.c:375 #10 sys_write+0x87 sys/kern/sys_generic.c:295 #11 syscall+0x50f mi_syscall sys/sys/syscall_mi.h:103 [inline] #11 syscall+0x50f sys/arch/amd64/amd64/trap.c:606 #12 Xsyscall+0x128 exclusive rwlock netlock r = 0 (0xffffffff82c31500) #0 witness_lock+0x447 #1 tun_dev_write+0x1f8 sys/net/if_tun.c:913 #2 spec_write+0xd9 sys/kern/spec_vnops.c:302 #3 VOP_WRITE+0xc3 sys/kern/vfs_vops.c:245 #4 vn_write+0x15b sys/kern/vfs_vnops.c:408 #5 dofilewritev+0x1a0 sys/kern/sys_generic.c:375 #6 sys_write+0x87 sys/kern/sys_generic.c:295 #7 syscall+0x50f mi_syscall sys/sys/syscall_mi.h:103 [inline] #7 syscall+0x50f sys/arch/amd64/amd64/trap.c:606 #8 Xsyscall+0x128 exclusive kernel_lock &kernel_lock r = 0 (0xffffffff82d352e8) #0 witness_lock+0x447 #1 vn_write+0x46 sys/kern/vfs_vnops.c:393 #2 dofilewritev+0x1a0 sys/kern/sys_generic.c:375 #3 sys_write+0x87 sys/kern/sys_generic.c:295 #4 syscall+0x50f mi_syscall sys/sys/syscall_mi.h:103 [inline] #4 syscall+0x50f sys/arch/amd64/amd64/trap.c:606 #5 Xsyscall+0x128 Process 45555 (syz-executor.0) thread 0xffff80002a1e5558 (242799) exclusive rrwlock inode r = 0 (0xfffffd8068d025f8) #0 witness_lock+0x447 #1 rw_enter+0x3c8 sys/kern/kern_rwlock.c:309 #2 rrw_enter+0x8c sys/kern/kern_rwlock.c:464 #3 VOP_LOCK+0x8b sys/kern/vfs_vops.c:518 #4 ufs_ihashins+0x46 sys/ufs/ufs/ufs_ihash.c:140 #5 ffs_vget+0x141 sys/ufs/ffs/ffs_vfsops.c:1343 #6 ffs_inode_alloc+0x1c2 sys/ufs/ffs/ffs_alloc.c:394 #7 ufs_makeinode+0xb2 sys/ufs/ufs/ufs_vnops.c:1817 #8 ufs_create+0x45 sys/ufs/ufs/ufs_vnops.c:151 #9 VOP_CREATE+0xc0 sys/kern/vfs_vops.c:103 #10 vn_open+0x27b sys/kern/vfs_vnops.c:116 #11 doopenat+0x26e sys/kern/vfs_syscalls.c:1126 #12 syscall+0x4e6 sys/arch/amd64/amd64/trap.c:606 #13 Xsyscall+0x128 exclusive rrwlock inode r = 0 (0xfffffd807e8d55f8) #0 witness_lock+0x447 #1 rw_enter+0x3c8 sys/kern/kern_rwlock.c:309 #2 rrw_enter+0x8c sys/kern/kern_rwlock.c:464 #3 VOP_LOCK+0x8b sys/kern/vfs_vops.c:518 #4 vn_lock+0x84 sys/kern/vfs_vnops.c:564 #5 vfs_lookup+0xd5 sys/kern/vfs_lookup.c:418 #6 namei+0x55a sys/kern/vfs_lookup.c:250 #7 vn_open+0x17b sys/kern/vfs_vnops.c:107 #8 doopenat+0x26e sys/kern/vfs_syscalls.c:1126 #9 syscall+0x4e6 sys/arch/amd64/amd64/trap.c:606 #10 Xsyscall+0x128 Process 54752 (syz-executor.7) thread 0xffff80002a2a12a8 (494563) exclusive rrwlock inode r = 0 (0xfffffd8068d02a38) #0 witness_lock+0x447 #1 rw_enter+0x3c8 sys/kern/kern_rwlock.c:309 #2 rrw_enter+0x8c sys/kern/kern_rwlock.c:464 #3 VOP_LOCK+0x8b sys/kern/vfs_vops.c:518 #4 ufs_ihashins+0x46 sys/ufs/ufs/ufs_ihash.c:140 #5 ffs_vget+0x141 sys/ufs/ffs/ffs_vfsops.c:1343 #6 ffs_inode_alloc+0x1c2 sys/ufs/ffs/ffs_alloc.c:394 #7 ufs_mkdir+0xf8 sys/ufs/ufs/ufs_vnops.c:1149 #8 VOP_MKDIR+0xc3 sys/kern/vfs_vops.c:388 #9 domkdirat+0x125 sys/kern/vfs_syscalls.c:3073 #10 syscall+0x4e6 sys/arch/amd64/amd64/trap.c:606 #11 Xsyscall+0x128 exclusive rrwlock inode r = 0 (0xfffffd80737e7b38) #0 witness_lock+0x447 #1 rw_enter+0x3c8 sys/kern/kern_rwlock.c:309 #2 rrw_enter+0x8c sys/kern/kern_rwlock.c:464 #3 VOP_LOCK+0x8b sys/kern/vfs_vops.c:518 #4 vn_lock+0x84 sys/kern/vfs_vnops.c:564 #5 vfs_lookup+0xd5 sys/kern/vfs_lookup.c:418 #6 namei+0x55a sys/kern/vfs_lookup.c:250 #7 domkdirat+0x79 sys/kern/vfs_syscalls.c:3058 #8 syscall+0x4e6 sys/arch/amd64/amd64/trap.c:606 #9 Xsyscall+0x128 Process 17403 (syz-executor.1) thread 0xffff80002a26e540 (253958) exclusive rrwlock inode r = 0 (0xfffffd806e4c30a8) #0 witness_lock+0x447 #1 rw_enter+0x3c8 sys/kern/kern_rwlock.c:309 #2 rrw_enter+0x8c sys/kern/kern_rwlock.c:464 #3 VOP_LOCK+0x8b sys/kern/vfs_vops.c:518 #4 vn_lock+0x84 sys/kern/vfs_vnops.c:564 #5 vget+0x200 sys/kern/vfs_subr.c:676 #6 ufs_ihashget+0x121 sys/ufs/ufs/ufs_ihash.c:119 #7 ffs_vget+0x7c sys/ufs/ffs/ffs_vfsops.c:1314 #8 ufs_lookup+0x13ba sys/ufs/ufs/ufs_lookup.c:487 #9 VOP_LOOKUP+0x5c sys/kern/vfs_vops.c:85 #10 vfs_lookup+0x6e2 sys/kern/vfs_lookup.c:566 #11 namei+0x55a sys/kern/vfs_lookup.c:250 #12 dounlinkat+0x9d sys/kern/vfs_syscalls.c:1847 #13 syscall+0x4e6 sys/arch/amd64/amd64/trap.c:606 #14 Xsyscall+0x128 exclusive rrwlock inode r = 0 (0xfffffd80648251b0) #0 witness_lock+0x447 #1 rw_enter+0x3c8 sys/kern/kern_rwlock.c:309 #2 rrw_enter+0x8c sys/kern/kern_rwlock.c:464 #3 VOP_LOCK+0x8b sys/kern/vfs_vops.c:518 #4 vn_lock+0x84 sys/kern/vfs_vnops.c:564 #5 vfs_lookup+0xd5 sys/kern/vfs_lookup.c:418 #6 namei+0x55a sys/kern/vfs_lookup.c:250 #7 dounlinkat+0x9d sys/kern/vfs_syscalls.c:1847 #8 syscall+0x4e6 sys/arch/amd64/amd64/trap.c:606 #9 Xsyscall+0x128 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10211 6498K 6968K 166960K 19168 0 pcb 13 18K 20K 166960K 782 0 rtable 196 8K 9K 166960K 1055 0 pf 34 9K 10K 166960K 207 0 ifaddr 40 15K 17K 166960K 178 0 ifgroup 59 2K 2K 166960K 295 0 sysctl 3 0K 0K 166960K 9 0 counters 66 36K 36K 166960K 216 0 ioctlops 0 0K 4K 166960K 1792 0 iov 0 0K 32K 166960K 684 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1489 93K 94K 166960K 5085 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 1K 9K 166960K 83 0 VM map 2 1K 1K 166960K 2 0 sem 12 1K 1K 166960K 108 0 dirhash 12 2K 2K 166960K 66 0 ACPI 1697 195K 286K 166960K 12548 0 file desc 17 61K 93K 166960K 7960 0 sigio 1 0K 0K 166960K 169 0 proc 69 91K 115K 166960K 1283 0 subproc 117 7K 7K 166960K 316 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 622 0 in_multi 76 5K 7K 166960K 357 0 ether_multi 1 0K 0K 166960K 8 0 mrt 1 0K 0K 166960K 4 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 103 466K 466K 166960K 103 0 exec 0 0K 1K 166960K 1294 0 tdb 3 0K 0K 166960K 3 0 pagedep 1 8K 8K 166960K 1 0 inodedep 1 32K 32K 166960K 1 0 newblk 1 0K 0K 166960K 1 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 429 196K 201K 166960K 80843 0 UVM aobj 131 6K 6K 166960K 135 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 0K 166960K 141 0 NDP 13 0K 1K 166960K 134 0 temp 74 5924K 137000K 166960K 113322 0 kqueue 12 18K 26K 166960K 495 0 SYN cache 2 16K 16K 166960K 2 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 24 0 0 1 0 1 1 0 8 0 rtpcb 120 349 0 345 4 3 1 3 0 8 0 rtentry 112 321 0 233 4 0 4 4 0 8 0 unpcb 144 3957 0 3942 42 41 1 6 0 8 0 syncache 312 65 0 65 14 14 0 1 0 8 0 tcpqe 32 172 0 172 11 11 0 1 0 8 0 tcpcb 808 2165 0 2145 74 68 6 15 0 8 0 arp 120 63 0 49 1 0 1 1 0 8 0 inpcb 368 5685 0 5662 112 106 6 17 0 8 0 nd6 136 70 0 52 1 0 1 1 0 8 0 pkpcb 40 216 0 216 5 5 0 2 0 8 0 kcovpl 48 24 0 15 1 0 1 1 0 8 0 ppxss 1256 39 0 39 11 11 0 1 0 8 0 pffrag 232 51 0 50 4 3 1 1 0 482 0 pffrnode 88 51 0 50 4 3 1 1 0 8 0 pffrent 40 275 0 274 4 3 1 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfstitem 24 213 0 205 1 0 1 1 0 8 0 pfstkey 128 213 0 205 1 0 1 1 0 8 0 pfstate 376 213 0 205 9 7 2 4 0 8 0 pfrule 1344 21 0 16 2 1 1 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 1151 0 782 35 8 27 30 0 8 0 art_table 32 1152 0 782 4 0 4 4 0 8 0 art_node 16 313 0 235 1 0 1 1 0 8 0 sysvmsgpl 40 11 0 6 1 0 1 1 0 8 0 semupl 112 6 0 6 2 2 0 1 0 8 0 semapl 112 104 0 94 1 0 1 1 0 8 0 shmpl 112 132 0 4 4 0 4 4 0 8 0 dirhash 1024 53 0 36 3 0 3 3 0 8 0 dino2pl 256 12562 0 11086 93 0 93 93 0 8 0 ffsino 272 12562 0 11086 99 0 99 99 0 8 0 nchpl 144 25551 0 23904 63 0 63 63 0 8 0 uvmvnodes 80 5926 0 0 121 0 121 121 0 8 0 vnodes 216 5926 0 0 330 0 330 330 0 8 0 namei 1024 86372 0 86370 5 4 1 2 0 8 0 percpumem 16 122 0 75 1 0 1 1 0 8 0 vcpupl 2048 30 0 0 4 0 4 4 0 8 0 vmpool 696 38 0 8 5 2 3 3 0 8 0 kstatmem 264 176 0 150 2 0 2 2 0 8 0 scxspl 216 75105 0 75103 27 26 1 8 1 8 0 plimitpl 152 1015 0 998 1 0 1 1 0 8 0 sigapl 424 8518 0 8448 18 9 9 16 0 8 0 futexpl 64 66750 0 66748 3 2 1 1 0 8 0 knotepl 120 430 0 0 12 1 11 12 0 8 0 kqueuepl 216 1370 0 1362 39 38 1 5 0 8 0 pipepl 320 1544 0 1512 53 47 6 11 0 8 3 fdescpl 496 8243 0 8213 5 0 5 5 0 8 0 filepl 152 54243 0 53981 118 104 14 23 0 8 1 lockfpl 104 7114 0 7112 8 7 1 4 0 8 0 lockfspl 48 1732 0 1730 2 1 1 2 0 8 0 sessionpl 144 43 0 25 1 0 1 1 0 8 0 pgrppl 48 198 0 180 1 0 1 1 0 8 0 ucredpl 104 5871 0 5853 1 0 1 1 0 8 0 zombiepl 144 8449 0 8448 4 3 1 3 0 8 0 processpl 1072 8518 0 8448 10 4 6 10 0 8 0 procpl 680 22855 0 22764 19 10 9 14 0 8 0 srpgc 96 21 0 21 6 6 0 1 0 8 0 sosppl 168 50 0 50 13 13 0 1 0 8 0 sockpl 488 10352 0 10310 217 207 10 37 0 8 0 mcl64k 65536 25 0 0 3 0 3 3 0 8 0 mcl16k 16384 18 0 0 3 1 2 3 0 8 0 mcl12k 12288 25 0 0 2 0 2 2 0 8 0 mcl9k 9216 15 0 0 2 0 2 2 0 8 0 mcl8k 8192 25 0 0 4 1 3 3 0 8 0 mcl4k 4096 50 0 0 5 1 4 4 0 8 0 mcl2k2 2112 10 0 0 1 0 1 1 0 8 0 mcl2k 2048 336 0 0 32 14 18 32 0 8 0 mtagpl 96 310 0 0 6 0 6 6 0 8 0 mbufpl 256 1498 0 0 65 0 65 65 0 8 0 bufpl 288 17505 0 11183 452 0 452 452 0 8 0 anonpl 24 891686 0 878195 200 97 103 117 0 186 0 amapchunkpl 152 256288 0 255443 112 74 38 48 0 158 0 amappl16 200 18651 0 18225 121 96 25 40 0 8 0 amappl15 192 19 0 19 1 1 0 1 0 8 0 amappl14 184 210 0 191 2 1 1 2 0 8 0 amappl13 176 13 0 13 2 2 0 1 0 8 0 amappl12 168 9162 0 9127 4 2 2 3 0 8 0 amappl11 160 56 0 40 1 0 1 1 0 8 0 amappl10 152 40 0 26 1 0 1 1 0 8 0 amappl9 144 245 0 245 19 18 1 1 0 8 1 amappl8 136 489 0 373 5 0 5 5 0 8 0 amappl7 128 264 0 235 2 0 2 2 0 8 0 amappl6 120 647 0 634 1 0 1 1 0 8 0 amappl5 112 254 0 243 1 0 1 1 0 8 0 amappl4 104 702 0 665 3 1 2 2 0 8 0 amappl3 96 49860 0 49769 5 2 3 3 0 8 0 amappl2 88 9023 0 8943 3 1 2 3 0 8 0 amappl1 80 38202 0 37613 23 9 14 23 0 8 0 amappl 88 79881 0 79635 7 0 7 7 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 134 0 4 3 0 3 3 0 8 0 uaddrrnd 24 8281 0 8221 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 8281 0 8221 1 0 1 1 0 8 0 vmmpekpl 168 66283 0 66214 4 0 4 4 0 8 0 vmmpepl 168 505749 0 503269 327 210 117 134 0 357 0 vmsppl 464 8280 0 8221 11 3 8 8 0 8 0 rwobjpl 56 130859 0 123152 121 9 112 113 0 8 0 pdppl 4096 16570 0 16472 670 566 104 104 0 8 6 pvpl 32 42101 0 0 342 2 340 340 0 265 0 pmappl 248 8280 0 8221 4 0 4 4 0 8 0 extentpl 40 56 0 38 1 0 1 1 0 8 0 phpool 112 1707 0 806 27 0 27 27 0 8 0 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+0x1e: addq $0x8,%rsp ddb{0}> trace x86_ipi_db(ffffffff82b82ff0) at x86_ipi_db+0x1e sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 __mp_lock(ffffffff82d350e0) at __mp_lock+0x122 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff82d350e0) at __mp_lock+0x122 sys/kern/kern_lock.c:147 intr_handler(ffff80002a269ec0,ffff800000077e80) at intr_handler+0x62 sys/arch/amd64/amd64/intr.c:532 Xintr_ioapic_edge16_untramp() at Xintr_ioapic_edge16_untramp+0x18f __mp_lock(ffffffff82d350e0) at __mp_lock+0x129 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff82d350e0) at __mp_lock+0x129 sys/kern/kern_lock.c:147 intr_handler(ffff80002a26a010,ffff80000007bd80) at intr_handler+0x62 sys/arch/amd64/amd64/intr.c:532 Xintr_ioapic_edge23_untramp() at Xintr_ioapic_edge23_untramp+0x18f __mp_lock(ffffffff82d350e0) at __mp_lock+0x122 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff82d350e0) at __mp_lock+0x122 sys/kern/kern_lock.c:147 syscall(ffff80002a26a1e0) at syscall+0x4cc mi_syscall sys/sys/syscall_mi.h:103 [inline] syscall(ffff80002a26a1e0) at syscall+0x4cc sys/arch/amd64/amd64/trap.c:606 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x75621c675670, count: -12 ddb{0}> machine ddbcpu 1 Stopped at db_enter+0x1c: addq $0x8,%rsp ddb{1}> trace db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 witness_checkorder(fffffd806d74a218,9,0) at witness_checkorder+0x10b7 witness_debugger sys/kern/subr_witness.c:2510 [inline] witness_checkorder(fffffd806d74a218,9,0) at witness_checkorder+0x10b7 sys/kern/subr_witness.c:1110 rw_enter_write(fffffd806d74a208) at rw_enter_write+0x5c sys/kern/kern_rwlock.c:128 route_input(fffffd806d469c00,0,2) at route_input+0x101 sys/net/rtsock.c:520 rtm_send(fffffd8068360478,1,0,0) at rtm_send+0x1b2 sys/net/rtsock.c:1744 rt_clone(ffff80002e3057b0,ffff80002e305828,0) at rt_clone+0xd1 sys/net/route.c:279 rtalloc(ffff80002e305828,1,0) at rtalloc+0x91 rt_match sys/net/route.c:248 [inline] rtalloc(ffff80002e305828,1,0) at rtalloc+0x91 sys/net/route.c:377 icmp_reflect(fffffd806d47f700,0,0) at icmp_reflect+0x1bd sys/netinet/ip_icmp.c:740 icmp_error(fffffd806d469c00,3,3,0,0) at icmp_error+0x61 sys/netinet/ip_icmp.c:300 pf_test_rule(ffff80002e305ae8,ffff80002e305be0,ffff80002e305bf0,ffff80002e305bd0,ffff80002e305bb0,fffffd80645c19f0) at pf_test_rule+0x98d sys/net/pf.c:4476 pf_test(2,1,ffff800000d45000,ffff80002e305d18) at pf_test+0x1d1c sys/net/pf.c:7676 ip_input_if(ffff80002e305d18,ffff80002e305d24,4,0,ffff800000d45000) at ip_input_if+0x131 sys/netinet/ip_input.c:417 ipv4_input(ffff800000d45000,fffffd80645c1900) at ipv4_input+0x4c sys/netinet/ip_input.c:291 ether_input(ffff800000d45000,fffffd80645c1900) at ether_input+0x58a tun_dev_write(5d03,ffff80002e306088,ffff80002f159000,2) at tun_dev_write+0x203 sys/net/if_tun.c:914 spec_write(ffff80002e305ee0) at spec_write+0xd9 sys/kern/spec_vnops.c:302 VOP_WRITE(fffffd806d3fed88,ffff80002e306088,11,fffffd807f7d6548) at VOP_WRITE+0xc3 sys/kern/vfs_vops.c:245 vn_write(fffffd80649e9d30,ffff80002e306088,0) at vn_write+0x15b sys/kern/vfs_vnops.c:408 dofilewritev(ffff80002a1e5aa8,c8,ffff80002e306088,0,ffff80002e306170) at dofilewritev+0x1a0 sys/kern/sys_generic.c:375 sys_write(ffff80002a1e5aa8,ffff80002e306120,ffff80002e306170) at sys_write+0x87 sys/kern/sys_generic.c:295 syscall(ffff80002e3061f0) at syscall+0x50f mi_syscall sys/sys/syscall_mi.h:103 [inline] syscall(ffff80002e3061f0) at syscall+0x50f sys/arch/amd64/amd64/trap.c:606 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xa2ecd224270, count: -22