witness: lock order reversal: 1st 0xffffffff82c39940 pf_lock (pf_lock) 2nd 0xfffffd806f67ed88 solock (&so->so_lock) lock order "&so->so_lock"(rwlock) -> "pf_lock"(rwlock) first seen at: #0 rw_enter_write+0x5c sys/kern/kern_rwlock.c:128 #1 pf_remove_divert_state+0xbf sys/net/pf.c:1833 #2 in_pcbdisconnect+0x2c sys/netinet/in_pcb.c:578 #3 udp_disconnect+0xf1 sys/netinet/udp_usrreq.c:1189 #4 soconnect+0xc1 pru_disconnect sys/sys/protosw.h:334 [inline] #4 soconnect+0xc1 sodisconnect sys/kern/uipc_socket.c:521 [inline] #4 soconnect+0xc1 sys/kern/uipc_socket.c:485 #5 sys_connect+0x207 sys/kern/uipc_syscalls.c:422 #6 syscall+0x50f mi_syscall sys/sys/syscall_mi.h:103 [inline] #6 syscall+0x50f sys/arch/amd64/amd64/trap.c:606 #7 Xsyscall+0x128 lock order "pf_lock"(rwlock) -> "&so->so_lock"(rwlock) first seen at: #0 rw_enter_write+0x5c sys/kern/kern_rwlock.c:128 #1 route_input+0x101 sys/net/rtsock.c:520 #2 rtm_send+0x1b2 sys/net/rtsock.c:1744 #3 rt_clone+0xd1 sys/net/route.c:279 #4 rtalloc+0x91 rt_match sys/net/route.c:248 [inline] #4 rtalloc+0x91 sys/net/route.c:377 #5 icmp_reflect+0x1bd sys/netinet/ip_icmp.c:740 #6 icmp_error+0x61 sys/netinet/ip_icmp.c:300 #7 pf_test_rule+0x98d sys/net/pf.c:4476 #8 pf_test+0x1d1c sys/net/pf.c:7676 #9 ip_input_if+0x131 sys/netinet/ip_input.c:417 #10 ipv4_input+0x4c sys/netinet/ip_input.c:291 #11 ether_input+0x58a #12 tun_dev_write+0x203 sys/net/if_tun.c:914 #13 spec_write+0xd9 sys/kern/spec_vnops.c:302 #14 VOP_WRITE+0xc3 sys/kern/vfs_vops.c:245 #15 vn_write+0x15b sys/kern/vfs_vnops.c:408 #16 dofilewritev+0x1a0 sys/kern/sys_generic.c:375 #17 sys_write+0x87 sys/kern/sys_generic.c:295 #18 syscall+0x50f mi_syscall sys/sys/syscall_mi.h:103 [inline] #18 syscall+0x50f sys/arch/amd64/amd64/trap.c:606 Stopped at db_enter+0x1c: addq $0x8,%rsp ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic the kernel did not panic ddb{0}> trace db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 witness_checkorder(fffffd806f67ed88,9,0) at witness_checkorder+0x10b7 witness_debugger sys/kern/subr_witness.c:2510 [inline] witness_checkorder(fffffd806f67ed88,9,0) at witness_checkorder+0x10b7 sys/kern/subr_witness.c:1110 rw_enter_write(fffffd806f67ed78) at rw_enter_write+0x5c sys/kern/kern_rwlock.c:128 route_input(fffffd8074214600,0,2) at route_input+0x101 sys/net/rtsock.c:520 rtm_send(fffffd806ed33a10,1,0,0) at rtm_send+0x1b2 sys/net/rtsock.c:1744 rt_clone(ffff80003234ddc0,ffff80003234de38,0) at rt_clone+0xd1 sys/net/route.c:279 rtalloc(ffff80003234de38,1,0) at rtalloc+0x91 rt_match sys/net/route.c:248 [inline] rtalloc(ffff80003234de38,1,0) at rtalloc+0x91 sys/net/route.c:377 icmp_reflect(fffffd8067e65500,0,0) at icmp_reflect+0x1bd sys/netinet/ip_icmp.c:740 icmp_error(fffffd8074214600,3,3,0,0) at icmp_error+0x61 sys/netinet/ip_icmp.c:300 pf_test_rule(ffff80003234e0f8,ffff80003234e1f0,ffff80003234e200,ffff80003234e1e0,ffff80003234e1c0,fffffd806d2b0ef0) at pf_test_rule+0x98d sys/net/pf.c:4476 pf_test(2,1,ffff800000e51000,ffff80003234e328) at pf_test+0x1d1c sys/net/pf.c:7676 ip_input_if(ffff80003234e328,ffff80003234e334,4,0,ffff800000e51000) at ip_input_if+0x131 sys/netinet/ip_input.c:417 ipv4_input(ffff800000e51000,fffffd806d2b0e00) at ipv4_input+0x4c sys/netinet/ip_input.c:291 ether_input(ffff800000e51000,fffffd806d2b0e00) at ether_input+0x58a tun_dev_write(5d04,ffff80003234e698,ffff800038dd5000,2) at tun_dev_write+0x203 sys/net/if_tun.c:914 spec_write(ffff80003234e4f0) at spec_write+0xd9 sys/kern/spec_vnops.c:302 VOP_WRITE(fffffd805abb2d80,ffff80003234e698,11,fffffd807f7d7478) at VOP_WRITE+0xc3 sys/kern/vfs_vops.c:245 vn_write(fffffd806f15d430,ffff80003234e698,0) at vn_write+0x15b sys/kern/vfs_vnops.c:408 dofilewritev(ffff80002a203810,c8,ffff80003234e698,0,ffff80003234e780) at dofilewritev+0x1a0 sys/kern/sys_generic.c:375 sys_write(ffff80002a203810,ffff80003234e730,ffff80003234e780) at sys_write+0x87 sys/kern/sys_generic.c:295 syscall(ffff80003234e800) at syscall+0x50f mi_syscall sys/sys/syscall_mi.h:103 [inline] syscall(ffff80003234e800) at syscall+0x50f sys/arch/amd64/amd64/trap.c:606 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xfc488e4fdb0, count: -22 ddb{0}> show registers rdi 0x3 rsi 0x40000 acpi_pdirpa+0x2be63 rbp 0xffff80003234d9c0 rbx 0x3 rdx 0xffff800000d52740 rcx 0xffff80002a203810 rax 0x3ffff acpi_pdirpa+0x2be62 r8 0xffff80003234d930 r9 0x8080808080808080 r10 0x77ae26e11222e15b r11 0xd44f82505b734969 r12 0xfffffd8003a028f0 r13 0 r14 0xfffffd80039fc5f0 r15 0xfffffd80031d69c0 rip 0xffffffff819e368c db_enter+0x1c cs 0x8 rflags 0x246 rsp 0xffff80003234d9b0 ss 0x10 db_enter+0x1c: addq $0x8,%rsp ddb{0}> show proc PROC (syz-executor.4) tid=516543 pid=60796 tcnt=2 stat=onproc flags process=0 proc=4000000 runpri=32, usrpri=74, slppri=32, nice=20 wchan=0x0, wmesg=, ps_single=0x0 forw=0xffffffffffffffff, list=0xffff80002a204d50,0xffff80002a203578 process=0xffff80002a24ee40 user=0xffff800032349000, vmspace=0xfffffd80615ec078 estcpu=36, cpticks=1, pctcpu=0.0, user=0, sys=0, intr=0 ddb{0}> ps PID TID PPID UID S FLAGS WAIT COMMAND 92211 169523 36838 0 2 0 syz-executor.5 92211 421005 36838 0 3 0x4000080 fsleep syz-executor.5 92211 468853 36838 0 2 0x4000000 syz-executor.5 92211 193385 36838 0 3 0x4000080 fsleep syz-executor.5 66456 307675 11878 0 2 0 syz-executor.6 66456 479358 11878 0 7 0x4000000 syz-executor.6 66456 187885 11878 0 3 0x4000080 fsleep syz-executor.6 89620 451929 84501 0 3 0x80 nanoslp syz-executor.1 89620 252369 84501 0 3 0x4000080 fsleep syz-executor.1 89620 255929 84501 0 3 0x4000080 fsleep syz-executor.1 89620 199084 84501 0 3 0x4000080 fsleep syz-executor.1 60796 418206 96135 0 2 0 syz-executor.4 *60796 516543 96135 0 7 0x4000000 syz-executor.4 24520 435324 24089 0 2 0 syz-executor.2 24520 111623 24089 0 3 0x4000080 netio syz-executor.2 24520 146362 24089 0 3 0x4000080 netio syz-executor.2 24520 480426 24089 0 3 0x4000080 netio syz-executor.2 5995 31089 36591 0 3 0x80 nanoslp syz-executor.0 5995 308614 36591 0 2 0x4000000 syz-executor.0 5995 34826 36591 0 3 0x4000080 fsleep syz-executor.0 52358 321961 33808 0 3 0x80 nanoslp syz-executor.3 52358 339440 33808 0 3 0x4000080 netcon syz-executor.3 52358 52822 33808 0 3 0x4000080 fsleep syz-executor.3 36591 220946 69728 0 3 0x82 nanoslp syz-executor.0 11878 259522 69728 0 3 0x82 nanoslp syz-executor.6 36838 102552 69728 0 3 0x82 nanoslp syz-executor.5 84501 96003 69728 0 3 0x82 nanoslp syz-executor.1 24089 267875 69728 0 3 0x82 nanoslp syz-executor.2 96135 245249 69728 0 3 0x82 nanoslp syz-executor.4 51769 12112 69728 0 2 0x2 syz-executor.7 48015 324675 1 0 3 0x100083 ttyin getty 33808 449298 69728 0 3 0x82 nanoslp syz-executor.3 26173 484428 0 0 3 0x14200 acct acct 41983 254513 0 0 3 0x14200 bored sosplice 99188 65844 87186 0 3 0x100082 netio ndp 87186 445914 1 0 3 0x10008a sigsusp sh 61259 227942 74422 0 3 0x100082 netio arp 74422 340203 1 0 3 0x10008a sigsusp sh 69728 435133 59808 0 3 0x2000082 wait syz-fuzzer 69728 167176 59808 0 3 0x6000082 nanoslp syz-fuzzer 69728 274665 59808 0 3 0x6000082 wait syz-fuzzer 69728 349978 59808 0 3 0x6000082 thrsleep syz-fuzzer 69728 326901 59808 0 3 0x6000082 thrsleep syz-fuzzer 69728 286959 59808 0 3 0x6000082 wait syz-fuzzer 69728 250767 59808 0 3 0x6000082 thrsleep syz-fuzzer 69728 48213 59808 0 3 0x6000082 wait syz-fuzzer 69728 200179 59808 0 3 0x6000082 wait syz-fuzzer 69728 271395 59808 0 3 0x6000082 thrsleep syz-fuzzer 69728 400155 59808 0 3 0x6000082 thrsleep syz-fuzzer 69728 308475 59808 0 3 0x6000082 wait syz-fuzzer 69728 334657 59808 0 3 0x6000082 wait syz-fuzzer 69728 324020 59808 0 3 0x6000082 thrsleep syz-fuzzer 69728 437581 59808 0 3 0x6000082 kqread syz-fuzzer 69728 248567 59808 0 3 0x6000082 wait syz-fuzzer 59808 34307 11310 0 3 0x10008a sigsusp ksh 11310 16864 80505 0 3 0x9a kqread sshd 80505 84235 1 0 3 0x88 kqread sshd 65607 87343 80592 74 3 0x1100092 bpf pflogd 80592 334433 1 0 3 0x80 netio pflogd 23331 434558 50629 73 3 0x1100090 kqread syslogd 50629 15064 1 0 3 0x100082 netio syslogd 3313 307429 1 0 3 0x100080 kqread resolvd 27823 237938 5143 77 3 0x100092 kqread dhcpleased 71130 347805 5143 77 3 0x100092 kqread dhcpleased 5143 97865 1 0 3 0x80 kqread dhcpleased 82584 44917 0 0 3 0x14200 bored smr 88273 337111 0 0 2 0x14200 zerothread 30092 414462 0 0 3 0x14200 aiodoned aiodoned 71195 144954 0 0 3 0x14200 syncer update 4655 202721 0 0 3 0x14200 cleaner cleaner 46981 229599 0 0 3 0x14200 reaper reaper 47783 507993 0 0 3 0x14200 pgdaemon pagedaemon 10998 144814 0 0 3 0x14200 bored viomb 43101 87819 0 0 3 0x40014200 acpi0 acpi0 88960 79234 0 0 3 0x40014200 idle1 44207 459674 0 0 3 0x14200 bored softnet3 55030 411925 0 0 3 0x14200 bored softnet2 79191 317609 0 0 3 0x14200 bored softnet1 44099 253600 0 0 3 0x14200 bored softnet0 74938 254124 0 0 3 0x14200 bored systqmp 22395 336707 0 0 3 0x14200 bored systq 26635 159287 0 0 3 0x14200 tmoslp softclockmp 92062 426539 0 0 3 0x40014200 tmoslp softclock 22769 412754 0 0 3 0x40014200 idle0 1 398500 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{0}> show all locks Process 66456 (syz-executor.6) thread 0xffff80002a26f800 (479358) exclusive rrwlock inode r = 0 (0xfffffd80647fd808) #0 witness_lock+0x447 #1 rw_enter+0x3c8 sys/kern/kern_rwlock.c:309 #2 rrw_enter+0x8c sys/kern/kern_rwlock.c:464 #3 VOP_LOCK+0x8b sys/kern/vfs_vops.c:518 #4 vn_lock+0x84 sys/kern/vfs_vnops.c:564 #5 vn_write+0x127 sys/kern/vfs_vnops.c:405 #6 dofilewritev+0x1a0 sys/kern/sys_generic.c:375 #7 sys_pwritev+0xb6 sys/kern/vfs_syscalls.c:3356 #8 syscall+0x50f mi_syscall sys/sys/syscall_mi.h:103 [inline] #8 syscall+0x50f sys/arch/amd64/amd64/trap.c:606 #9 Xsyscall+0x128 Process 60796 (syz-executor.4) thread 0xffff80002a203810 (516543) exclusive rwlock pf_lock r = 0 (0xffffffff82c39940) #0 witness_lock+0x447 #1 pf_test+0x1cfd #2 ip_input_if+0x131 sys/netinet/ip_input.c:417 #3 ipv4_input+0x4c sys/netinet/ip_input.c:291 #4 ether_input+0x58a #5 tun_dev_write+0x203 sys/net/if_tun.c:914 #6 spec_write+0xd9 sys/kern/spec_vnops.c:302 #7 VOP_WRITE+0xc3 sys/kern/vfs_vops.c:245 #8 vn_write+0x15b sys/kern/vfs_vnops.c:408 #9 dofilewritev+0x1a0 sys/kern/sys_generic.c:375 #10 sys_write+0x87 sys/kern/sys_generic.c:295 #11 syscall+0x50f mi_syscall sys/sys/syscall_mi.h:103 [inline] #11 syscall+0x50f sys/arch/amd64/amd64/trap.c:606 #12 Xsyscall+0x128 exclusive rwlock netlock r = 0 (0xffffffff82bd0120) #0 witness_lock+0x447 #1 tun_dev_write+0x1f8 sys/net/if_tun.c:913 #2 spec_write+0xd9 sys/kern/spec_vnops.c:302 #3 VOP_WRITE+0xc3 sys/kern/vfs_vops.c:245 #4 vn_write+0x15b sys/kern/vfs_vnops.c:408 #5 dofilewritev+0x1a0 sys/kern/sys_generic.c:375 #6 sys_write+0x87 sys/kern/sys_generic.c:295 #7 syscall+0x50f mi_syscall sys/sys/syscall_mi.h:103 [inline] #7 syscall+0x50f sys/arch/amd64/amd64/trap.c:606 #8 Xsyscall+0x128 exclusive kernel_lock &kernel_lock r = 0 (0xffffffff82d581b0) #0 witness_lock+0x447 #1 vn_write+0x46 sys/kern/vfs_vnops.c:393 #2 dofilewritev+0x1a0 sys/kern/sys_generic.c:375 #3 sys_write+0x87 sys/kern/sys_generic.c:295 #4 syscall+0x50f mi_syscall sys/sys/syscall_mi.h:103 [inline] #4 syscall+0x50f sys/arch/amd64/amd64/trap.c:606 #5 Xsyscall+0x128 Process 51769 (syz-executor.7) thread 0xffff80002a21c7e8 (12112) exclusive rrwlock inode r = 0 (0xfffffd80647fdc48) #0 witness_lock+0x447 #1 rw_enter+0x3c8 sys/kern/kern_rwlock.c:309 #2 rrw_enter+0x8c sys/kern/kern_rwlock.c:464 #3 VOP_LOCK+0x8b sys/kern/vfs_vops.c:518 #4 vn_lock+0x84 sys/kern/vfs_vnops.c:564 #5 vget+0x200 sys/kern/vfs_subr.c:676 #6 ufs_ihashget+0x121 sys/ufs/ufs/ufs_ihash.c:119 #7 ffs_vget+0x7c sys/ufs/ffs/ffs_vfsops.c:1314 #8 ufs_lookup+0x13ba sys/ufs/ufs/ufs_lookup.c:487 #9 VOP_LOOKUP+0x5c sys/kern/vfs_vops.c:85 #10 vfs_lookup+0x6e2 sys/kern/vfs_lookup.c:566 #11 namei+0x55a sys/kern/vfs_lookup.c:250 #12 dounlinkat+0x9d sys/kern/vfs_syscalls.c:1847 #13 syscall+0x4e6 sys/arch/amd64/amd64/trap.c:606 #14 Xsyscall+0x128 exclusive rrwlock inode r = 0 (0xfffffd80647fd6f8) #0 witness_lock+0x447 #1 rw_enter+0x3c8 sys/kern/kern_rwlock.c:309 #2 rrw_enter+0x8c sys/kern/kern_rwlock.c:464 #3 VOP_LOCK+0x8b sys/kern/vfs_vops.c:518 #4 vn_lock+0x84 sys/kern/vfs_vnops.c:564 #5 vget+0x200 sys/kern/vfs_subr.c:676 #6 cache_lookup+0x2b4 sys/kern/vfs_cache.c:222 #7 ufs_lookup+0x1ac sys/ufs/ufs/ufs_lookup.c:162 #8 VOP_LOOKUP+0x5c sys/kern/vfs_vops.c:85 #9 vfs_lookup+0x6e2 sys/kern/vfs_lookup.c:566 #10 namei+0x55a sys/kern/vfs_lookup.c:250 #11 dounlinkat+0x9d sys/kern/vfs_syscalls.c:1847 #12 syscall+0x4e6 sys/arch/amd64/amd64/trap.c:606 #13 Xsyscall+0x128 ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10236 6572K 8614K 166960K 24897 0 pcb 13 16K 18K 166960K 798 0 rtable 259 7K 7K 166960K 1662 0 pf 32 9K 10K 166960K 396 0 ifaddr 48 18K 18K 166960K 283 0 ifgroup 55 2K 2K 166960K 596 0 sysctl 3 0K 0K 166960K 9 0 counters 64 36K 36K 166960K 366 0 ioctlops 0 0K 4K 166960K 2148 0 iov 0 0K 32K 166960K 1170 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1651 103K 103K 166960K 7830 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 1K 13K 166960K 211 0 VM map 2 1K 1K 166960K 2 0 sem 12 0K 1K 166960K 184 0 dirhash 12 2K 2K 166960K 81 0 ACPI 1697 195K 286K 166960K 12548 0 file desc 21 77K 93K 166960K 14334 0 sigio 0 0K 0K 166960K 573 0 proc 70 91K 128K 166960K 1804 0 subproc 130 8K 8K 166960K 464 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 741 0 in_multi 99 7K 7K 166960K 572 0 ether_multi 1 0K 0K 166960K 22 0 mrt 1 0K 0K 166960K 1 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 235 1049K 1049K 166960K 235 0 exec 0 0K 1K 166960K 2924 0 pfkey data 0 0K 0K 166960K 4 0 tdb 3 0K 0K 166960K 3 0 pagedep 1 8K 8K 166960K 1 0 inodedep 1 32K 32K 166960K 1 0 newblk 1 0K 0K 166960K 1 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 611 421K 429K 166960K 142986 0 UVM aobj 131 6K 6K 166960K 131 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 0K 166960K 486 0 NDP 12 0K 1K 166960K 227 0 temp 74 5932K 6064K 166960K 117984 0 kqueue 12 18K 26K 166960K 923 0 SYN cache 2 56K 64K 166960K 4 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 24 0 0 1 0 1 1 0 8 0 rtpcb 120 10359 0 10353 61 58 3 5 0 8 2 rtentry 112 560 0 439 4 0 4 4 0 8 0 unpcb 144 7121 0 7102 88 84 4 6 0 8 3 syncache 312 94 0 94 20 20 0 1 0 8 0 tcpqe 32 432 0 432 21 21 0 1 0 8 0 tcpcb 808 3548 0 3537 110 108 2 15 0 8 0 arp 120 126 0 104 1 0 1 1 0 8 0 ipq 40 5 0 5 1 1 0 1 0 8 0 ipqe 40 15 0 15 1 1 0 1 0 8 0 inpcb 368 9063 0 9049 132 124 8 17 0 8 5 nd6 136 106 0 79 1 0 1 1 0 8 0 pkpcb 40 23 0 23 7 7 0 1 0 8 0 kcovpl 48 33 0 23 1 0 1 1 0 8 0 ppxss 1256 57 0 57 12 12 0 1 0 8 0 pffrag 232 82 0 77 2 1 1 1 0 482 0 pffrnode 88 82 0 77 2 1 1 1 0 8 0 pffrent 40 394 0 389 2 1 1 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfstitem 24 403 0 388 1 0 1 1 0 8 0 pfstkey 128 403 0 388 3 2 1 2 0 8 0 pfstate 376 403 0 388 14 11 3 6 0 8 0 pfrule 1344 21 0 16 2 1 1 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 1650 0 1167 41 10 31 31 0 8 0 art_table 32 1651 0 1167 4 0 4 4 0 8 0 art_node 16 548 0 445 1 0 1 1 0 8 0 sysvmsgpl 40 12 0 11 2 1 1 1 0 8 0 semupl 112 2 0 2 1 1 0 1 0 8 0 semapl 112 182 0 172 1 0 1 1 0 8 0 shmpl 112 128 0 0 4 0 4 4 0 8 0 dirhash 1024 63 0 46 3 0 3 3 0 8 0 dino2pl 256 20290 0 18762 96 0 96 96 0 8 0 ffsino 272 20290 0 18762 103 0 103 103 0 8 0 nchpl 144 41029 0 39375 63 0 63 63 0 8 0 uvmvnodes 80 5926 0 0 121 0 121 121 0 8 0 vnodes 216 5926 0 0 330 0 330 330 0 8 0 namei 1024 135375 0 135375 7 6 1 3 0 8 1 percpumem 16 197 0 151 1 0 1 1 0 8 0 vcpupl 2048 89 0 1 11 0 11 11 0 8 0 vmpool 696 121 0 33 10 2 8 9 0 8 0 kstatmem 264 326 0 302 2 0 2 2 0 8 0 scxspl 216 115780 0 115780 36 35 1 8 1 8 1 plimitpl 152 1889 0 1871 1 0 1 1 0 8 0 sigapl 424 14830 0 14776 12 4 8 10 0 8 0 futexpl 64 113461 0 113453 2 1 1 1 0 8 0 knotepl 120 1212 0 0 22 0 22 22 0 8 0 kqueuepl 216 2350 0 2342 46 45 1 5 0 8 0 pipepl 320 3455 0 3421 97 93 4 10 0 8 0 fdescpl 496 14593 0 14559 5 0 5 5 0 8 0 filepl 152 93461 0 93172 161 146 15 22 0 8 3 lockfpl 104 3440 0 3438 4 3 1 2 0 8 0 lockfspl 48 1344 0 1342 1 0 1 1 0 8 0 sessionpl 144 50 0 31 1 0 1 1 0 8 0 pgrppl 48 249 0 230 1 0 1 1 0 8 0 ucredpl 104 10793 0 10777 1 0 1 1 0 8 0 zombiepl 144 14776 0 14776 3 2 1 1 0 8 1 processpl 1072 14830 0 14776 7 1 6 6 0 8 0 procpl 680 40465 0 40380 24 15 9 10 0 8 0 srpgc 96 36 0 36 14 14 0 1 0 8 0 sosppl 168 144 0 143 13 12 1 1 0 8 0 sockpl 488 26654 0 26615 435 421 14 35 0 8 8 mcl64k 65536 20 0 0 3 0 3 3 0 8 0 mcl16k 16384 25 0 0 3 0 3 3 0 8 0 mcl12k 12288 25 0 0 2 0 2 2 0 8 0 mcl9k 9216 17 0 0 2 0 2 2 0 8 0 mcl8k 8192 26 0 0 4 1 3 3 0 8 0 mcl4k 4096 37 0 0 4 1 3 3 0 8 0 mcl2k2 2112 18 0 0 2 0 2 2 0 8 0 mcl2k 2048 342 0 0 30 14 16 30 0 8 0 mtagpl 96 416 0 0 5 0 5 5 0 8 0 mbufpl 256 1831 0 0 94 0 94 94 0 8 0 bufpl 288 26086 0 19764 452 0 452 452 0 8 0 anonpl 24 1392550 0 1378597 198 97 101 116 0 186 0 amapchunkpl 152 451387 0 450362 123 82 41 53 0 158 0 amappl16 200 26267 0 25811 125 100 25 37 0 8 0 amappl15 192 9 0 8 2 1 1 1 0 8 0 amappl14 184 240 0 221 3 2 1 2 0 8 0 amappl13 176 32 0 29 1 0 1 1 0 8 0 amappl12 168 15642 0 15601 4 2 2 3 0 8 0 amappl11 160 56 0 42 1 0 1 1 0 8 0 amappl10 152 64 0 48 1 0 1 1 0 8 0 amappl9 144 322 0 321 1 0 1 1 0 8 0 amappl8 136 814 0 625 7 0 7 7 0 8 0 amappl7 128 277 0 247 2 0 2 2 0 8 0 amappl6 120 735 0 717 1 0 1 1 0 8 0 amappl5 112 374 0 362 1 0 1 1 0 8 0 amappl4 104 843 0 807 4 2 2 2 0 8 0 amappl3 96 88159 0 88044 23 20 3 4 0 8 0 amappl2 88 15746 0 15657 6 3 3 3 0 8 1 amappl1 80 60901 0 60279 23 9 14 23 0 8 0 amappl 88 141725 0 141402 8 0 8 8 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 130 0 0 3 0 3 3 0 8 0 uaddrrnd 24 14714 0 14592 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 14714 0 14592 1 0 1 1 0 8 0 vmmpekpl 168 112261 0 112177 5 0 5 5 0 8 0 vmmpepl 168 864629 0 861765 343 209 134 144 0 357 0 vmsppl 464 14713 0 14592 16 0 16 16 0 8 0 rwobjpl 56 212068 0 204151 130 18 112 115 0 8 0 pdppl 4096 29435 0 29272 740 575 165 166 0 8 2 pvpl 32 41950 0 0 341 2 339 339 0 265 0 pmappl 248 14713 0 14592 8 0 8 8 0 8 0 extentpl 40 56 0 38 1 0 1 1 0 8 0 phpool 112 1893 0 888 31 1 30 30 0 8 0 ddb{0}> machine ddbcpu 0 Invalid cpu 0 ddb{0}> trace db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 witness_checkorder(fffffd806f67ed88,9,0) at witness_checkorder+0x10b7 witness_debugger sys/kern/subr_witness.c:2510 [inline] witness_checkorder(fffffd806f67ed88,9,0) at witness_checkorder+0x10b7 sys/kern/subr_witness.c:1110 rw_enter_write(fffffd806f67ed78) at rw_enter_write+0x5c sys/kern/kern_rwlock.c:128 route_input(fffffd8074214600,0,2) at route_input+0x101 sys/net/rtsock.c:520 rtm_send(fffffd806ed33a10,1,0,0) at rtm_send+0x1b2 sys/net/rtsock.c:1744 rt_clone(ffff80003234ddc0,ffff80003234de38,0) at rt_clone+0xd1 sys/net/route.c:279 rtalloc(ffff80003234de38,1,0) at rtalloc+0x91 rt_match sys/net/route.c:248 [inline] rtalloc(ffff80003234de38,1,0) at rtalloc+0x91 sys/net/route.c:377 icmp_reflect(fffffd8067e65500,0,0) at icmp_reflect+0x1bd sys/netinet/ip_icmp.c:740 icmp_error(fffffd8074214600,3,3,0,0) at icmp_error+0x61 sys/netinet/ip_icmp.c:300 pf_test_rule(ffff80003234e0f8,ffff80003234e1f0,ffff80003234e200,ffff80003234e1e0,ffff80003234e1c0,fffffd806d2b0ef0) at pf_test_rule+0x98d sys/net/pf.c:4476 pf_test(2,1,ffff800000e51000,ffff80003234e328) at pf_test+0x1d1c sys/net/pf.c:7676 ip_input_if(ffff80003234e328,ffff80003234e334,4,0,ffff800000e51000) at ip_input_if+0x131 sys/netinet/ip_input.c:417 ipv4_input(ffff800000e51000,fffffd806d2b0e00) at ipv4_input+0x4c sys/netinet/ip_input.c:291 ether_input(ffff800000e51000,fffffd806d2b0e00) at ether_input+0x58a tun_dev_write(5d04,ffff80003234e698,ffff800038dd5000,2) at tun_dev_write+0x203 sys/net/if_tun.c:914 spec_write(ffff80003234e4f0) at spec_write+0xd9 sys/kern/spec_vnops.c:302 VOP_WRITE(fffffd805abb2d80,ffff80003234e698,11,fffffd807f7d7478) at VOP_WRITE+0xc3 sys/kern/vfs_vops.c:245 vn_write(fffffd806f15d430,ffff80003234e698,0) at vn_write+0x15b sys/kern/vfs_vnops.c:408 dofilewritev(ffff80002a203810,c8,ffff80003234e698,0,ffff80003234e780) at dofilewritev+0x1a0 sys/kern/sys_generic.c:375 sys_write(ffff80002a203810,ffff80003234e730,ffff80003234e780) at sys_write+0x87 sys/kern/sys_generic.c:295 syscall(ffff80003234e800) at syscall+0x50f mi_syscall sys/sys/syscall_mi.h:103 [inline] syscall(ffff80003234e800) at syscall+0x50f sys/arch/amd64/amd64/trap.c:606 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xfc488e4fdb0, count: -22 ddb{0}> machine ddbcpu 1 Stopped at x86_ipi_db+0x1e: addq $0x8,%rsp ddb{1}> trace x86_ipi_db(ffff800029d2bff0) at x86_ipi_db+0x1e sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 __mp_lock(ffffffff82d57fa8) at __mp_lock+0x122 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff82d57fa8) at __mp_lock+0x122 sys/kern/kern_lock.c:147 __mp_acquire_count(ffffffff82d57fa8,1) at __mp_acquire_count+0x48 sys/kern/kern_lock.c:227 mi_switch() at mi_switch+0x46d sys/kern/sched_bsd.c:470 sleep_finish(0,1) at sleep_finish+0x19b sys/kern/kern_synch.c:414 biowait(fffffd8064b607e0) at biowait+0x91 sys/kern/vfs_bio.c:1278 bwrite(fffffd8064b607e0) at bwrite+0x21c sys/kern/vfs_bio.c:769 ffs_write(ffff80003a879280) at ffs_write+0x6a5 VOP_WRITE(fffffd806bb23608,ffff80003a8794a0,5,fffffd807f7d7478) at VOP_WRITE+0xc3 sys/kern/vfs_vops.c:245 vn_write(fffffd8067e020a0,ffff80003a8794a0,1) at vn_write+0x1a5 sys/kern/vfs_vnops.c:408 dofilewritev(ffff80002a26f800,6,ffff80003a8794a0,1,ffff80003a879590) at dofilewritev+0x1a0 sys/kern/sys_generic.c:375 sys_pwritev(ffff80002a26f800,ffff80003a879540,ffff80003a879590) at sys_pwritev+0xb6 sys/kern/vfs_syscalls.c:3356 syscall(ffff80003a879610) at syscall+0x50f mi_syscall sys/sys/syscall_mi.h:103 [inline] syscall(ffff80003a879610) at syscall+0x50f sys/arch/amd64/amd64/trap.c:606 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xe7288747550, count: -16