witness: reversal: inode fdlock

witness: reversal: inode fdlock
Status: closed as invalid on 2019/10/29 11:01
Reported-by: syzbot+c278abadcbeb7150b6c2@syzkaller.appspotmail.com
First crash: 362d, last: 149d

Sample crash report:

login: witness: lock order reversal:
 1st 0xfffffd806d8bc5f8 inode (&ip->i_lock)
 2nd 0xfffffd806e6d7438 fdlock (&newfdp->fd_fd.fd_lock)
lock order "&newfdp->fd_fd.fd_lock"(rwlock) -> "&ip->i_lock"(rrwlock) first seen at:
#0  witness_checkorder+0x6a7 sys/kern/subr_witness.c:879
#1  rw_enter+0xd1 sys/kern/kern_rwlock.c:247
#2  rrw_enter+0x4f sys/kern/kern_rwlock.c:435
#3  VOP_LOCK+0x4b sys/kern/vfs_vops.c:602
#4  vn_closefile+0x119 vn_lock sys/kern/vfs_vnops.c:556 [inline]
#4  vn_closefile+0x119 vn_close sys/kern/vfs_vnops.c:288 [inline]
#4  vn_closefile+0x119 sys/kern/vfs_vnops.c:582
#5  fdrop+0xc9 sys/kern/kern_descrip.c:1269
#6  closef+0x11d sys/kern/kern_descrip.c:1253
#7  finishdup+0x2b6 sys/kern/kern_descrip.c:682
#8  dodup3+0x5c5 sys/kern/kern_descrip.c:377
#9  syscall+0x552 mi_syscall sys/sys/syscall_mi.h:92 [inline]
#9  syscall+0x552 sys/arch/amd64/amd64/trap.c:555
#10 Xsyscall+0x128
lock order "&ip->i_lock"(rrwlock) -> "&newfdp->fd_fd.fd_lock"(rwlock) first seen at:
#0  witness_checkorder+0x6a7 sys/kern/subr_witness.c:879
#1  rw_enter_write+0x5b sys/kern/kern_rwlock.c:125
#2  doopenat+0x610 sys/kern/vfs_syscalls.c:1214
#3  syscall+0x552 mi_syscall sys/sys/syscall_mi.h:92 [inline]
#3  syscall+0x552 sys/arch/amd64/amd64/trap.c:555
#4  Xsyscall+0x128
Stopped at      db_enter+0x18:  addq    $0x8,%rsp
ddb{1}> 
ddb{1}> set $lines = 0
ddb{1}> set $maxwidth = 0
ddb{1}> show panic
the kernel did not panic
ddb{1}> trace
db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398
witness_checkorder(fffffd806e6d7438,9,0) at witness_checkorder+0xffc witness_debugger sys/kern/subr_witness.c:2506 [inline]
witness_checkorder(fffffd806e6d7438,9,0) at witness_checkorder+0xffc sys/kern/subr_witness.c:1086
rw_enter_write(fffffd806e6d7428) at rw_enter_write+0x5b sys/kern/kern_rwlock.c:125
doopenat(ffff800020b28c70,ffffff9c,200000c0,0,0,ffff800020be1b20) at doopenat+0x610 sys/kern/vfs_syscalls.c:1214
syscall(ffff800020be1b90) at syscall+0x552 mi_syscall sys/sys/syscall_mi.h:92 [inline]
syscall(ffff800020be1b90) at syscall+0x552 sys/arch/amd64/amd64/trap.c:555
Xsyscall(6,0,840a1110138,0,840a1110118,840a1110110) at Xsyscall+0x128
end of kernel
end trace frame: 0x84381170600, count: -6
ddb{1}> show registers
rdi                              0x3
rsi               0xffffffff821bcce0    __sancov_gen_cov_switch_values.122
rbp               0xffff800020be1760
rbx                              0x3
rdx                             0x8b
rcx                              0x3
rax                              0x3
r8                0xffffffff8111ee15    witness_checkorder+0xfd5
r9                               0x5
r10               0x88e1117e8bfeb5c8
r11               0x1678a6a2d3a6fa57
r12               0xfffffd800267ac00
r13                                0
r14               0xffffffff822fe490    w_lodata+0x4ec30
r15               0xffffffff82303b30    w_lodata+0x542d0
rip               0xffffffff81e3a658    db_enter+0x18
cs                               0x8
rflags                         0x246
rsp               0xffff800020be1750
ss                              0x10
db_enter+0x18:  addq    $0x8,%rsp
ddb{1}> show proc
PROC (syz-executor0169) pid=342021 stat=onproc
    flags process=0 proc=4000000<THREAD>
    pri=32, usrpri=86, nice=20
    forw=0xffffffffffffffff, list=0xffff800020b618c0,0xffff800020b28a08
    process=0xffff800020b7d880 user=0xffff800020bdc000, vmspace=0xfffffd807effd8a0
    estcpu=36, cpticks=0, pctcpu=0.0
    user=0, sys=0, intr=0
ddb{1}> ps
   PID     TID   PPID    UID  S       FLAGS  WAIT          COMMAND
 81008   82046  96471      0  7           0                syz-executor0169
 81008   69170  96471      0  2   0x4000000                syz-executor0169
 43928    2261   3641      0  3        0x80  nanosleep     syz-executor0169
 43928  268085   3641      0  3   0x4000080  fsleep        syz-executor0169
 43928  265857   3641      0  3   0x4000080  fsleep        syz-executor0169
*43928  342021   3641      0  7   0x4000000                syz-executor0169
 96471  261132  14729      0  3        0x80  nanosleep     syz-executor0169
  3641  473323  14729      0  3        0x80  nanosleep     syz-executor0169
 14729  187301  36509      0  3        0x82  nanosleep     syz-executor0169
 36509  129435  78283      0  3    0x10008a  pause         ksh
 78283  370707   9578      0  3        0x92  select        sshd
 41635   96045      1      0  3    0x100083  ttyin         getty
  9578    4989      1      0  3        0x80  select        sshd
 55481  311226  90345     74  3    0x100092  bpf           pflogd
 90345  121153      1      0  3        0x80  netio         pflogd
  2572  163714  51853     73  3    0x100090  kqread        syslogd
 51853  204266      1      0  3    0x100082  netio         syslogd
 90373  304658      1     77  3    0x100090  poll          dhclient
 60751  429965      1      0  3        0x80  poll          dhclient
 77541  325038      0      0  3     0x14200  pgzero        zerothread
 56350   22830      0      0  3     0x14200  aiodoned      aiodoned
 13133   57463      0      0  3     0x14200  syncer        update
 76653  471411      0      0  3     0x14200  cleaner       cleaner
 11474  479753      0      0  3     0x14200  reaper        reaper
 79086  352619      0      0  3     0x14200  pgdaemon      pagedaemon
 25198  361860      0      0  3     0x14200  bored         crynlk
 87097  189584      0      0  3     0x14200  bored         crypto
 88125  123357      0      0  3  0x40014200  acpi0         acpi0
  2890  263499      0      0  3  0x40014200                idle1
 34870  120230      0      0  3     0x14200  bored         softnet
 60104  291797      0      0  3     0x14200  bored         systqmp
 59685  287726      0      0  3     0x14200  bored         systq
  9279  511458      0      0  3  0x40014200  bored         softclock
 93853  427659      0      0  3  0x40014200                idle0
 22521  407886      0      0  3     0x14200  bored         smr
     1  449830      0      0  3        0x82  wait          init
     0       0     -1      0  3     0x10200  scheduler     swapper
ddb{1}> show all locks
Process 43928 (syz-executor0169) thread 0xffff800020b28c70 (342021)
exclusive rrwlock inode r = 0 (0xfffffd806d8bc5f8)
#0  witness_lock+0x52e sys/kern/subr_witness.c:1163
#1  rw_enter+0x46d sys/kern/kern_rwlock.c:306
#2  rrw_enter+0x4f sys/kern/kern_rwlock.c:435
#3  VOP_LOCK+0x4b sys/kern/vfs_vops.c:602
#4  vn_lock+0x6e sys/kern/vfs_vnops.c:556
#5  spec_open+0x431 sys/kern/spec_vnops.c:159
#6  VOP_OPEN+0x6a sys/kern/vfs_vops.c:153
#7  vn_open+0x495 sys/kern/vfs_vnops.c:174
#8  doopenat+0x28e sys/kern/vfs_syscalls.c:1145
#9  syscall+0x552 mi_syscall sys/sys/syscall_mi.h:92 [inline]
#9  syscall+0x552 sys/arch/amd64/amd64/trap.c:555
#10 Xsyscall+0x128
exclusive kernel_lock &kernel_lock r = 0 (0xffffffff82363690)
#0  witness_lock+0x52e sys/kern/subr_witness.c:1163
#1  __mp_acquire_count+0x51 sys/kern/kern_lock.c:227
#2  mi_switch+0x38f sys/kern/sched_bsd.c:441
#3  sleep_finish+0x113 sys/kern/kern_synch.c:373
#4  rw_enter+0x366 sys/kern/kern_rwlock.c:282
#5  rrw_enter+0x4f sys/kern/kern_rwlock.c:435
#6  VOP_LOCK+0x4b sys/kern/vfs_vops.c:602
#7  vn_lock+0x6e sys/kern/vfs_vnops.c:556
#8  spec_open+0x431 sys/kern/spec_vnops.c:159
#9  VOP_OPEN+0x6a sys/kern/vfs_vops.c:153
#10 vn_open+0x495 sys/kern/vfs_vnops.c:174
#11 doopenat+0x28e sys/kern/vfs_syscalls.c:1145
#12 syscall+0x552 mi_syscall sys/sys/syscall_mi.h:92 [inline]
#12 syscall+0x552 sys/arch/amd64/amd64/trap.c:555
#13 Xsyscall+0x128
ddb{1}> show malloc
           Type InUse  MemUse  HighUse   Limit  Requests Type Lim Kern Lim
         devbuf  9454   6383K    6384K  78643K     10541        0        0
            pcb    13      8K       8K  78643K        13        0        0
         rtable    61      2K       2K  78643K       125        0        0
         ifaddr    25      7K       7K  78643K        26        0        0
       counters    39     33K      33K  78643K        39        0        0
       ioctlops     0      0K       4K  78643K      1467        0        0
          mount     1      1K       1K  78643K         1        0        0
         vnodes  1181     74K      74K  78643K      1462        0        0
      UFS quota     1     32K      32K  78643K         1        0        0
      UFS mount     5     36K      36K  78643K         5        0        0
            shm     2      1K       1K  78643K         2        0        0
         VM map     2      1K       1K  78643K         2        0        0
            sem     2      0K       0K  78643K         2        0        0
        dirhash    12      2K       2K  78643K        12        0        0
           ACPI  1808    196K     290K  78643K     12765        0        0
      file desc     3      4K       5K  78643K      1048        0        0
           proc    52     50K      58K  78643K       334        0        0
    NFS srvsock     1      0K       0K  78643K         1        0        0
     NFS daemon     1     16K      16K  78643K         1        0        0
       in_multi    11      0K       0K  78643K        11        0        0
    ether_multi     1      0K       0K  78643K         1        0        0
    ISOFS mount     1     32K      32K  78643K         1        0        0
  MSDOSFS mount     1     16K      16K  78643K         1        0        0
           ttys    30    132K     132K  78643K        30        0        0
           exec     0      0K       1K  78643K       179        0        0
        pagedep     1      8K       8K  78643K         1        0        0
       inodedep     1     32K      32K  78643K         1        0        0
         newblk     1      0K       0K  78643K         1        0        0
        VM swap     7     26K      26K  78643K         7        0        0
       UVM amap    70      3K       3K  78643K      1885        0        0
       UVM aobj     2      2K       2K  78643K         2        0        0
        memdesc     1      4K       4K  78643K         1        0        0
    crypto data     1      1K       1K  78643K         1        0        0
            NDP     4      0K       0K  78643K         4        0        0
           temp    39   2728K    2792K  78643K      1993        0        0
      SYN cache     2     16K      16K  78643K         2        0        0
ddb{1}> show all pools
Name      Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle
arp         64        2    0        0     1     0     1     1     0     8    0
plcache    128       20    0        0     1     0     1     1     0     8    0
rtpcb       80       15    0       13     1     0     1     1     0     8    0
rtentry    112       23    0        1     1     0     1     1     0     8    0
unpcb      120       29    0       19     1     0     1     1     0     8    0
syncache   264        5    0        5     2     1     1     1     0     8    1
tcpcb      544        8    0        5     1     0     1     1     0     8    0
inpcb      280       29    0       23     1     0     1     1     0     8    0
pfosfp      40      846    0      423     5     0     5     5     0     8    0
pfosfpen   112     1428    0      714    21     0    21    21     0     8    0
pfstitem    24        8    0        6     2     1     1     1     0     8    0
pfstkey    112        8    0        6     2     1     1     1     0     8    0
pfstate    328        8    0        6     2     1     1     1     0     8    0
pfrule     1360      21    0       16     2     1     1     2     0     8    0
art_heap8  4096       1    0        0     1     0     1     1     0     8    0
art_heap4  256       96    0        0     6     0     6     6     0     8    0
art_table   32       97    0        0     1     0     1     1     0     8    0
art_node    16       22    0        2     1     0     1     1     0     8    0
dirhash    1024      17    0        0     3     0     3     3     0     8    0
dino1pl    128     1859    0      476    45     0    45    45     0     8    0
ffsino     272     1859    0      476    93     0    93    93     0     8    0
nchpl      144     1984    0      443    58     0    58    58     0     8    0
uvmvnodes   72     1869    0        0    34     0    34    34     0     8    0
vnodes     200     1869    0        0    99     0    99    99     0     8    0
namei      1024    6083    0     6083     2     1     1     1     0     8    1
percpumem   16       30    0        0     1     0     1     1     0     8    0
scxspl     192     3070    0     3070    32    31     1     6     0     8    1
plimitpl   152       14    0        8     1     0     1     1     0     8    0
sigapl     432      556    0      540     2     0     2     2     0     8    0
futexpl     56     1214    0     1212     1     0     1     1     0     8    0
knotepl    112        5    0        0     1     0     1     1     0     8    0
kqueuepl   104        1    0        0     1     0     1     1     0     8    0
pipepl     112      134    0      127     2     1     1     1     0     8    0
fdescpl    488      557    0      540     3     0     3     3     0     8    0
filepl     152     2377    0     2325     3     0     3     3     0     8    0
lockfpl    104        6    0        6     1     1     0     1     0     8    0
lockfspl    48        3    0        3     1     1     0     1     0     8    0
sessionpl  112       18    0        9     1     0     1     1     0     8    0
pgrppl      48       18    0        9     1     0     1     1     0     8    0
ucredpl     96     1099    0     1090     1     0     1     1     0     8    0
zombiepl   144      540    0      540     2     1     1     1     0     8    1
processpl  896      572    0      540     4     0     4     4     0     8    0
procpl     632      941    0      905     4     0     4     4     0     8    0
sockpl     384       73    0       55     2     0     2     2     0     8    0
mcl4k      4096       4    0        0     1     0     1     1     0     8    0
mcl2k      2048      88    0        0    10     0    10    10     0     8    0
mtagpl      80        1    0        0     1     0     1     1     0     8    0
mbufpl     256      101    0        0     6     0     6     6     0     8    0
bufpl      256     2175    0      276   119     0   119   119     0     8    0
anonpl      16    34263    0    32906     7     1     6     7     0   125    0
amapchunkpl 152    1626    0     1577     3     0     3     3     0   158    0
amappl16   192      435    0      428     1     0     1     1     0     8    0
amappl14   176       38    0       34     1     0     1     1     0     8    0
amappl12   160        4    0        4     1     1     0     1     0     8    0
amappl11   152       46    0       31     1     0     1     1     0     8    0
amappl10   144       60    0       57     1     0     1     1     0     8    0
amappl9    136      435    0      434     1     0     1     1     0     8    0
amappl8    128      451    0      442     1     0     1     1     0     8    0
amappl7    120       19    0       17     1     0     1     1     0     8    0
amappl6    112       45    0       41     1     0     1     1     0     8    0
amappl5    104      125    0      112     1     0     1     1     0     8    0
amappl4     96      475    0      447     1     0     1     1     0     8    0
amappl3     88      155    0      144     1     0     1     1     0     8    0
amappl2     80     3975    0     3902     2     0     2     2     0     8    0
amappl1     72    19519    0    19058    16     6    10    16     0     8    0
amappl      80     1141    0     1112     1     0     1     1     0    84    0
dma4096    4096       1    0        1     1     1     0     1     0     8    0
dma256     256        6    0        6     1     1     0     1     0     8    0
dma64       64      259    0      259     1     1     0     1     0     8    0
dma32       32        7    0        7     1     1     0     1     0     8    0
dma16       16       17    0       17     1     1     0     1     0     8    0
aobjpl      64        1    0        0     1     0     1     1     0     8    0
uaddrrnd    24      557    0      540     1     0     1     1     0     8    0
uaddrbest   32        2    0        0     1     0     1     1     0     8    0
uaddr       24      557    0      540     1     0     1     1     0     8    0
vmmpekpl   168     7052    0     7031     2     0     2     2     0     8    0
vmmpepl    168    51294    0    50300    57    13    44    52     0   357    0
vmsppl     368      556    0      540     2     0     2     2     0     8    0
pdppl      4096    1122    0     1080     6     0     6     6     0     8    0
pvpl        32   110653    0   107446    28     0    28    28     0   265    0
pmappl     232      556    0      540     1     0     1     1     0     8    0
extentpl    40       41    0       26     1     0     1     1     0     8    0
phpool     112      254    0        4     8     0     8     8     0     8    0

Crashes (523):
Manager	Time	Kernel	Commit	Syzkaller	Config	Log	Report	Syz repro	C repro
ci-openbsd-multicore	2019/07/28 00:22	openbsd	6181cd34	c85e1c5b	.config	log	report	syz	C
ci-openbsd-multicore	2019/04/26 14:49	openbsd	7d9a3feb	b617407b	.config	log	report	syz	C
ci-openbsd-multicore	2019/04/22 01:28	openbsd	735677bb	b0e8efcb	.config	log	report	syz	C
ci-openbsd-multicore	2019/01/25 09:46	openbsd	6f9e9be9	b5d78bce	.config	log	report	syz	C
ci-openbsd-multicore	2019/08/26 10:25	openbsd	56a8f046	d21c5d9d	.config	log	report
ci-openbsd-multicore	2019/08/25 22:22	openbsd	a1d6d126	d21c5d9d	.config	log	report
ci-openbsd-multicore	2019/08/25 17:24	openbsd	a1d6d126	d21c5d9d	.config	log	report
ci-openbsd-multicore	2019/08/25 13:33	openbsd	a1d6d126	d21c5d9d	.config	log	report
ci-openbsd-multicore	2019/08/25 04:58	openbsd	1507cfe1	d21c5d9d	.config	log	report
ci-openbsd-multicore	2019/08/24 23:46	openbsd	1507cfe1	d21c5d9d	.config	log	report
ci-openbsd-multicore	2019/08/24 17:06	openbsd	1507cfe1	78ded196	.config	log	report
ci-openbsd-multicore	2019/08/24 06:28	openbsd	9be55947	78ded196	.config	log	report
ci-openbsd-multicore	2019/08/24 01:12	openbsd	afd5172b	78ded196	.config	log	report
ci-openbsd-multicore	2019/08/23 22:10	openbsd	afd5172b	78ded196	.config	log	report
ci-openbsd-multicore	2019/08/23 11:28	openbsd	9f57e42b	ca6f3cfa	.config	log	report
ci-openbsd-multicore	2019/08/23 05:05	openbsd	9f57e42b	ca6f3cfa	.config	log	report
ci-openbsd-multicore	2019/08/22 19:54	openbsd	d4b29719	c6c81a0b	.config	log	report
ci-openbsd-multicore	2019/08/22 17:41	openbsd	d4b29719	c6c81a0b	.config	log	report
ci-openbsd-multicore	2019/08/22 15:25	openbsd	d4b29719	c6c81a0b	.config	log	report
ci-openbsd-multicore	2019/08/22 10:10	openbsd	d4b29719	984250d5	.config	log	report
ci-openbsd-multicore	2019/08/22 08:56	openbsd	d4b29719	984250d5	.config	log	report
ci-openbsd-multicore	2019/08/21 16:21	openbsd	ceaaae1d	4ea67ff8	.config	log	report
ci-openbsd-multicore	2019/08/21 14:44	openbsd	ceaaae1d	4ea67ff8	.config	log	report
ci-openbsd-multicore	2019/08/21 11:57	openbsd	ceaaae1d	4ea67ff8	.config	log	report
ci-openbsd-multicore	2019/08/21 07:22	openbsd	ceaaae1d	4ea67ff8	.config	log	report
ci-openbsd-multicore	2019/08/21 04:38	openbsd	288889b9	6b8391d0	.config	log	report
ci-openbsd-multicore	2019/08/20 23:03	openbsd	288889b9	6b8391d0	.config	log	report
ci-openbsd-multicore	2019/08/20 21:41	openbsd	288889b9	6b8391d0	.config	log	report
ci-openbsd-multicore	2019/08/20 19:30	openbsd	288889b9	6b8391d0	.config	log	report
ci-openbsd-multicore	2019/08/20 17:14	openbsd	55fd3d87	cfc9868f	.config	log	report
ci-openbsd-multicore	2019/08/20 15:23	openbsd	55fd3d87	cfc9868f	.config	log	report
ci-openbsd-multicore	2019/08/20 14:00	openbsd	55fd3d87	cfc9868f	.config	log	report
ci-openbsd-multicore	2019/08/20 09:22	openbsd	55fd3d87	cfc9868f	.config	log	report
ci-openbsd-multicore	2019/08/20 04:15	openbsd	85b8fad3	ae348fb7	.config	log	report
ci-openbsd-multicore	2019/08/20 01:07	openbsd	85b8fad3	ae348fb7	.config	log	report
ci-openbsd-multicore	2019/08/19 18:13	openbsd	85b8fad3	ae348fb7	.config	log	report
ci-openbsd-multicore	2019/08/19 14:20	openbsd	85b8fad3	b8ceabfc	.config	log	report
ci-openbsd-multicore	2019/08/19 10:31	openbsd	682277b9	b8ceabfc	.config	log	report
ci-openbsd-multicore	2019/08/19 03:24	openbsd	682277b9	b8ceabfc	.config	log	report
ci-openbsd-multicore	2019/08/18 22:39	openbsd	682277b9	b8ceabfc	.config	log	report
ci-openbsd-multicore	2019/08/18 16:44	openbsd	682277b9	55bf8926	.config	log	report
ci-openbsd-multicore	2019/08/18 14:49	openbsd	f7a04f0f	55bf8926	.config	log	report
ci-openbsd-multicore	2019/08/18 12:22	openbsd	f7a04f0f	55bf8926	.config	log	report
ci-openbsd-multicore	2019/08/18 11:07	openbsd	f7a04f0f	55bf8926	.config	log	report
ci-openbsd-multicore	2019/08/18 04:34	openbsd	f7a04f0f	55bf8926	.config	log	report
ci-openbsd-multicore	2019/08/18 02:32	openbsd	67dad36e	55bf8926	.config	log	report
ci-openbsd-multicore	2019/08/17 22:04	openbsd	67dad36e	55bf8926	.config	log	report
ci-openbsd-pf	2019/03/05 13:36	openbsd	da8fceb2	3419571c	.config	log	report
ci-openbsd-multicore	2019/01/25 09:27	openbsd	6f9e9be9	b5d78bce	.config	log	report