syzbot


multicore test error: uvm_fault: spllower

Status: fixed on 2020/12/09 20:34
Reported-by: syzbot+a293948d449052374fbe@syzkaller.appspotmail.com
Fix commit: 3ba77c9295b2 Revert previous extension of the SCHED_LOCK(), the state isn't passed down.
First crash: 1452d, last: 1452d

Sample crash report:
login: uvm_fault(0xffffffff828b3da8, 0xffff80012a2395b0, 0, 1) -> e
kernel: page fault trap, code=0
Stopped at      spllower+0x34:  movq    0x5c0(%r12,%rax,8),%rbx
ddb{1}> 
ddb{1}> set $lines = 0
ddb{1}> set $maxwidth = 0
ddb{1}> show panic
kernel page fault
uvm_fault(0xffffffff828b3da8, 0xffff80012a2395b0, 0, 1) -> e
spllower(ffff80002129a000) at spllower+0x34 sys/arch/amd64/amd64/intr.c:716
end trace frame: 0xffff8000234a5990, count: 0
ddb{1}> trace
spllower(ffff80002129a000) at spllower+0x34 sys/arch/amd64/amd64/intr.c:716
single_thread_check_locked(ffff8000212919b8,0) at single_thread_check_locked+0x196 sys/kern/kern_sig.c:1970
userret(ffff8000212919b8) at userret+0x335 single_thread_check sys/kern/kern_sig.c:1991 [inline]
userret(ffff8000212919b8) at userret+0x335 sys/kern/kern_sig.c:1936
syscall(ffff8000234a5ac0) at syscall+0x55c mi_syscall_return sys/sys/syscall_mi.h:129 [inline]
syscall(ffff8000234a5ac0) at syscall+0x55c sys/arch/amd64/amd64/trap.c:612
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x73754c5d580, count: -5
ddb{1}> show registers
rdi               0xffff80002129a000
rsi                            0x99f
rbp               0xffff8000234a5930
rbx                           0x1000    __ALIGN_SIZE
rdx               0xffff80002129a000
rcx                            0x99e
rax                       0x2129a000
r8                0xffffffff822b4d90    setrunnable+0xa0
r9                               0x5
r10               0xa70638e813e5e886
r11                0xf6f54406225e8f8
r12               0xffff800020d68ff0
r13                              0x1
r14                       0x2129a000
r15               0xffff8000212919b8
rip               0xffffffff81795ab4    spllower+0x34
cs                               0x8
rflags                       0x10246    __ALIGN_SIZE+0xf246
rsp               0xffff8000234a58e0
ss                              0x10
spllower+0x34:  movq    0x5c0(%r12,%rax,8),%rbx
ddb{1}> show proc
PROC (syz-executor.0) pid=105890 stat=onproc
    flags process=1010<SUGID,SINGLEEXIT> proc=4080000<SUSPSINGLE,THREAD>
    pri=32, usrpri=51, nice=20
    forw=0xffffffffffffffff, list=0xffff8000211c2a48,0xffffffff828f9680
    process=0xffff8000ffff94a8 user=0xffff8000234a0000, vmspace=0xfffffd806ea52b88
    estcpu=36, cpticks=0, pctcpu=0.0
    user=0, sys=0, intr=0
ddb{1}> ps
   PID     TID   PPID    UID  S       FLAGS  WAIT          COMMAND
 81531  197657  61023  32767  7      0x3010                syz-executor.0
*81531  105890  61023  32767  7   0x4081010                syz-executor.0
 61023  201302  73934  32767  3        0x90  nanosleep     syz-executor.0
 73934  323678  59157      0  3        0x82  wait          syz-executor.0
 59157  410050  46451      0  3        0x82  kqread        syz-fuzzer
 59157   94216  46451      0  3   0x4000082  thrsleep      syz-fuzzer
 59157  421896  46451      0  3   0x4000082  thrsleep      syz-fuzzer
 59157  334180  46451      0  3   0x4000082  thrsleep      syz-fuzzer
 59157   37436  46451      0  3   0x4000082  thrsleep      syz-fuzzer
 59157  150811  46451      0  3   0x4000082  thrsleep      syz-fuzzer
 59157  473423  46451      0  3   0x4000082  thrsleep      syz-fuzzer
 46451  252999   4557      0  3    0x10008a  pause         ksh
  4557  174437  74567      0  3        0x92  select        sshd
 75695    2300      1      0  3    0x100083  ttyin         getty
 74567  432040      1      0  3        0x80  select        sshd
 96188  313621  18362     73  3    0x100090  kqread        syslogd
 18362  404860      1      0  3    0x100082  netio         syslogd
 21865   87770      1     77  3    0x100090  poll          dhclient
 17500  284743      1      0  3        0x80  poll          dhclient
 69724  399837      0      0  3     0x14200  bored         smr
 46542   52158      0      0  3     0x14200  pgzero        zerothread
 80579  212794      0      0  3     0x14200  aiodoned      aiodoned
 41442   99409      0      0  3     0x14200  syncer        update
 10583   97729      0      0  3     0x14200  cleaner       cleaner
 71750  415247      0      0  3     0x14200  reaper        reaper
 17327  469407      0      0  3     0x14200  pgdaemon      pagedaemon
 66192  518716      0      0  3     0x14200  bored         crynlk
 72517  293260      0      0  3     0x14200  bored         crypto
 79565  282994      0      0  3     0x14200  bored         viomb
 85310  202814      0      0  3  0x40014200  acpi0         acpi0
 72194  263263      0      0  3  0x40014200                idle1
  3870  296590      0      0  3     0x14200  bored         softnet
 72544   66642      0      0  3     0x14200  bored         systqmp
 21620  462570      0      0  3     0x14200  bored         systq
 67635  410281      0      0  3  0x40014200  bored         softclock
 29879  281452      0      0  3  0x40014200                idle0
     1  188213      0      0  3        0x82  wait          init
     0       0     -1      0  3     0x10200  scheduler     swapper
ddb{1}> show all locks
Process 81531 (syz-executor.0) thread 0xffff8000212919b8 (105890)
exclusive kernel_lock &kernel_lock r = 0 (0xffffffff828f9890)
#0  witness_lock+0x4b0 stacktrace_save sys/sys/stacktrace.h:36 [inline]
#0  witness_lock+0x4b0 sys/kern/subr_witness.c:1176
#1  kpageflttrap+0x23f sys/arch/amd64/amd64/trap.c:277
#2  kerntrap+0xef sys/arch/amd64/amd64/trap.c:321
#3  alltraps_kern_meltdown+0x7b
#4  spllower+0x34 sys/arch/amd64/amd64/intr.c:716
#5  single_thread_check_locked+0x196 sys/kern/kern_sig.c:1970
#6  userret+0x335 single_thread_check sys/kern/kern_sig.c:1991 [inline]
#6  userret+0x335 sys/kern/kern_sig.c:1936
#7  syscall+0x55c mi_syscall_return sys/sys/syscall_mi.h:129 [inline]
#7  syscall+0x55c sys/arch/amd64/amd64/trap.c:612
#8  Xsyscall+0x128
ddb{1}> show malloc
           Type InUse  MemUse  HighUse   Limit  Requests Type Lim
         devbuf  9472   6346K    6346K  78643K     10562        0
            pcb    13      8K       8K  78643K        13        0
         rtable    83      2K       2K  78643K       153        0
         ifaddr    32      8K       8K  78643K        32        0
       counters    41     33K      33K  78643K        41        0
       ioctlops     0      0K       2K  78643K        14        0
          mount     1      1K       1K  78643K         1        0
         vnodes  1199     75K      75K  78643K      1204        0
      UFS quota     1     32K      32K  78643K         1        0
      UFS mount     5     36K      36K  78643K         5        0
            shm     2      1K       1K  78643K         2        0
         VM map     2      1K       1K  78643K         2        0
            sem     2      0K       0K  78643K         2        0
        dirhash    12      2K       2K  78643K        12        0
           ACPI  1825    197K     290K  78643K     13109        0
      file desc     4     12K      16K  78643K        19        0
           proc    48     50K      70K  78643K       325        0
        subproc    17      1K       1K  78643K        17        0
    NFS srvsock     1      0K       0K  78643K         1        0
     NFS daemon     1     16K      16K  78643K         1        0
       in_multi    22      1K       1K  78643K        22        0
    ether_multi     1      0K       0K  78643K         1        0
    ISOFS mount     1     32K      32K  78643K         1        0
  MSDOSFS mount     1     16K      16K  78643K         1        0
           ttys    19     95K      95K  78643K        19        0
           exec     0      0K       2K  78643K       295        0
        pagedep     1      8K       8K  78643K         1        0
       inodedep     1     32K      32K  78643K         1        0
         newblk     1      0K       0K  78643K         1        0
        VM swap     7     26K      26K  78643K         7        0
       UVM amap    61     11K      11K  78643K       615        0
       UVM aobj     3      2K       2K  78643K         3        0
        memdesc     1      4K       4K  78643K         1        0
    crypto data     1      1K       1K  78643K         1        0
            NDP     6      0K       0K  78643K         6        0
           temp    40   3954K    4017K  78643K      1667        0
         kqueue     3      4K       4K  78643K         3        0
      SYN cache     2     16K      16K  78643K         2        0
ddb{1}> show all pools
Name      Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle
arp         64        4    0        0     1     0     1     1     0     8    0
plcache    128       20    0        0     1     0     1     1     0     8    0
rtpcb      120       17    0       15     1     0     1     1     0     8    0
rtentry    112       34    0        1     1     0     1     1     0     8    0
unpcb      120       23    0       15     1     0     1     1     0     8    0
syncache   296        4    0        4     1     0     1     1     0     8    1
tcpcb      736       10    0        4     1     0     1     1     0     8    0
inpcb      296       28    0       22     1     0     1     1     0     8    0
nd6         48        3    0        0     1     0     1     1     0     8    0
kcovpl      48        1    0        0     1     0     1     1     0     8    0
art_heap8  4096       1    0        0     1     0     1     1     0     8    0
art_heap4  256      144    0        0     9     0     9     9     0     8    0
art_table   32      145    0        0     2     0     2     2     0     8    0
art_node    16       33    0        3     1     0     1     1     0     8    0
dirhash    1024      17    0        0     3     0     3     3     0     8    0
dino2pl    256     1412    0       21    87     0    87    87     0     8    0
ffsino     272     1412    0       21    93     0    93    93     0     8    0
nchpl      144     1627    0       36    59     0    59    59     0     8    0
uvmvnodes   72     1438    0        0    27     0    27    27     0     8    0
vnodes     208     1438    0        0    76     0    76    76     0     8    0
namei      1024    3829    0     3829     1     0     1     1     0     8    1
percpumem   16       31    0        0     1     0     1     1     0     8    0
scxspl     216     4776    0     4776     2     1     1     2     0     8    1
plimitpl   152       14    0        7     1     0     1     1     0     8    0
sigapl     424      212    0      182     4     0     4     4     0     8    0
futexpl     56        2    0        2     1     0     1     1     0     8    1
knotepl    112       62    0       50     1     0     1     1     0     8    0
kqueuepl   152        2    0        0     1     0     1     1     0     8    0
pipepl     304       69    0       61     1     0     1     1     0     8    0
fdescpl    496      196    0      182     2     0     2     2     0     8    0
filepl     152      921    0      849     3     0     3     3     0     8    0
lockfpl    104        5    0        4     1     0     1     1     0     8    0
lockfspl    48        3    0        2     1     0     1     1     0     8    0
sessionpl  144       16    0        7     1     0     1     1     0     8    0
pgrppl      48       16    0        7     1     0     1     1     0     8    0
ucredpl     96       51    0       43     1     0     1     1     0     8    0
zombiepl   144      182    0      182     1     0     1     1     0     8    1
processpl  1056     212    0      182     4     1     3     3     0     8    1
procpl     656      219    0      182     4     0     4     4     0     8    0
sockpl     400       68    0       52     2     0     2     2     0     8    0
mcl4k      4096       1    0        0     1     0     1     1     0     8    0
mcl2k      2048      80    0        0    10     0    10    10     0     8    0
mtagpl      96        1    0        0     1     0     1     1     0     8    0
mbufpl     256      106    0        0     7     0     7     7     0     8    0
bufpl      280     3445    0      173   234     0   234   234     0     8    0
anonpl      16    22577    0    20233    16     2    14    14     0   124    4
amapchunkpl 152     765    0      664     6     0     6     6     0   158    1
amappl16   192      168    0      100     4     0     4     4     0     8    0
amappl15   184        1    0        1     1     1     0     1     0     8    0
amappl14   176       26    0       18     1     0     1     1     0     8    0
amappl13   168       18    0       16     1     0     1     1     0     8    0
amappl12   160       11    0        9     2     1     1     1     0     8    0
amappl11   152       42    0       33     1     0     1     1     0     8    0
amappl10   144        6    0        2     1     0     1     1     0     8    0
amappl9    136      391    0      390     1     0     1     1     0     8    0
amappl8    128       86    0       70     1     0     1     1     0     8    0
amappl7    120      202    0      196     1     0     1     1     0     8    0
amappl6    112       54    0       46     1     0     1     1     0     8    0
amappl5    104      318    0      303     1     0     1     1     0     8    0
amappl4     96      243    0      220     1     0     1     1     0     8    0
amappl3     88      100    0       92     1     0     1     1     0     8    0
amappl2     80      835    0      776     2     0     2     2     0     8    0
amappl1     72    13177    0    12733    22     4    18    18     0     8    8
amappl      80      417    0      383     1     0     1     1     0    84    0
dma4096    4096       1    0        1     1     1     0     1     0     8    0
dma1024    1024       1    0        0     1     0     1     1     0     8    0
dma256     256        6    0        6     1     1     0     1     0     8    0
dma128     128      253    0      253     1     1     0     1     0     8    0
dma64       64        6    0        6     1     1     0     1     0     8    0
dma32       32        7    0        7     1     1     0     1     0     8    0
dma16       16       18    0       17     1     0     1     1     0     8    0
aobjpl      64        2    0        0     1     0     1     1     0     8    0
uaddrrnd    24      196    0      182     1     0     1     1     0     8    0
uaddrbest   32        2    0        0     1     0     1     1     0     8    0
uaddr       24      196    0      182     1     0     1     1     0     8    0
vmmpekpl   168     5706    0     5689     1     0     1     1     0     8    0
vmmpepl    168    29861    0    28720    80     7    73    73     0   357   23
vmsppl     368      195    0      182     2     0     2     2     0     8    0
pdppl      4096     399    0      364    47    10    37    37     0     8    2
pvpl        32   108846    0   103608   117     0   117   117     0   265   74
pmappl     232      195    0      182     1     0     1     1     0     8    0
extentpl    40       58    0       40     1     0     1     1     0     8    0
phpool     112      267    0       12     8     0     8     8     0     8    0
ddb{1}> machine ddbcpu 0
Stopped at      x86_ipi_db+0x1a:        addq    $0x8,%rsp
ddb{0}> trace
x86_ipi_db(ffffffff826c3ff0) at x86_ipi_db+0x1a sys/arch/amd64/amd64/db_interface.c:352
x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_lock(ffffffff828f9688) at __mp_lock+0x122 __mp_lock_spin sys/kern/kern_lock.c:116 [inline]
__mp_lock(ffffffff828f9688) at __mp_lock+0x122 sys/kern/kern_lock.c:147
__mp_acquire_count(ffffffff828f9688,1) at __mp_acquire_count+0x4c sys/kern/kern_lock.c:227
mi_switch() at mi_switch+0x390 sys/kern/sched_bsd.c:433
sleep_finish(ffff80002349f270,1) at sleep_finish+0x111 sys/kern/kern_synch.c:418
single_thread_set(ffff8000211c2a48,3,0) at single_thread_set+0x3f4 single_thread_wait sys/kern/kern_sig.c:2112 [inline]
single_thread_set(ffff8000211c2a48,3,0) at single_thread_set+0x3f4 sys/kern/kern_sig.c:2092
exit1(ffff8000211c2a48,0,0,1) at exit1+0x98 sys/kern/kern_exit.c:138
sys_exit(ffff8000211c2a48,ffff80002349f3b0,ffff80002349f400) at sys_exit+0x16 sys/kern/kern_exit.c:95
syscall(ffff80002349f480) at syscall+0x4a1 mi_syscall sys/sys/syscall_mi.h:102 [inline]
syscall(ffff80002349f480) at syscall+0x4a1 sys/arch/amd64/amd64/trap.c:590
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x7f7ffffd2440, count: -12
ddb{0}> machine ddbcpu 1
Stopped at      spllower+0x34:  movq    0x5c0(%r12,%rax,8),%rbx
ddb{1}> trace
spllower(ffff80002129a000) at spllower+0x34 sys/arch/amd64/amd64/intr.c:716
single_thread_check_locked(ffff8000212919b8,0) at single_thread_check_locked+0x196 sys/kern/kern_sig.c:1970
userret(ffff8000212919b8) at userret+0x335 single_thread_check sys/kern/kern_sig.c:1991 [inline]
userret(ffff8000212919b8) at userret+0x335 sys/kern/kern_sig.c:1936
syscall(ffff8000234a5ac0) at syscall+0x55c mi_syscall_return sys/sys/syscall_mi.h:129 [inline]
syscall(ffff8000234a5ac0) at syscall+0x55c sys/arch/amd64/amd64/trap.c:612
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x73754c5d580, count: -5

Crashes (12):
Time Kernel Commit Syzkaller Config Log Report Syz repro C repro VM info Assets (help?) Manager Title
2020/12/03 00:27 openbsd cbb32f9c11ef 8c9190ef .config console log report ci-openbsd-setuid
2020/12/03 00:27 openbsd cbb32f9c11ef 8c9190ef .config console log report ci-openbsd-setuid
2020/12/03 00:27 openbsd cbb32f9c11ef 8c9190ef .config console log report ci-openbsd-setuid
2020/12/03 00:08 openbsd cbb32f9c11ef 8c9190ef .config console log report ci-openbsd-multicore
2020/12/03 00:08 openbsd cbb32f9c11ef 8c9190ef .config console log report ci-openbsd-multicore
2020/12/03 00:08 openbsd cbb32f9c11ef 8c9190ef .config console log report ci-openbsd-multicore
2020/12/02 23:58 openbsd 1ea95950577f 8c9190ef .config console log report ci-openbsd-setuid
2020/12/02 23:58 openbsd 1ea95950577f 8c9190ef .config console log report ci-openbsd-setuid
2020/12/02 23:58 openbsd 1ea95950577f 8c9190ef .config console log report ci-openbsd-setuid
2020/12/02 23:40 openbsd 1ea95950577f 8c9190ef .config console log report ci-openbsd-multicore
2020/12/02 23:40 openbsd 1ea95950577f 8c9190ef .config console log report ci-openbsd-multicore
2020/12/02 23:40 openbsd 1ea95950577f 8c9190ef .config console log report ci-openbsd-multicore
* Struck through repros no longer work on HEAD.