possible deadlock in __generic_file

possible deadlock in __generic_file_fsync
Status: upstream: reported C repro on 2018/10/16 14:25
Reported-by: syzbot+5cd33f0e6abe2bb3e397@syzkaller.appspotmail.com
First crash: 937d, last: 797d

Cause bisection: introduced by (bisect log) :
commit 9022ada8ab6f1f1a932a3c93815061042e6548a5
Author: Linus Torvalds <torvalds@linux-foundation.org>
Date: Fri Aug 24 20:16:36 2018 +0000

Merge branch 'for-4.19' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq

Crash: possible deadlock in flush_workqueue (log)
Repro: C syz .config

Fix bisection: fixed by (bisect log) [no-op commit]:
commit f1770e3ca4ec10ce43825de2f855a1831c711195
Author: Jiri Olsa <jolsa@kernel.org>
Date: Fri Oct 12 11:20:16 2018 +0000

tools power x86_energy_perf_policy: Override CFLAGS assignments and add LDFLAGS to build command

similar bugs (3):
Kernel	Title	Repro	Count	Last	Reported	Patched	Status
linux-4.19	possible deadlock in __generic_file_fsync	C	256	9h03m	697d	0/1	upstream: reported C repro on 2019/06/13 18:12
linux-4.14	possible deadlock in __generic_file_fsync	syz	146	19h43m	690d	0/1	upstream: reported syz repro on 2019/06/19 20:20
android-414	possible deadlock in __generic_file_fsync	syz	262	525d	759d	0/1	public: reported syz repro on 2019/04/12 00:01

Sample crash report:

startpar (5771) used greatest stack depth: 15744 bytes left

======================================================
WARNING: possible circular locking dependency detected
4.20.0-rc3+ #122 Not tainted
------------------------------------------------------
kworker/0:2/2794 is trying to acquire lock:
000000004036ff52 (&sb->s_type->i_mutex_key#9){+.+.}, at: inode_lock include/linux/fs.h:757 [inline]
000000004036ff52 (&sb->s_type->i_mutex_key#9){+.+.}, at: __generic_file_fsync+0xb5/0x200 fs/libfs.c:981

but task is already holding lock:
000000005a81f984 ((work_completion)(&dio->complete_work)){+.+.}, at: process_one_work+0xb9a/0x1c40 kernel/workqueue.c:2128

which lock already depends on the new lock.


the existing dependency chain (in reverse order) is:

-> #2 ((work_completion)(&dio->complete_work)){+.+.}:
       process_one_work+0xc0a/0x1c40 kernel/workqueue.c:2129
       worker_thread+0x17f/0x1390 kernel/workqueue.c:2296
       kthread+0x35a/0x440 kernel/kthread.c:246
       ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:352

-> #1 ((wq_completion)"dio/%s"sb->s_id){+.+.}:
       flush_workqueue+0x30a/0x1e10 kernel/workqueue.c:2655
       drain_workqueue+0x2a9/0x640 kernel/workqueue.c:2820
       destroy_workqueue+0xc6/0x9c0 kernel/workqueue.c:4155
       sb_init_dio_done_wq+0x74/0x90 fs/direct-io.c:634
       do_blockdev_direct_IO+0x1325/0x9db0 fs/direct-io.c:1283
       __blockdev_direct_IO+0x9d/0xc6 fs/direct-io.c:1417
       ext4_direct_IO_write fs/ext4/inode.c:3774 [inline]
       ext4_direct_IO+0xbe6/0x2230 fs/ext4/inode.c:3901
       generic_file_direct_write+0x275/0x4b0 mm/filemap.c:3043
       __generic_file_write_iter+0x2ff/0x630 mm/filemap.c:3222
       ext4_file_write_iter+0x390/0x1420 fs/ext4/file.c:266
       call_write_iter include/linux/fs.h:1857 [inline]
       aio_write+0x3b1/0x610 fs/aio.c:1561
       io_submit_one+0xaa1/0xf80 fs/aio.c:1835
       __do_sys_io_submit fs/aio.c:1916 [inline]
       __se_sys_io_submit fs/aio.c:1887 [inline]
       __x64_sys_io_submit+0x1b7/0x580 fs/aio.c:1887
       do_syscall_64+0x1b9/0x820 arch/x86/entry/common.c:290
       entry_SYSCALL_64_after_hwframe+0x49/0xbe

-> #0 (&sb->s_type->i_mutex_key#9){+.+.}:
       lock_acquire+0x1ed/0x520 kernel/locking/lockdep.c:3844
       down_write+0x8a/0x130 kernel/locking/rwsem.c:70
       inode_lock include/linux/fs.h:757 [inline]
       __generic_file_fsync+0xb5/0x200 fs/libfs.c:981
       ext4_sync_file+0xa4f/0x1510 fs/ext4/fsync.c:120
       vfs_fsync_range+0x140/0x220 fs/sync.c:197
       generic_write_sync include/linux/fs.h:2781 [inline]
       dio_complete+0x75c/0x9e0 fs/direct-io.c:329
       dio_aio_complete_work+0x20/0x30 fs/direct-io.c:341
       process_one_work+0xc90/0x1c40 kernel/workqueue.c:2153
       worker_thread+0x17f/0x1390 kernel/workqueue.c:2296
       kthread+0x35a/0x440 kernel/kthread.c:246
       ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:352

other info that might help us debug this:

Chain exists of:
  &sb->s_type->i_mutex_key#9 --> (wq_completion)"dio/%s"sb->s_id --> (work_completion)(&dio->complete_work)

 Possible unsafe locking scenario:

       CPU0                    CPU1
       ----                    ----
  lock((work_completion)(&dio->complete_work));
                               lock((wq_completion)"dio/%s"sb->s_id);
                               lock((work_completion)(&dio->complete_work));
  lock(&sb->s_type->i_mutex_key#9);

 *** DEADLOCK ***

2 locks held by kworker/0:2/2794:
 #0: 0000000025665699 ((wq_completion)"dio/%s"sb->s_id){+.+.}, at: __write_once_size include/linux/compiler.h:209 [inline]
 #0: 0000000025665699 ((wq_completion)"dio/%s"sb->s_id){+.+.}, at: arch_atomic64_set arch/x86/include/asm/atomic64_64.h:34 [inline]
 #0: 0000000025665699 ((wq_completion)"dio/%s"sb->s_id){+.+.}, at: atomic64_set include/asm-generic/atomic-instrumented.h:40 [inline]
 #0: 0000000025665699 ((wq_completion)"dio/%s"sb->s_id){+.+.}, at: atomic_long_set include/asm-generic/atomic-long.h:59 [inline]
 #0: 0000000025665699 ((wq_completion)"dio/%s"sb->s_id){+.+.}, at: set_work_data kernel/workqueue.c:617 [inline]
 #0: 0000000025665699 ((wq_completion)"dio/%s"sb->s_id){+.+.}, at: set_work_pool_and_clear_pending kernel/workqueue.c:644 [inline]
 #0: 0000000025665699 ((wq_completion)"dio/%s"sb->s_id){+.+.}, at: process_one_work+0xb43/0x1c40 kernel/workqueue.c:2124
 #1: 000000005a81f984 ((work_completion)(&dio->complete_work)){+.+.}, at: process_one_work+0xb9a/0x1c40 kernel/workqueue.c:2128

stack backtrace:
CPU: 0 PID: 2794 Comm: kworker/0:2 Not tainted 4.20.0-rc3+ #122
Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011
Workqueue: dio/sda1 dio_aio_complete_work
Call Trace:
 __dump_stack lib/dump_stack.c:77 [inline]
 dump_stack+0x244/0x39d lib/dump_stack.c:113
 print_circular_bug.isra.35.cold.54+0x1bd/0x27d kernel/locking/lockdep.c:1221
 check_prev_add kernel/locking/lockdep.c:1863 [inline]
 check_prevs_add kernel/locking/lockdep.c:1976 [inline]
 validate_chain kernel/locking/lockdep.c:2347 [inline]
 __lock_acquire+0x3399/0x4c20 kernel/locking/lockdep.c:3341
 lock_acquire+0x1ed/0x520 kernel/locking/lockdep.c:3844
 down_write+0x8a/0x130 kernel/locking/rwsem.c:70
 inode_lock include/linux/fs.h:757 [inline]
 __generic_file_fsync+0xb5/0x200 fs/libfs.c:981
 ext4_sync_file+0xa4f/0x1510 fs/ext4/fsync.c:120
 vfs_fsync_range+0x140/0x220 fs/sync.c:197
 generic_write_sync include/linux/fs.h:2781 [inline]
 dio_complete+0x75c/0x9e0 fs/direct-io.c:329
 dio_aio_complete_work+0x20/0x30 fs/direct-io.c:341
 process_one_work+0xc90/0x1c40 kernel/workqueue.c:2153
 worker_thread+0x17f/0x1390 kernel/workqueue.c:2296
 kthread+0x35a/0x440 kernel/kthread.c:246
 ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:352
syz-executor810 (5947) used greatest stack depth: 14752 bytes left
syz-executor810 (5966) used greatest stack depth: 13976 bytes left
syz-executor810 (6126) used greatest stack depth: 12408 bytes left

Crashes (80):
Manager	Time	Kernel	Commit	Syzkaller	Config	Log	Report	Syz repro	C repro
ci-upstream-kasan-gce-smack-root	2018/11/21 09:49	upstream	c8ce94b8	5d9a3924	.config	log	report	syz	C
ci-upstream-kasan-gce-root	2018/10/20 16:12	upstream	270b77a0	ecb386fe	.config	log	report	syz	C
ci-upstream-kasan-gce-smack-root	2018/10/30 17:04	upstream	11743c56	8dbb755a	.config	log	report	syz
ci-upstream-kasan-gce-selinux-root	2018/10/30 07:25	upstream	4b427452	2f1090da	.config	log	report	syz
ci-upstream-kasan-gce-smack-root	2018/10/21 13:16	upstream	23469de6	ecb386fe	.config	log	report	syz
ci-upstream-kasan-gce-smack-root	2018/10/19 03:37	upstream	fa520c47	9aba67b5	.config	log	report	syz
ci-upstream-kasan-gce-root	2018/10/19 02:09	upstream	fa520c47	9aba67b5	.config	log	report	syz
ci-upstream-linux-next-kasan-gce-root	2018/10/20 11:42	linux-next	8c60c36d	ecb386fe	.config	log	report	syz
ci-upstream-linux-next-kasan-gce-root	2018/10/19 04:33	linux-next	9bab6434	9aba67b5	.config	log	report	syz
ci-upstream-kasan-gce-selinux-root	2019/03/05 07:54	upstream	cd2a3bf0	bb91cf81	.config	log	report
ci-upstream-kasan-gce-selinux-root	2019/03/02 19:47	upstream	c93d9218	1c0e457a	.config	log	report
ci-upstream-kasan-gce-smack-root	2019/03/01 02:57	upstream	7d762d69	09aeeba4	.config	log	report
ci-upstream-kasan-gce-selinux-root	2019/02/27 23:18	upstream	7d762d69	34ec456b	.config	log	report
ci-upstream-kasan-gce-selinux-root	2019/02/27 10:33	upstream	7d762d69	083cfd0e	.config	log	report
ci-upstream-kasan-gce-smack-root	2019/02/26 22:29	upstream	7d762d69	f2468c12	.config	log	report
ci-upstream-kasan-gce-smack-root	2019/02/23 10:58	upstream	cb268d80	18107ce0	.config	log	report
ci-upstream-kasan-gce-smack-root	2019/02/21 23:28	upstream	8a61716f	7ff74a98	.config	log	report
ci-upstream-kasan-gce-selinux-root	2019/02/20 21:41	upstream	2137397c	c95f0707	.config	log	report
ci-upstream-kasan-gce-root	2019/02/20 01:48	upstream	40e196a9	4df543c9	.config	log	report
ci-upstream-kasan-gce-root	2019/02/17 14:53	upstream	64c0133e	3e98cc30	.config	log	report
ci-upstream-kasan-gce-selinux-root	2019/02/16 16:44	upstream	5ded5871	f42dee6d	.config	log	report
ci-upstream-kasan-gce-smack-root	2019/02/15 21:34	upstream	cb5b020a	f6f233c0	.config	log	report
ci-upstream-kasan-gce-selinux-root	2019/02/14 16:35	upstream	1f947a7a	76dd003f	.config	log	report
ci-upstream-kasan-gce-smack-root	2019/01/04 14:21	upstream	96d4f267	7da23925	.config	log	report
ci-upstream-kasan-gce-selinux-root	2019/01/03 07:14	upstream	85f78456	06a2b89f	.config	log	report
ci-upstream-kasan-gce-root	2018/12/30 08:39	upstream	19530313	35e3f847	.config	log	report
ci-upstream-kasan-gce-root	2018/12/30 01:05	upstream	19530313	35e3f847	.config	log	report
ci-upstream-kasan-gce-smack-root	2018/12/28 23:54	upstream	00c569b5	e33ad0f1	.config	log	report
ci-upstream-kasan-gce-smack-root	2018/12/26 14:26	upstream	5694cecd	8a41a0ad	.config	log	report
ci-upstream-kasan-gce-root	2018/12/24 02:44	upstream	3c730b10	e3bd7ab8	.config	log	report
ci-upstream-kasan-gce-smack-root	2018/12/16 12:36	upstream	6531e115	def91db3	.config	log	report
ci-upstream-kasan-gce-selinux-root	2018/12/14 08:21	upstream	65e08c5e	fe7127be	.config	log	report
ci-upstream-kasan-gce-smack-root	2018/12/09 21:05	upstream	0844895a	979179d6	.config	log	report
ci-upstream-kasan-gce-selinux-root	2018/12/08 15:30	upstream	5f179793	60562a1d	.config	log	report
ci-upstream-kasan-gce-root	2018/12/06 09:49	upstream	d0897090	764b42c4	.config	log	report
ci-upstream-kasan-gce-selinux-root	2018/12/04 23:33	upstream	0072a0c1	f162ad97	.config	log	report
ci-upstream-kasan-gce-selinux-root	2018/12/02 22:01	upstream	6a512726	e0d8c853	.config	log	report
ci-upstream-kasan-gce-root	2018/11/30 01:11	upstream	f92a2ebb	66071e27	.config	log	report
ci-upstream-kasan-gce-selinux-root	2018/11/28 05:25	upstream	ef78e5ec	4b6d14f2	.config	log	report
ci-upstream-kasan-gce-selinux-root	2018/11/27 13:31	upstream	ef78e5ec	4b6d14f2	.config	log	report
ci-upstream-kasan-gce-root	2018/11/26 11:21	upstream	2e6e902d	ac912200	.config	log	report
ci-upstream-kasan-gce-root	2018/11/26 05:42	upstream	d6d460b8	3d3ec907	.config	log	report
ci-upstream-kasan-gce-smack-root	2018/11/25 22:34	upstream	d6d460b8	3d3ec907	.config	log	report
ci-upstream-kasan-gce-selinux-root	2018/11/25 08:29	upstream	e195ca6c	3d3ec907	.config	log	report
ci-upstream-kasan-gce-root	2018/11/22 20:41	upstream	edeca3a7	87815d9d	.config	log	report
ci-upstream-kasan-gce-smack-root	2018/11/21 10:11	upstream	c8ce94b8	5d9a3924	.config	log	report
ci-upstream-kasan-gce-root	2018/11/21 07:28	upstream	c8ce94b8	5d9a3924	.config	log	report
ci-upstream-kasan-gce-root	2018/11/20 17:58	upstream	06e68fed	9aca6b52	.config	log	report
ci-upstream-kasan-gce-selinux-root	2018/11/20 05:48	upstream	f2ce1065	9bc2a903	.config	log	report
ci-upstream-kasan-gce-root	2018/11/12 20:56	upstream	ccda4af0	74dbb806	.config	log	report
ci-upstream-kasan-gce-smack-root	2018/11/09 05:23	upstream	24ccea7e	8fd01d3a	.config	log	report
ci-upstream-kasan-gce-selinux-root	2018/11/06 05:46	upstream	163c8d54	8bd6bd63	.config	log	report
ci-upstream-kasan-gce-root	2018/10/31 00:00	upstream	11743c56	8dbb755a	.config	log	report
ci-upstream-kasan-gce-smack-root	2018/10/25 21:25	upstream	bd6bf7c1	a8292de9	.config	log	report
ci-upstream-kasan-gce-selinux-root	2018/10/22 00:42	upstream	467e050e	ecb386fe	.config	log	report
ci-upstream-kasan-gce-smack-root	2018/10/20 22:45	upstream	b0d04fb5	ecb386fe	.config	log	report
ci-upstream-kasan-gce-root	2018/10/19 00:21	upstream	fa520c47	9aba67b5	.config	log	report
ci-upstream-kasan-gce-smack-root	2018/10/16 22:23	upstream	b955a910	1ba7fd7e	.config	log	report
ci-upstream-linux-next-kasan-gce-root	2019/02/28 10:41	linux-next	42fd8df9	09aeeba4	.config	log	report
ci-upstream-linux-next-kasan-gce-root	2019/02/26 07:52	linux-next	2b46440e	8022bafd	.config	log	report
ci-upstream-linux-next-kasan-gce-root	2019/02/25 02:09	linux-next	94a47529	7a06e792	.config	log	report
ci-upstream-linux-next-kasan-gce-root	2019/02/22 05:21	linux-next	550f4769	7ff74a98	.config	log	report
ci-upstream-linux-next-kasan-gce-root	2019/02/15 02:14	linux-next	b3418f8b	76dd003f	.config	log	report
ci-upstream-linux-next-kasan-gce-root	2018/11/29 17:05	linux-next	442b8cea	4b6d14f2	.config	log	report
ci-upstream-linux-next-kasan-gce-root	2018/11/25 20:17	linux-next	442b8cea	3d3ec907	.config	log	report
ci-upstream-linux-next-kasan-gce-root	2018/11/21 06:06	linux-next	442b8cea	5d9a3924	.config	log	report
ci-upstream-linux-next-kasan-gce-root	2018/11/17 13:20	linux-next	442b8cea	b08ee62a	.config	log	report
ci-upstream-linux-next-kasan-gce-root	2018/11/04 20:03	linux-next	25e9471b	8bd6bd63	.config	log	report
ci-upstream-linux-next-kasan-gce-root	2018/11/04 03:17	linux-next	25e9471b	8bd6bd63	.config	log	report
ci-upstream-linux-next-kasan-gce-root	2018/11/01 23:17	linux-next	4db9d11b	1f38e9ae	.config	log	report
ci-upstream-linux-next-kasan-gce-root	2018/10/30 16:14	linux-next	6201f31a	8dbb755a	.config	log	report
ci-upstream-linux-next-kasan-gce-root	2018/10/27 17:27	linux-next	8c60c36d	8efba39a	.config	log	report
ci-upstream-linux-next-kasan-gce-root	2018/10/23 23:27	linux-next	8c60c36d	a8292de9	.config	log	report
ci-upstream-linux-next-kasan-gce-root	2018/10/23 02:38	linux-next	8c60c36d	ecb386fe	.config	log	report
ci-upstream-linux-next-kasan-gce-root	2018/10/20 05:24	linux-next	8c60c36d	ecb386fe	.config	log	report
ci-upstream-linux-next-kasan-gce-root	2018/10/16 11:57	linux-next	6d5d8241	8cd30605	.config	log	report