bisecting cause commit starting from 761c6d7ec820f123b931e7b8ef7ec7c8564e450f building syzkaller on 6972b10616d785401dea17cec890cca8916424a7 testing commit 761c6d7ec820f123b931e7b8ef7ec7c8564e450f compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.1 kernel signature: b96a39ac12a2c1828b775890b0d9a36477d9834226e1ee12c691ad30a29b151d run #0: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #1: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #2: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #3: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #4: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #5: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #6: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #7: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #8: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #9: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #10: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #11: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #12: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #13: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #14: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #15: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #16: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #17: crashed: general protection fault in free_swap_cache run #18: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #19: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end testing release v5.13 testing commit 62fb9874f5da54fdb243003b386128037319b219 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.1 kernel signature: 909f5b513362d755e99068620c19525cb2afd91ca8725228219fd46574a48866 run #0: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #1: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #2: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #3: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #4: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #5: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #6: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #7: crashed: KASAN: out-of-bounds Write in ext4_write_inline_data_end run #8: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #9: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end testing release v5.12 testing commit 9f4ad9e425a1d3b6a34617b8ea226d56a119a717 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.1 kernel signature: bb4e26dd8f4a005b0fd76547575ef29621d4d502b5251ddaf0a5696974104629 run #0: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #1: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #2: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #3: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #4: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #5: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #6: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #7: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #8: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #9: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end testing release v5.11 testing commit f40ddce88593482919761f74910f42f4b84c004b compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.1 kernel signature: 3222e6ed80513dd888ba5df1ed0bb0fed90b38311471a07f8ebfef14f678fd42 run #0: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #1: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #2: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #3: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #4: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #5: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #6: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #7: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #8: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #9: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end testing release v5.10 testing commit 2c85ebc57b3e1817b6ce1a6b703928e113a90442 compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.1 kernel signature: 35c88fb67e0626f0232c5254d6e9f188ff33bdc5ecf75a2e20b39b0354577e82 run #0: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #1: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #2: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #3: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #4: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #5: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #6: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #7: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #8: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #9: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end testing release v5.9 testing commit bbf5c979011a099af5dc76498918ed7df445635b compiler: gcc (GCC) 10.2.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.1 kernel signature: 4753fc6113ccf19b73886623e431f1d42203e5df7900561c23ce4babeaf8f756 run #0: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #1: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #2: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #3: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #4: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #5: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #6: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #7: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #8: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #9: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end testing release v5.8 testing commit bcf876870b95592b52519ed4aafcf9d95999bc9c compiler: gcc (GCC) 8.4.1 20210217, GNU ld (GNU Binutils for Debian) 2.35.1 kernel signature: 64bef9c619e4f9660afe7c971d6b25d7fe2654ae1e3be1137534ef774e1b1619 run #0: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #1: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #2: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #3: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #4: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #5: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #6: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #7: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #8: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #9: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end testing release v5.7 testing commit 3d77e6a8804abcc0504c904bd6e5cdf3a5cf8162 compiler: gcc version 8.4.1 20210217 (GCC), GNU ld (GNU Binutils for Debian) 2.35.1 kernel signature: f76fe31c01316385978519b5b4e89add706a0c3d5fe63afc28097ce825bd70af run #0: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #1: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #2: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #3: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #4: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #5: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #6: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #7: crashed: KASAN: slab-out-of-bounds Write in ext4_write_inline_data_end run #8: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end run #9: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end testing release v5.6 testing commit 7111951b8d4973bda27ff663f2cf18b663d15b48 compiler: gcc version 8.4.1 20210217 (GCC) kernel signature: ce1c40f29ea7dac1ab1efcb8a4c28e91d707e6db2dd3c7ba2f4d3ff05339b929 all runs: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end testing release v5.5 testing commit d5226fa6dbae0569ee43ecfc08bdcd6770fc4755 compiler: gcc version 8.4.1 20210217 (GCC) kernel signature: 52e3d19b3dd1254ec0fc70d5e850d8b373790b6a1ce2b4832bb7a2a2d3945561 all runs: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end testing release v5.4 testing commit 219d54332a09e8d8741c1e1982f5eae56099de85 compiler: gcc version 8.4.1 20210217 (GCC) kernel signature: aaf25273084462a2ad615141adb1da70de0e843313526173854dc50400e6d06c all runs: crashed: KASAN: use-after-free Write in ext4_write_inline_data_end testing release v5.3 testing commit 4d856f72c10ecb060868ed10ff1b1453943fc6c8 compiler: gcc version 8.4.1 20210217 (GCC) kernel signature: 6a118b86ece6a00ec309dfe958551a7e37b62c5a38c74bfffbdf71d5ff2118cf all runs: crashed: KASAN: use-after-free Write in ext4_write_inline_data testing release v5.2 testing commit 0ecfebd2b52404ae0c54a878c872bb93363ada36 compiler: gcc version 8.4.1 20210217 (GCC) kernel signature: dc0a4e06aabc5a71b7bb3c51a117c73e328564b4a4f21cfd43ea9747d8ce137d run #0: crashed: BUG: MAX_STACK_TRACE_ENTRIES too low! run #1: crashed: BUG: MAX_STACK_TRACE_ENTRIES too low! run #2: crashed: BUG: MAX_STACK_TRACE_ENTRIES too low! run #3: crashed: BUG: MAX_STACK_TRACE_ENTRIES too low! run #4: crashed: BUG: MAX_STACK_TRACE_ENTRIES too low! run #5: crashed: BUG: MAX_STACK_TRACE_ENTRIES too low! run #6: crashed: BUG: MAX_STACK_TRACE_ENTRIES too low! run #7: crashed: BUG: MAX_STACK_TRACE_ENTRIES too low! run #8: OK run #9: OK testing release v5.1 testing commit e93c9c99a629c61837d5a7fc2120cd2b6c70dbdd compiler: gcc version 8.4.1 20210217 (GCC) kernel signature: 6f28a54ae9631f7a38ae82812fc557e8efc8855fdd74a27360b3ac8c36083bee all runs: crashed: BUG: MAX_STACK_TRACE_ENTRIES too low! testing release v5.0 testing commit 1c163f4c7b3f621efff9b28a47abb36f7378d783 compiler: gcc version 8.4.1 20210217 (GCC) kernel signature: cc4af3f53694d3fa6e61b52de0e00edfb6449f3b5e2c0a6ea7e7eb72c3e13ece all runs: OK # git bisect start e93c9c99a629c61837d5a7fc2120cd2b6c70dbdd 1c163f4c7b3f621efff9b28a47abb36f7378d783 Bisecting: 7074 revisions left to test after this (roughly 13 steps) [b5dd0c658c31b469ccff1b637e5124851e7a4a1c] Merge branch 'akpm' (patches from Andrew) testing commit b5dd0c658c31b469ccff1b637e5124851e7a4a1c compiler: gcc version 8.4.1 20210217 (GCC) kernel signature: 61af8cf1c51e8853a710ecadbb95b3a9a63c95cd38cd9b30e029d4ab97e0f4df all runs: crashed: BUG: MAX_STACK_TRACE_ENTRIES too low! # git bisect bad b5dd0c658c31b469ccff1b637e5124851e7a4a1c Bisecting: 3569 revisions left to test after this (roughly 12 steps) [3478588b5136966c80c571cf0006f08e9e5b8f04] Merge branch 'locking-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip testing commit 3478588b5136966c80c571cf0006f08e9e5b8f04 compiler: gcc version 8.4.1 20210217 (GCC) kernel signature: 1d726204187ff3388d7e342fbb4293a36d9b143a473f5c31dab6cbb608306e7f all runs: crashed: BUG: MAX_STACK_TRACE_ENTRIES too low! # git bisect bad 3478588b5136966c80c571cf0006f08e9e5b8f04 Bisecting: 1673 revisions left to test after this (roughly 11 steps) [1a2566085650be593d464c4d73ac2d20ff67c058] Merge tag 'wireless-drivers-next-for-davem-2019-02-22' of git://git.kernel.org/pub/scm/linux/kernel/git/kvalo/wireless-drivers-next testing commit 1a2566085650be593d464c4d73ac2d20ff67c058 compiler: gcc version 8.4.1 20210217 (GCC) kernel signature: 65e9cb0635a6f9b5bdcde8262c086d8c055141300f588ca41213e048172cc167 all runs: OK # git bisect good 1a2566085650be593d464c4d73ac2d20ff67c058 Bisecting: 1091 revisions left to test after this (roughly 10 steps) [18a4d8bf250a33c015955f0dec27259780ef6448] Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/net testing commit 18a4d8bf250a33c015955f0dec27259780ef6448 compiler: gcc version 8.4.1 20210217 (GCC) kernel signature: 55a51cd13c21f7afda38feb556945c2dbc91dba3e7164aca3195fbf7cc9d6b1c all runs: OK # git bisect good 18a4d8bf250a33c015955f0dec27259780ef6448 Bisecting: 580 revisions left to test after this (roughly 9 steps) [6456300356433873309a1cae6aa05e77d6b59153] Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/net-next testing commit 6456300356433873309a1cae6aa05e77d6b59153 compiler: gcc version 8.4.1 20210217 (GCC) kernel signature: 7a1233a00fbbd2ce01d1715c784375f4c1f75cd73d00277e462b5703c8530f9e all runs: OK # git bisect good 6456300356433873309a1cae6aa05e77d6b59153 Bisecting: 310 revisions left to test after this (roughly 8 steps) [8feed3efa8022107bcb3432ac3ec9917e078ae70] Merge branch 'parisc-5.1-1' of git://git.kernel.org/pub/scm/linux/kernel/git/deller/parisc-linux testing commit 8feed3efa8022107bcb3432ac3ec9917e078ae70 compiler: gcc version 8.4.1 20210217 (GCC) kernel signature: ab0793688085f283024531806438a7e4c2b96d503bdc9b89974585dbec8987a1 run #0: basic kernel testing failed: general protection fault in batadv_iv_ogm_queue_add run #1: OK run #2: OK run #3: OK run #4: OK run #5: OK run #6: OK run #7: OK run #8: OK run #9: OK # git bisect good 8feed3efa8022107bcb3432ac3ec9917e078ae70 Bisecting: 137 revisions left to test after this (roughly 7 steps) [b1b988a6a035212f5ea205155c49ce449beedee8] Merge branch 'timers-2038-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip testing commit b1b988a6a035212f5ea205155c49ce449beedee8 compiler: gcc version 8.4.1 20210217 (GCC) kernel signature: 60ec87433957dc6b54761db6d00b05d0c9fc4ccf97cf682f00735fc1444df965 all runs: OK # git bisect good b1b988a6a035212f5ea205155c49ce449beedee8 Bisecting: 64 revisions left to test after this (roughly 6 steps) [e7ffb4eb9a6d89678e7f62461737899f88dab64e] Merge branches 'doc.2019.01.26a', 'fixes.2019.01.26a', 'sil.2019.01.26a', 'spdx.2019.02.09a', 'srcu.2019.01.26a' and 'torture.2019.01.26a' into HEAD testing commit e7ffb4eb9a6d89678e7f62461737899f88dab64e compiler: gcc version 8.4.1 20210217 (GCC) kernel signature: e52179f4e78c329b111c35be4ec349c24e97db746150b8829616c89999fa0899 all runs: OK # git bisect good e7ffb4eb9a6d89678e7f62461737899f88dab64e Bisecting: 32 revisions left to test after this (roughly 5 steps) [15ea86b58c71d05e0921bebcf707aa30e43e9e25] locking/lockdep: Fix reported required memory size (2/2) testing commit 15ea86b58c71d05e0921bebcf707aa30e43e9e25 compiler: gcc version 8.4.1 20210217 (GCC) kernel signature: d439e518f78ee23d9b75398bc7719f74ffb100cd73ce19be43234cb244d9850c all runs: OK # git bisect good 15ea86b58c71d05e0921bebcf707aa30e43e9e25 Bisecting: 16 revisions left to test after this (roughly 4 steps) [669de8bda87b92ab9a2fc663b3f5743c2ad1ae9f] kernel/workqueue: Use dynamic lockdep keys for workqueues testing commit 669de8bda87b92ab9a2fc663b3f5743c2ad1ae9f compiler: gcc version 8.4.1 20210217 (GCC) kernel signature: eead89ba29d17f074383a04fe5b23de4c59d11232d37ff224165ac596b63d775 all runs: crashed: WARNING: locking bug in ext4_sync_fs # git bisect bad 669de8bda87b92ab9a2fc663b3f5743c2ad1ae9f Bisecting: 7 revisions left to test after this (roughly 3 steps) [a0b0fd53e1e67639b303b15939b9c653dbe7a8c4] locking/lockdep: Free lock classes that are no longer in use testing commit a0b0fd53e1e67639b303b15939b9c653dbe7a8c4 compiler: gcc version 8.4.1 20210217 (GCC) kernel signature: 93bfc9e7ffdc6a6affcf0e5dabcc40700dd0a67867f3abc73f67e6fc2bcb5bea all runs: OK # git bisect good a0b0fd53e1e67639b303b15939b9c653dbe7a8c4 Bisecting: 3 revisions left to test after this (roughly 2 steps) [de4643a77356a77bce73f64275b125b4b71a69cf] locking/lockdep: Reuse lock chains that have been freed testing commit de4643a77356a77bce73f64275b125b4b71a69cf compiler: gcc version 8.4.1 20210217 (GCC) kernel signature: c78b3fd24044617d999df133b0264dc3855f1350d3c93712415fb48d5b22442d all runs: OK # git bisect good de4643a77356a77bce73f64275b125b4b71a69cf Bisecting: 1 revision left to test after this (roughly 1 step) [4bf508621855613ca2ac782f70c3171e0e8bb011] locking/lockdep: Verify whether lock objects are small enough to be used as class keys testing commit 4bf508621855613ca2ac782f70c3171e0e8bb011 compiler: gcc version 8.4.1 20210217 (GCC) kernel signature: 758ec8668a624b98e67b9439d49a06724e21ba660f2a72e5c098cbf865f7aeb3 all runs: OK # git bisect good 4bf508621855613ca2ac782f70c3171e0e8bb011 Bisecting: 0 revisions left to test after this (roughly 0 steps) [108c14858b9ea224686e476c8f5ec345a0df9e27] locking/lockdep: Add support for dynamic keys testing commit 108c14858b9ea224686e476c8f5ec345a0df9e27 compiler: gcc version 8.4.1 20210217 (GCC) kernel signature: 7cfb974f8aa109cf602d4b0314620231562b63c3c2ebba628364b17044568f3a all runs: OK # git bisect good 108c14858b9ea224686e476c8f5ec345a0df9e27 669de8bda87b92ab9a2fc663b3f5743c2ad1ae9f is the first bad commit commit 669de8bda87b92ab9a2fc663b3f5743c2ad1ae9f Author: Bart Van Assche Date: Thu Feb 14 15:00:54 2019 -0800 kernel/workqueue: Use dynamic lockdep keys for workqueues The following commit: 87915adc3f0a ("workqueue: re-add lockdep dependencies for flushing") improved deadlock checking in the workqueue implementation. Unfortunately that patch also introduced a few false positive lockdep complaints. This patch suppresses these false positives by allocating the workqueue mutex lockdep key dynamically. An example of a false positive lockdep complaint suppressed by this patch can be found below. The root cause of the lockdep complaint shown below is that the direct I/O code can call alloc_workqueue() from inside a work item created by another alloc_workqueue() call and that both workqueues share the same lockdep key. This patch avoids that that lockdep complaint is triggered by allocating the work queue lockdep keys dynamically. In other words, this patch guarantees that a unique lockdep key is associated with each work queue mutex. ====================================================== WARNING: possible circular locking dependency detected 4.19.0-dbg+ #1 Not tainted fio/4129 is trying to acquire lock: 00000000a01cfe1a ((wq_completion)"dio/%s"sb->s_id){+.+.}, at: flush_workqueue+0xd0/0x970 but task is already holding lock: 00000000a0acecf9 (&sb->s_type->i_mutex_key#14){+.+.}, at: ext4_file_write_iter+0x154/0x710 which lock already depends on the new lock. the existing dependency chain (in reverse order) is: -> #2 (&sb->s_type->i_mutex_key#14){+.+.}: down_write+0x3d/0x80 __generic_file_fsync+0x77/0xf0 ext4_sync_file+0x3c9/0x780 vfs_fsync_range+0x66/0x100 dio_complete+0x2f5/0x360 dio_aio_complete_work+0x1c/0x20 process_one_work+0x481/0x9f0 worker_thread+0x63/0x5a0 kthread+0x1cf/0x1f0 ret_from_fork+0x24/0x30 -> #1 ((work_completion)(&dio->complete_work)){+.+.}: process_one_work+0x447/0x9f0 worker_thread+0x63/0x5a0 kthread+0x1cf/0x1f0 ret_from_fork+0x24/0x30 -> #0 ((wq_completion)"dio/%s"sb->s_id){+.+.}: lock_acquire+0xc5/0x200 flush_workqueue+0xf3/0x970 drain_workqueue+0xec/0x220 destroy_workqueue+0x23/0x350 sb_init_dio_done_wq+0x6a/0x80 do_blockdev_direct_IO+0x1f33/0x4be0 __blockdev_direct_IO+0x79/0x86 ext4_direct_IO+0x5df/0xbb0 generic_file_direct_write+0x119/0x220 __generic_file_write_iter+0x131/0x2d0 ext4_file_write_iter+0x3fa/0x710 aio_write+0x235/0x330 io_submit_one+0x510/0xeb0 __x64_sys_io_submit+0x122/0x340 do_syscall_64+0x71/0x220 entry_SYSCALL_64_after_hwframe+0x49/0xbe other info that might help us debug this: Chain exists of: (wq_completion)"dio/%s"sb->s_id --> (work_completion)(&dio->complete_work) --> &sb->s_type->i_mutex_key#14 Possible unsafe locking scenario: CPU0 CPU1 ---- ---- lock(&sb->s_type->i_mutex_key#14); lock((work_completion)(&dio->complete_work)); lock(&sb->s_type->i_mutex_key#14); lock((wq_completion)"dio/%s"sb->s_id); *** DEADLOCK *** 1 lock held by fio/4129: #0: 00000000a0acecf9 (&sb->s_type->i_mutex_key#14){+.+.}, at: ext4_file_write_iter+0x154/0x710 stack backtrace: CPU: 3 PID: 4129 Comm: fio Not tainted 4.19.0-dbg+ #1 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.10.2-1 04/01/2014 Call Trace: dump_stack+0x86/0xc5 print_circular_bug.isra.32+0x20a/0x218 __lock_acquire+0x1c68/0x1cf0 lock_acquire+0xc5/0x200 flush_workqueue+0xf3/0x970 drain_workqueue+0xec/0x220 destroy_workqueue+0x23/0x350 sb_init_dio_done_wq+0x6a/0x80 do_blockdev_direct_IO+0x1f33/0x4be0 __blockdev_direct_IO+0x79/0x86 ext4_direct_IO+0x5df/0xbb0 generic_file_direct_write+0x119/0x220 __generic_file_write_iter+0x131/0x2d0 ext4_file_write_iter+0x3fa/0x710 aio_write+0x235/0x330 io_submit_one+0x510/0xeb0 __x64_sys_io_submit+0x122/0x340 do_syscall_64+0x71/0x220 entry_SYSCALL_64_after_hwframe+0x49/0xbe Signed-off-by: Bart Van Assche Signed-off-by: Peter Zijlstra (Intel) Cc: Andrew Morton Cc: Johannes Berg Cc: Linus Torvalds Cc: Paul E. McKenney Cc: Peter Zijlstra Cc: Tejun Heo Cc: Thomas Gleixner Cc: Waiman Long Cc: Will Deacon Link: https://lkml.kernel.org/r/20190214230058.196511-20-bvanassche@acm.org [ Reworked the changelog a bit. ] Signed-off-by: Ingo Molnar include/linux/workqueue.h | 28 ++++------------------ kernel/workqueue.c | 59 +++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 54 insertions(+), 33 deletions(-) culprit signature: eead89ba29d17f074383a04fe5b23de4c59d11232d37ff224165ac596b63d775 parent signature: 7cfb974f8aa109cf602d4b0314620231562b63c3c2ebba628364b17044568f3a revisions tested: 29, total time: 7h3m49.541880016s (build: 3h24m52.933193446s, test: 3h34m48.176326021s) first bad commit: 669de8bda87b92ab9a2fc663b3f5743c2ad1ae9f kernel/workqueue: Use dynamic lockdep keys for workqueues recipients (to): ["bvanassche@acm.org" "mingo@kernel.org" "peterz@infradead.org"] recipients (cc): [] crash: WARNING: locking bug in ext4_sync_fs EXT4-fs error (device loop0): ext4_add_entry:2078: inode #2: comm syz-executor.0: Directory hole found EXT4-fs error (device loop4): ext4_add_entry:2078: inode #2: comm syz-executor.4: Directory hole found EXT4-fs error (device loop4): ext4_add_entry:2078: inode #2: comm syz-executor.4: Directory hole found EXT4-fs error (device loop0): ext4_add_entry:2078: inode #2: comm syz-executor.0: Directory hole found WARNING: CPU: 1 PID: 11562 at kernel/locking/lockdep.c:747 __list_add include/linux/list.h:66 [inline] WARNING: CPU: 1 PID: 11562 at kernel/locking/lockdep.c:747 list_add_tail include/linux/list.h:93 [inline] WARNING: CPU: 1 PID: 11562 at kernel/locking/lockdep.c:747 list_move_tail include/linux/list.h:183 [inline] WARNING: CPU: 1 PID: 11562 at kernel/locking/lockdep.c:747 register_lock_class+0xdab/0x1770 kernel/locking/lockdep.c:1117 Kernel panic - not syncing: panic_on_warn set ... CPU: 1 PID: 11562 Comm: syz-executor.5 Not tainted 5.0.0-rc8-syzkaller #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x86/0xca lib/dump_stack.c:113 panic+0x1e7/0x3ac kernel/panic.c:214 __warn.cold.7+0x1b/0x37 kernel/panic.c:571 report_bug+0x1a1/0x200 lib/bug.c:186 fixup_bug arch/x86/kernel/traps.c:178 [inline] fixup_bug arch/x86/kernel/traps.c:173 [inline] do_error_trap+0x11b/0x1c0 arch/x86/kernel/traps.c:271 do_invalid_op+0x36/0x40 arch/x86/kernel/traps.c:290 invalid_op+0x14/0x20 arch/x86/entry/entry_64.S:973 RIP: 0010:look_up_lock_class kernel/locking/lockdep.c:747 [inline] RIP: 0010:register_lock_class+0xdab/0x1770 kernel/locking/lockdep.c:1060 Code: 07 00 00 4c 89 fa 4d 89 7d 08 48 b8 00 00 00 00 00 fc ff df 48 c1 ea 03 80 3c 02 00 0f 85 06 07 00 00 4d 89 2f e9 9f f8 ff ff <0f> 0b e9 ce f4 ff ff e8 99 d3 fc 01 48 c7 c7 98 43 71 89 48 89 fa RSP: 0018:ffff8880aef9f8a0 EFLAGS: 00010083 RAX: 0000000000000004 RBX: ffffffff8b3d89c0 RCX: 0000000000000003 RDX: 1ffffffff1706f3c RSI: ffff888237666540 RDI: ffff8880b34cd2c8 RBP: ffff8880aef9f970 R08: ffff8880b34cd2b0 R09: 0000000000000000 R10: ffff8880aef9f948 R11: ffff8880ba348b3b R12: 1ffff11015df3f1d R13: 0000000000000000 R14: 0000000000000000 R15: ffff8880a412a180 __lock_acquire+0x101/0x38a0 kernel/locking/lockdep.c:3550 lock_acquire+0x111/0x2d0 kernel/locking/lockdep.c:4178 flush_workqueue+0xf2/0x1360 kernel/workqueue.c:2680 ext4_sync_fs+0x11d/0x770 fs/ext4/super.c:5026 __sync_filesystem fs/sync.c:39 [inline] sync_filesystem+0xd4/0x200 fs/sync.c:64 generic_shutdown_super+0x69/0x330 fs/super.c:443 kill_block_super+0x96/0xe0 fs/super.c:1186 deactivate_locked_super+0x77/0xd0 fs/super.c:330 deactivate_super fs/super.c:361 [inline] deactivate_super+0x13f/0x160 fs/super.c:357 cleanup_mnt+0xa3/0x140 fs/namespace.c:1096 __cleanup_mnt+0xd/0x10 fs/namespace.c:1103 task_work_run+0x108/0x180 kernel/task_work.c:113 tracehook_notify_resume include/linux/tracehook.h:188 [inline] exit_to_usermode_loop+0x129/0x150 arch/x86/entry/common.c:166 prepare_exit_to_usermode arch/x86/entry/common.c:197 [inline] syscall_return_slowpath arch/x86/entry/common.c:268 [inline] do_syscall_64+0x3a4/0x440 arch/x86/entry/common.c:293 entry_SYSCALL_64_after_hwframe+0x49/0xbe RIP: 0033:0x467b0a Code: 48 c7 c2 bc ff ff ff f7 d8 64 89 02 b8 ff ff ff ff eb d2 e8 b8 04 00 00 0f 1f 84 00 00 00 00 00 49 89 ca b8 a5 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 c7 c1 bc ff ff ff f7 d8 64 89 01 48 RSP: 002b:00007fa3576d3fa8 EFLAGS: 00000246 ORIG_RAX: 00000000000000a5 RAX: ffffffffffffffec RBX: 0000000020000200 RCX: 0000000000467b0a RDX: 0000000020000000 RSI: 0000000020000100 RDI: 00007fa3576d4000 RBP: 00007fa3576d4040 R08: 00007fa3576d4040 R09: 0000000020000000 R10: 0000000000000000 R11: 0000000000000246 R12: 0000000020000000 R13: 0000000020000100 R14: 00007fa3576d4000 R15: 0000000020000300 Kernel Offset: disabled Rebooting in 86400 seconds..