possible deadlock in blkcg_deactivate_policy (2)

Status: upstream: reported C repro on 2023/12/04 11:30
Bug presence: origin:lts-only
[Documentation on labels]
Reported-by: syzbot+bf9ed1d36bc87fe3d0fb@syzkaller.appspotmail.com
First crash: 965d, last: 923d
▶ ▼ Bug presence (2)
Date Name Commit Repro Result
2023/12/04 linux-6.1.y (ToT) c6114c845984 C [report] possible deadlock in blkcg_deactivate_policy
2023/12/04 upstream (ToT) 33cc938e65a9 C Didn't crash
▶ ▼ Similar bugs (2)
Kernel Title Rank 🛈 Repro Cause bisect Fix bisect Count Last Reported Patched Status
linux-6.1 possible deadlock in blkcg_deactivate_policy 4 C done 168 1188d 1234d 3/3 fixed on 2023/05/30 15:47
upstream possible deadlock in blkcg_deactivate_policy block cgroups 4 C done error 93 1311d 1562d 0/29 auto-obsoleted due to no activity on 2023/05/12 00:27
▶ ▼ Fix bisection attempts (1)
Created Duration User Patch Repo Result
2023/12/20 00:36 2h00m fix candidate upstream OK (0) job log

Sample crash report:
 do_el0_svc+0x64/0x218 arch/arm64/kernel/syscall.c:206
 el0_svc+0x58/0x168 arch/arm64/kernel/entry-common.c:637
 el0t_64_sync_handler+0x84/0xf0 arch/arm64/kernel/entry-common.c:655
 el0t_64_sync+0x18c/0x190 arch/arm64/kernel/entry.S:585
======================================================
WARNING: possible circular locking dependency detected
6.1.65-syzkaller #0 Not tainted
------------------------------------------------------
syz-executor383/4225 is trying to acquire lock:
ffff0000dd8f70a8 ((&sq->pending_timer)){+.-.}-{0:0}, at: del_timer_sync+0x74/0x210 kernel/time/timer.c:1417

but task is already holding lock:
ffff800019a62cb0 (&blkcg->lock){....}-{2:2}, at: spin_lock include/linux/spinlock.h:350 [inline]
ffff800019a62cb0 (&blkcg->lock){....}-{2:2}, at: blkcg_deactivate_policy+0x1ac/0x4a8 block/blk-cgroup.c:1476

which lock already depends on the new lock.


the existing dependency chain (in reverse order) is:

-> #2 (&blkcg->lock){....}-{2:2}:
       __raw_spin_lock include/linux/spinlock_api_smp.h:133 [inline]
       _raw_spin_lock+0x54/0x6c kernel/locking/spinlock.c:154
       spin_lock include/linux/spinlock.h:350 [inline]
       blkg_create+0x9f4/0x1158 block/blk-cgroup.c:320
       blkcg_init_disk+0xd0/0x318 block/blk-cgroup.c:1242
       __alloc_disk_node+0x26c/0x484 block/genhd.c:1413
       __blk_alloc_disk+0x40/0xbc block/genhd.c:1452
       brd_alloc+0x2ac/0x5c8 drivers/block/brd.c:397
       brd_init+0x108/0x1c4 drivers/block/brd.c:484
       do_one_initcall+0x260/0xacc init/main.c:1296
       do_initcall_level+0x154/0x214 init/main.c:1369
       do_initcalls+0x58/0xac init/main.c:1385
       do_basic_setup+0x8c/0xa0 init/main.c:1404
       kernel_init_freeable+0x3a4/0x528 init/main.c:1624
       kernel_init+0x24/0x29c init/main.c:1512
       ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:864

-> #1 (&q->queue_lock){..-.}-{2:2}:
       __raw_spin_lock_irq include/linux/spinlock_api_smp.h:119 [inline]
       _raw_spin_lock_irq+0x70/0x9c kernel/locking/spinlock.c:170
       spin_lock_irq include/linux/spinlock.h:375 [inline]
       throtl_pending_timer_fn+0x104/0xdcc block/blk-throttle.c:1200
       call_timer_fn+0x1c0/0xa1c kernel/time/timer.c:1474
       expire_timers kernel/time/timer.c:1519 [inline]
       __run_timers+0x554/0x718 kernel/time/timer.c:1790
       run_timer_softirq+0x7c/0x114 kernel/time/timer.c:1803
       __do_softirq+0x314/0xe38 kernel/softirq.c:571
       ____do_softirq+0x14/0x20 arch/arm64/kernel/irq.c:79
       call_on_irq_stack+0x24/0x4c arch/arm64/kernel/entry.S:893
       do_softirq_own_stack+0x20/0x2c arch/arm64/kernel/irq.c:84
       invoke_softirq kernel/softirq.c:452 [inline]
       __irq_exit_rcu+0x264/0x4d4 kernel/softirq.c:650
       irq_exit_rcu+0x14/0x84 kernel/softirq.c:662
       __el1_irq arch/arm64/kernel/entry-common.c:472 [inline]
       el1_interrupt+0x38/0x68 arch/arm64/kernel/entry-common.c:486
       el1h_64_irq_handler+0x18/0x24 arch/arm64/kernel/entry-common.c:491
       el1h_64_irq+0x64/0x68 arch/arm64/kernel/entry.S:581
       arch_local_irq_restore arch/arm64/include/asm/irqflags.h:122 [inline]
       lock_acquire+0x2ac/0x7cc kernel/locking/lockdep.c:5665
       __might_fault+0xc4/0x124 mm/memory.c:5811
       _copy_from_user include/linux/uaccess.h:124 [inline]
       copy_from_user include/linux/uaccess.h:161 [inline]
       get_timespec64+0xa0/0x2d0 kernel/time/time.c:787
       __do_sys_clock_nanosleep kernel/time/posix-timers.c:1297 [inline]
       __se_sys_clock_nanosleep kernel/time/posix-timers.c:1285 [inline]
       __arm64_sys_clock_nanosleep+0x1e4/0x38c kernel/time/posix-timers.c:1285
       __invoke_syscall arch/arm64/kernel/syscall.c:38 [inline]
       invoke_syscall+0x98/0x2c0 arch/arm64/kernel/syscall.c:52
       el0_svc_common+0x138/0x258 arch/arm64/kernel/syscall.c:142
       do_el0_svc+0x64/0x218 arch/arm64/kernel/syscall.c:206
       el0_svc+0x58/0x168 arch/arm64/kernel/entry-common.c:637
       el0t_64_sync_handler+0x84/0xf0 arch/arm64/kernel/entry-common.c:655
       el0t_64_sync+0x18c/0x190 arch/arm64/kernel/entry.S:585

-> #0 ((&sq->pending_timer)){+.-.}-{0:0}:
       check_prev_add kernel/locking/lockdep.c:3090 [inline]
       check_prevs_add kernel/locking/lockdep.c:3209 [inline]
       validate_chain kernel/locking/lockdep.c:3825 [inline]
       __lock_acquire+0x3338/0x7680 kernel/locking/lockdep.c:5049
       lock_acquire+0x26c/0x7cc kernel/locking/lockdep.c:5662
       del_timer_sync+0x9c/0x210 kernel/time/timer.c:1417
       throtl_pd_free+0x20/0x48 block/blk-throttle.c:493
       blkcg_deactivate_policy+0x2cc/0x4a8 block/blk-cgroup.c:1480
       blk_throtl_exit+0x9c/0x13c block/blk-throttle.c:2413
       blkcg_init_disk+0x2a4/0x318 block/blk-cgroup.c:1266
       __alloc_disk_node+0x26c/0x484 block/genhd.c:1413
       __blk_mq_alloc_disk+0x114/0x1e4 block/blk-mq.c:4100
       loop_add+0x2c4/0x8c8 drivers/block/loop.c:2001
       loop_control_ioctl+0xec/0x650 drivers/block/loop.c:2173
       vfs_ioctl fs/ioctl.c:51 [inline]
       __do_sys_ioctl fs/ioctl.c:870 [inline]
       __se_sys_ioctl fs/ioctl.c:856 [inline]
       __arm64_sys_ioctl+0x14c/0x1c8 fs/ioctl.c:856
       __invoke_syscall arch/arm64/kernel/syscall.c:38 [inline]
       invoke_syscall+0x98/0x2c0 arch/arm64/kernel/syscall.c:52
       el0_svc_common+0x138/0x258 arch/arm64/kernel/syscall.c:142
       do_el0_svc+0x64/0x218 arch/arm64/kernel/syscall.c:206
       el0_svc+0x58/0x168 arch/arm64/kernel/entry-common.c:637
       el0t_64_sync_handler+0x84/0xf0 arch/arm64/kernel/entry-common.c:655
       el0t_64_sync+0x18c/0x190 arch/arm64/kernel/entry.S:585

other info that might help us debug this:

Chain exists of:
  (&sq->pending_timer) --> &q->queue_lock --> &blkcg->lock

 Possible unsafe locking scenario:

       CPU0                    CPU1
       ----                    ----
  lock(&blkcg->lock);
                               lock(&q->queue_lock);
                               lock(&blkcg->lock);
  lock((&sq->pending_timer));

 *** DEADLOCK ***

2 locks held by syz-executor383/4225:
 #0: ffff0000d54880d0 (&q->queue_lock){..-.}-{2:2}, at: spin_lock_irq include/linux/spinlock.h:375 [inline]
 #0: ffff0000d54880d0 (&q->queue_lock){..-.}-{2:2}, at: blkcg_deactivate_policy+0xf4/0x4a8 block/blk-cgroup.c:1469
 #1: ffff800019a62cb0 (&blkcg->lock){....}-{2:2}, at: spin_lock include/linux/spinlock.h:350 [inline]
 #1: ffff800019a62cb0 (&blkcg->lock){....}-{2:2}, at: blkcg_deactivate_policy+0x1ac/0x4a8 block/blk-cgroup.c:1476

stack backtrace:
CPU: 1 PID: 4225 Comm: syz-executor383 Not tainted 6.1.65-syzkaller #0
Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 10/09/2023
Call trace:
 dump_backtrace+0x1c8/0x1f4 arch/arm64/kernel/stacktrace.c:158
 show_stack+0x2c/0x3c arch/arm64/kernel/stacktrace.c:165
 __dump_stack lib/dump_stack.c:88 [inline]
 dump_stack_lvl+0x108/0x170 lib/dump_stack.c:106
 dump_stack+0x1c/0x58 lib/dump_stack.c:113
 print_circular_bug+0x150/0x1b8 kernel/locking/lockdep.c:2048
 check_noncircular+0x2cc/0x378 kernel/locking/lockdep.c:2170
 check_prev_add kernel/locking/lockdep.c:3090 [inline]
 check_prevs_add kernel/locking/lockdep.c:3209 [inline]
 validate_chain kernel/locking/lockdep.c:3825 [inline]
 __lock_acquire+0x3338/0x7680 kernel/locking/lockdep.c:5049
 lock_acquire+0x26c/0x7cc kernel/locking/lockdep.c:5662
 del_timer_sync+0x9c/0x210 kernel/time/timer.c:1417
 throtl_pd_free+0x20/0x48 block/blk-throttle.c:493
 blkcg_deactivate_policy+0x2cc/0x4a8 block/blk-cgroup.c:1480
 blk_throtl_exit+0x9c/0x13c block/blk-throttle.c:2413
 blkcg_init_disk+0x2a4/0x318 block/blk-cgroup.c:1266
 __alloc_disk_node+0x26c/0x484 block/genhd.c:1413
 __blk_mq_alloc_disk+0x114/0x1e4 block/blk-mq.c:4100
 loop_add+0x2c4/0x8c8 drivers/block/loop.c:2001
 loop_control_ioctl+0xec/0x650 drivers/block/loop.c:2173
 vfs_ioctl fs/ioctl.c:51 [inline]
 __do_sys_ioctl fs/ioctl.c:870 [inline]
 __se_sys_ioctl fs/ioctl.c:856 [inline]
 __arm64_sys_ioctl+0x14c/0x1c8 fs/ioctl.c:856
 __invoke_syscall arch/arm64/kernel/syscall.c:38 [inline]
 invoke_syscall+0x98/0x2c0 arch/arm64/kernel/syscall.c:52
 el0_svc_common+0x138/0x258 arch/arm64/kernel/syscall.c:142
 do_el0_svc+0x64/0x218 arch/arm64/kernel/syscall.c:206
 el0_svc+0x58/0x168 arch/arm64/kernel/entry-common.c:637
 el0t_64_sync_handler+0x84/0xf0 arch/arm64/kernel/entry-common.c:655
 el0t_64_sync+0x18c/0x190 arch/arm64/kernel/entry.S:585

Crashes (3):
Time Kernel Commit Syzkaller Config Log Report Syz repro C repro VM info Assets (help?) Manager Title
2023/12/04 11:30 linux-6.1.y c6114c845984 f819d6f7 .config console log report syz C [disk image] [vmlinux] [kernel image] ci2-linux-6-1-kasan-arm64 possible deadlock in blkcg_deactivate_policy
2024/01/15 06:15 linux-6.1.y 7c58bfa711cb 551587c1 .config console log report info [disk image] [vmlinux] [kernel image] ci2-linux-6-1-kasan-arm64 possible deadlock in blkcg_deactivate_policy
2023/12/05 05:45 linux-6.1.y c6114c845984 f819d6f7 .config console log report info [disk image] [vmlinux] [kernel image] ci2-linux-6-1-kasan-arm64 possible deadlock in blkcg_deactivate_policy
* Struck through repros no longer work on HEAD.