BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=0 stuck for 152s! BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=-20 stuck for 151s! Showing busy workqueues and worker pools: workqueue events: flags=0x0 pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=2/256 pending: bpf_prog_free_deferred, free_obj_work ====================================================== WARNING: possible circular locking dependency detected 4.14.71+ #8 Not tainted ------------------------------------------------------ swapper/0/0 is trying to acquire lock: (&port_lock_key){-.-.}, at: [] serial8250_console_write+0x6af/0x800 drivers/tty/serial/8250/8250_port.c:3232 but task is already holding lock: (&(&pool->lock)->rlock){-.-.}, at: [] show_workqueue_state.cold.23+0x278/0x1053 kernel/workqueue.c:4495 which lock already depends on the new lock. the existing dependency chain (in reverse order) is: -> #3 (&(&pool->lock)->rlock){-.-.}: __raw_spin_lock include/linux/spinlock_api_smp.h:142 [inline] _raw_spin_lock+0x2a/0x40 kernel/locking/spinlock.c:152 spin_lock include/linux/spinlock.h:317 [inline] __queue_work+0x184/0xe40 kernel/workqueue.c:1419 queue_work_on+0x8f/0xa0 kernel/workqueue.c:1488 queue_work include/linux/workqueue.h:491 [inline] schedule_work include/linux/workqueue.h:549 [inline] put_pwq+0x131/0x180 kernel/workqueue.c:1092 put_pwq_unlocked.part.5+0x30/0x70 kernel/workqueue.c:1109 put_pwq_unlocked kernel/workqueue.c:1103 [inline] destroy_workqueue+0x4dc/0x600 kernel/workqueue.c:4150 ext4_fill_super+0x7342/0xb5e0 fs/ext4/super.c:4425 mount_bdev+0x2ab/0x370 fs/super.c:1149 mount_fs+0x28f/0x31a fs/super.c:1255 vfs_kern_mount.part.9+0xce/0x4a0 fs/namespace.c:1056 vfs_kern_mount fs/namespace.c:1038 [inline] do_new_mount fs/namespace.c:2561 [inline] do_mount+0x404/0x26e0 fs/namespace.c:2891 SYSC_mount fs/namespace.c:3107 [inline] SyS_mount+0xe5/0x100 fs/namespace.c:3084 do_syscall_64+0x19b/0x4b0 arch/x86/entry/common.c:289 entry_SYSCALL_64_after_hwframe+0x42/0xb7 -> #2 (&pool->lock/1){..-.}: __raw_spin_lock include/linux/spinlock_api_smp.h:142 [inline] _raw_spin_lock+0x2a/0x40 kernel/locking/spinlock.c:152 spin_lock include/linux/spinlock.h:317 [inline] __queue_work+0x184/0xe40 kernel/workqueue.c:1419 queue_work_on+0x8f/0xa0 kernel/workqueue.c:1488 pty_write+0x172/0x1c0 drivers/tty/pty.c:124 n_tty_write+0x969/0xda0 drivers/tty/n_tty.c:2342 do_tty_write drivers/tty/tty_io.c:957 [inline] tty_write+0x397/0x810 drivers/tty/tty_io.c:1041 __vfs_write+0xf4/0x5c0 fs/read_write.c:482 vfs_write+0x17f/0x4d0 fs/read_write.c:546 SYSC_write fs/read_write.c:593 [inline] SyS_write+0xc2/0x1a0 fs/read_write.c:585 do_syscall_64+0x19b/0x4b0 arch/x86/entry/common.c:289 entry_SYSCALL_64_after_hwframe+0x42/0xb7 -> #1 (&(&port->lock)->rlock){-.-.}: __raw_spin_lock_irqsave include/linux/spinlock_api_smp.h:110 [inline] _raw_spin_lock_irqsave+0x41/0x50 kernel/locking/spinlock.c:160 tty_port_tty_get+0x1b/0x80 drivers/tty/tty_port.c:287 tty_port_default_wakeup+0x11/0x40 drivers/tty/tty_port.c:46 serial8250_tx_chars+0x417/0xa70 drivers/tty/serial/8250/8250_port.c:1810 serial8250_handle_irq.part.15+0x1b1/0x1f0 drivers/tty/serial/8250/8250_port.c:1883 serial8250_handle_irq drivers/tty/serial/8250/8250_port.c:1869 [inline] serial8250_default_handle_irq+0x9b/0x110 drivers/tty/serial/8250/8250_port.c:1899 serial8250_interrupt+0xea/0x1a0 drivers/tty/serial/8250/8250_core.c:129 __handle_irq_event_percpu+0xf8/0x770 kernel/irq/handle.c:147 handle_irq_event_percpu+0x76/0x150 kernel/irq/handle.c:187 handle_irq_event+0xa1/0x12d kernel/irq/handle.c:204 handle_edge_irq+0x1f0/0x7f0 kernel/irq/chip.c:770 generic_handle_irq_desc include/linux/irqdesc.h:158 [inline] handle_irq+0x242/0x328 arch/x86/kernel/irq_64.c:78 do_IRQ+0x7d/0x1b0 arch/x86/kernel/irq.c:230 ret_from_intr+0x0/0x22 arch_local_irq_restore arch/x86/include/asm/paravirt.h:778 [inline] __raw_spin_unlock_irqrestore include/linux/spinlock_api_smp.h:160 [inline] _raw_spin_unlock_irqrestore+0x59/0x70 kernel/locking/spinlock.c:192 spin_unlock_irqrestore include/linux/spinlock.h:372 [inline] uart_write+0x2a7/0x480 drivers/tty/serial/serial_core.c:611 process_output_block drivers/tty/n_tty.c:582 [inline] n_tty_write+0x4f5/0xda0 drivers/tty/n_tty.c:2319 do_tty_write drivers/tty/tty_io.c:957 [inline] tty_write+0x397/0x810 drivers/tty/tty_io.c:1041 redirected_tty_write+0x9a/0xa0 drivers/tty/tty_io.c:1062 __vfs_write+0xf4/0x5c0 fs/read_write.c:482 vfs_write+0x17f/0x4d0 fs/read_write.c:546 SYSC_write fs/read_write.c:593 [inline] SyS_write+0xc2/0x1a0 fs/read_write.c:585 do_syscall_64+0x19b/0x4b0 arch/x86/entry/common.c:289 entry_SYSCALL_64_after_hwframe+0x42/0xb7 -> #0 (&port_lock_key){-.-.}: lock_acquire+0x10f/0x380 kernel/locking/lockdep.c:3991 __raw_spin_lock_irqsave include/linux/spinlock_api_smp.h:110 [inline] _raw_spin_lock_irqsave+0x41/0x50 kernel/locking/spinlock.c:160 serial8250_console_write+0x6af/0x800 drivers/tty/serial/8250/8250_port.c:3232 call_console_drivers kernel/printk/printk.c:1574 [inline] console_unlock+0x5b9/0xb50 kernel/printk/printk.c:2242 vprintk_emit+0x112/0x150 kernel/printk/printk.c:1766 vprintk_func+0x58/0x159 kernel/printk/printk_safe.c:401 printk+0xa7/0xcf kernel/printk/printk.c:1838 show_pwq kernel/workqueue.c:4402 [inline] show_workqueue_state.cold.23+0x3d5/0x1053 kernel/workqueue.c:4497 wq_watchdog_timer_fn+0x436/0x4d0 kernel/workqueue.c:5481 call_timer_fn+0x163/0x6a0 kernel/time/timer.c:1279 expire_timers+0x1f3/0x4a0 kernel/time/timer.c:1318 __run_timers kernel/time/timer.c:1634 [inline] run_timer_softirq+0x3a5/0x560 kernel/time/timer.c:1649 __do_softirq+0x215/0x997 kernel/softirq.c:288 invoke_softirq kernel/softirq.c:368 [inline] irq_exit+0x10f/0x150 kernel/softirq.c:409 exiting_irq arch/x86/include/asm/apic.h:648 [inline] smp_apic_timer_interrupt+0x188/0x5f0 arch/x86/kernel/apic/apic.c:1064 apic_timer_interrupt+0x84/0x90 arch/x86/entry/entry_64.S:787 native_safe_halt+0x2/0x10 arch/x86/include/asm/irqflags.h:57 arch_safe_halt arch/x86/include/asm/paravirt.h:94 [inline] default_idle+0x50/0x380 arch/x86/kernel/process.c:501 cpuidle_idle_call kernel/sched/idle.c:159 [inline] do_idle+0x2da/0x3b0 kernel/sched/idle.c:268 cpu_startup_entry+0xc9/0xe0 kernel/sched/idle.c:374 start_kernel+0x6fb/0x739 init/main.c:710 secondary_startup_64+0xa5/0xb0 arch/x86/kernel/head_64.S:239 other info that might help us debug this: Chain exists of: &port_lock_key --> &pool->lock/1 --> &(&pool->lock)->rlock Possible unsafe locking scenario: CPU0 CPU1 ---- ---- lock(&(&pool->lock)->rlock); lock(&pool->lock/1); lock(&(&pool->lock)->rlock); lock(&port_lock_key); *** DEADLOCK *** 4 locks held by swapper/0/0: #0: (kernel/workqueue.c:5419){+.-.}, at: [] lockdep_copy_map include/linux/lockdep.h:174 [inline] #0: (kernel/workqueue.c:5419){+.-.}, at: [] call_timer_fn+0xd7/0x6a0 kernel/time/timer.c:1269 #1: (rcu_read_lock_sched){....}, at: [] show_workqueue_state+0x0/0x100 kernel/workqueue.c:4367 #2: (&(&pool->lock)->rlock){-.-.}, at: [] show_workqueue_state.cold.23+0x278/0x1053 kernel/workqueue.c:4495 #3: (console_lock){+.+.}, at: [] vprintk_emit+0x104/0x150 kernel/printk/printk.c:1765 stack backtrace: CPU: 0 PID: 0 Comm: swapper/0 Not tainted 4.14.71+ #8 Call Trace: __dump_stack lib/dump_stack.c:17 [inline] dump_stack+0xb9/0x11b lib/dump_stack.c:53 print_circular_bug.isra.18.cold.43+0x2d3/0x40c kernel/locking/lockdep.c:1258 check_prev_add kernel/locking/lockdep.c:1901 [inline] check_prevs_add kernel/locking/lockdep.c:2018 [inline] validate_chain kernel/locking/lockdep.c:2460 [inline] __lock_acquire+0x2ff9/0x4320 kernel/locking/lockdep.c:3487 lock_acquire+0x10f/0x380 kernel/locking/lockdep.c:3991 __raw_spin_lock_irqsave include/linux/spinlock_api_smp.h:110 [inline] _raw_spin_lock_irqsave+0x41/0x50 kernel/locking/spinlock.c:160 serial8250_console_write+0x6af/0x800 drivers/tty/serial/8250/8250_port.c:3232 call_console_drivers kernel/printk/printk.c:1574 [inline] console_unlock+0x5b9/0xb50 kernel/printk/printk.c:2242 vprintk_emit+0x112/0x150 kernel/printk/printk.c:1766 vprintk_func+0x58/0x159 kernel/printk/printk_safe.c:401 printk+0xa7/0xcf kernel/printk/printk.c:1838 show_pwq kernel/workqueue.c:4402 [inline] show_workqueue_state.cold.23+0x3d5/0x1053 kernel/workqueue.c:4497 wq_watchdog_timer_fn+0x436/0x4d0 kernel/workqueue.c:5481 call_timer_fn+0x163/0x6a0 kernel/time/timer.c:1279 expire_timers+0x1f3/0x4a0 kernel/time/timer.c:1318 __run_timers kernel/time/timer.c:1634 [inline] run_timer_softirq+0x3a5/0x560 kernel/time/timer.c:1649 __do_softirq+0x215/0x997 kernel/softirq.c:288 invoke_softirq kernel/softirq.c:368 [inline] irq_exit+0x10f/0x150 kernel/softirq.c:409 exiting_irq arch/x86/include/asm/apic.h:648 [inline] smp_apic_timer_interrupt+0x188/0x5f0 arch/x86/kernel/apic/apic.c:1064 apic_timer_interrupt+0x84/0x90 arch/x86/entry/entry_64.S:787 RIP: 0010:native_safe_halt+0x2/0x10 arch/x86/include/asm/irqflags.h:57 RSP: 0018:ffffffffae807d28 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff10 RAX: 0000000000000007 RBX: ffffffffaec28668 RCX: 0000000000000000 RDX: 0000000000000000 RSI: 0000000000000001 RDI: ffffffffae82ca2c RBP: ffffffffaec32aa8 R08: 0000000000000001 R09: 0000000000000000 R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000 R13: 1ffffffff5d00fae R14: 0000000000000000 R15: ffffffffaec32b64 arch_safe_halt arch/x86/include/asm/paravirt.h:94 [inline] default_idle+0x50/0x380 arch/x86/kernel/process.c:501 cpuidle_idle_call kernel/sched/idle.c:159 [inline] do_idle+0x2da/0x3b0 kernel/sched/idle.c:268 cpu_startup_entry+0xc9/0xe0 kernel/sched/idle.c:374 start_kernel+0x6fb/0x739 init/main.c:710 secondary_startup_64+0xa5/0xb0 arch/x86/kernel/head_64.S:239 workqueue events_highpri: flags=0x10 pwq 3: cpus=1 node=0 flags=0x0 nice=-20 active=1/256 pending: flush_backlog BAR(370) workqueue events_long: flags=0x0 pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 pending: gc_worker workqueue mm_percpu_wq: flags=0x8 pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 pending: vmstat_update workqueue netns: flags=0xe000a pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/1 in-flight: 370:cleanup_net workqueue kblockd: flags=0x18 pwq 3: cpus=1 node=0 flags=0x0 nice=-20 active=1/256 pending: blk_mq_requeue_work workqueue ipv6_addrconf: flags=0x40008 pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/1 in-flight: 19720:addrconf_verify_work pool 0: cpus=0 node=0 flags=0x0 nice=0 hung=0s workers=26 idle: 19710 19723 19717 19727 19721 7734 3 4575 72 19712 19716 19708 7640 22 19715 19709 19713 19714 19718 19719 19724 19725 19711 19722 19726 pool 4: cpus=0-1 flags=0x4 nice=0 hung=46s workers=10 idle: 2877 64 2093 2172 19292 5 2176 19300 19295 INFO: task jbd2/sda1-8:74 blocked for more than 140 seconds. Not tainted 4.14.71+ #8 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. jbd2/sda1-8 D27384 74 2 0x80000000 Call Trace: schedule+0x7f/0x1b0 kernel/sched/core.c:3490 io_schedule+0x1c/0x70 kernel/sched/core.c:5097 bit_wait_io+0x9/0x90 kernel/sched/wait_bit.c:249 __wait_on_bit+0xad/0x130 kernel/sched/wait_bit.c:51 out_of_line_wait_on_bit+0xd1/0x100 kernel/sched/wait_bit.c:64 wait_on_bit_io include/linux/wait_bit.h:106 [inline] __wait_on_buffer+0x58/0x70 fs/buffer.c:125 wait_on_buffer include/linux/buffer_head.h:354 [inline] journal_wait_on_commit_record fs/jbd2/commit.c:177 [inline] jbd2_journal_commit_transaction+0x484a/0x648c fs/jbd2/commit.c:868 kjournald2+0x244/0x820 fs/jbd2/journal.c:233 kthread+0x348/0x420 kernel/kthread.c:232 ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:402 INFO: task syz-executor2:19621 blocked for more than 140 seconds. Not tainted 4.14.71+ #8 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. syz-executor2 D27848 19621 1842 0x00000004 Call Trace: schedule+0x7f/0x1b0 kernel/sched/core.c:3490 jbd2_log_wait_commit+0x2a5/0x3f0 fs/jbd2/journal.c:730 jbd2_journal_stop+0x8df/0x1280 fs/jbd2/transaction.c:1782 __ext4_journal_stop+0xcd/0x1e0 fs/ext4/ext4_jbd2.c:103 ext4_change_inode_journal_flag+0x298/0x390 fs/ext4/inode.c:6059 ext4_ioctl_setflags+0x4ac/0x540 fs/ext4/ioctl.c:304 ext4_ioctl+0x1079/0x35e0 fs/ext4/ioctl.c:636 vfs_ioctl fs/ioctl.c:46 [inline] file_ioctl fs/ioctl.c:500 [inline] do_vfs_ioctl+0x1a0/0x1030 fs/ioctl.c:684 SYSC_ioctl fs/ioctl.c:701 [inline] SyS_ioctl+0x7e/0xb0 fs/ioctl.c:692 do_syscall_64+0x19b/0x4b0 arch/x86/entry/common.c:289 entry_SYSCALL_64_after_hwframe+0x42/0xb7 RIP: 0033:0x457679 RSP: 002b:00007ff989a10c78 EFLAGS: 00000246 ORIG_RAX: 0000000000000010 RAX: ffffffffffffffda RBX: 00007ff989a116d4 RCX: 0000000000457679 RDX: 0000000020000040 RSI: 0000000040086602 RDI: 0000000000000003 RBP: 000000000072bf00 R08: 0000000000000000 R09: 0000000000000000 R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff R13: 00000000004cf418 R14: 00000000004bf740 R15: 0000000000000000 INFO: task syz-executor2:19626 blocked for more than 140 seconds. Not tainted 4.14.71+ #8 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. syz-executor2 D29592 19626 1842 0x00000004 Call Trace: schedule+0x7f/0x1b0 kernel/sched/core.c:3490 __rwsem_down_write_failed_common kernel/locking/rwsem-xadd.c:565 [inline] rwsem_down_write_failed+0x390/0x730 kernel/locking/rwsem-xadd.c:594 call_rwsem_down_write_failed+0x13/0x20 arch/x86/lib/rwsem.S:105 __down_write arch/x86/include/asm/rwsem.h:126 [inline] down_write+0x4f/0x90 kernel/locking/rwsem.c:56 inode_lock include/linux/fs.h:713 [inline] do_truncate2+0xfd/0x1c0 fs/open.c:61 handle_truncate fs/namei.c:2998 [inline] do_last fs/namei.c:3417 [inline] path_openat+0xaea/0x23a0 fs/namei.c:3550 do_filp_open+0x197/0x270 fs/namei.c:3584 do_sys_open+0x2ef/0x580 fs/open.c:1071 do_syscall_64+0x19b/0x4b0 arch/x86/entry/common.c:289 entry_SYSCALL_64_after_hwframe+0x42/0xb7 RIP: 0033:0x457679 RSP: 002b:00007ff9899efc78 EFLAGS: 00000246 ORIG_RAX: 0000000000000055 RAX: ffffffffffffffda RBX: 00007ff9899f06d4 RCX: 0000000000457679 RDX: 0000000000000000 RSI: 0000000000000000 RDI: 00000000200000c0 RBP: 000000000072bfa0 R08: 0000000000000000 R09: 0000000000000000 R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff R13: 00000000004cc688 R14: 00000000004bdcdc R15: 0000000000000001 INFO: task syz-executor2:19656 blocked for more than 140 seconds. Not tainted 4.14.71+ #8 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. syz-executor2 D29720 19656 1842 0x00000004 Call Trace: schedule+0x7f/0x1b0 kernel/sched/core.c:3490 __rwsem_down_write_failed_common kernel/locking/rwsem-xadd.c:565 [inline] rwsem_down_write_failed+0x390/0x730 kernel/locking/rwsem-xadd.c:594 call_rwsem_down_write_failed+0x13/0x20 arch/x86/lib/rwsem.S:105 __down_write arch/x86/include/asm/rwsem.h:126 [inline] down_write+0x4f/0x90 kernel/locking/rwsem.c:56 inode_lock include/linux/fs.h:713 [inline] ext4_ioctl+0x106d/0x35e0 fs/ext4/ioctl.c:635 vfs_ioctl fs/ioctl.c:46 [inline] file_ioctl fs/ioctl.c:500 [inline] do_vfs_ioctl+0x1a0/0x1030 fs/ioctl.c:684 SYSC_ioctl fs/ioctl.c:701 [inline] SyS_ioctl+0x7e/0xb0 fs/ioctl.c:692 do_syscall_64+0x19b/0x4b0 arch/x86/entry/common.c:289 entry_SYSCALL_64_after_hwframe+0x42/0xb7 RIP: 0033:0x457679 RSP: 002b:00007ff9899adc78 EFLAGS: 00000246 ORIG_RAX: 0000000000000010 RAX: ffffffffffffffda RBX: 00007ff9899ae6d4 RCX: 0000000000457679 RDX: 0000000020000040 RSI: 0000000040086602 RDI: 0000000000000003 RBP: 000000000072c0e0 R08: 0000000000000000 R09: 0000000000000000 R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff R13: 00000000004cf418 R14: 00000000004bf740 R15: 0000000000000003 INFO: lockdep is turned off. NMI backtrace for cpu 0 CPU: 0 PID: 23 Comm: khungtaskd Not tainted 4.14.71+ #8 Call Trace: __dump_stack lib/dump_stack.c:17 [inline] dump_stack+0xb9/0x11b lib/dump_stack.c:53 nmi_cpu_backtrace.cold.0+0x47/0x85 lib/nmi_backtrace.c:101 nmi_trigger_cpumask_backtrace+0x121/0x146 lib/nmi_backtrace.c:62 trigger_all_cpu_backtrace include/linux/nmi.h:138 [inline] check_hung_uninterruptible_tasks kernel/hung_task.c:196 [inline] watchdog+0x574/0xa70 kernel/hung_task.c:252 kthread+0x348/0x420 kernel/kthread.c:232 ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:402 Sending NMI from CPU 0 to CPUs 1: NMI backtrace for cpu 1 CPU: 1 PID: 19602 Comm: syz-executor0 Not tainted 4.14.71+ #8 task: ffff88018653de00 task.stack: ffff8801c4378000 RIP: 0010:__sanitizer_cov_trace_pc+0x0/0x60 RSP: 0018:ffff8801c437f9a8 EFLAGS: 00000246 RAX: ffffffffac7a23aa RBX: dffffc0000000000 RCX: 0000000000040000 RDX: 1ffffffff5c52bdb RSI: ffffc90000521000 RDI: ffffffffae295ed8 RBP: ffff8801c437fae0 R08: 0000000000000000 R09: 0000000000000000 R10: 0000000000000000 R11: 0000000000000000 R12: ffffffffae295960 R13: 1ffff1003886ff3d R14: ffffed003886ff62 R15: ffffc90000002038 FS: 00007fdba2095700(0000) GS:ffff8801dbb00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 0000000020000100 CR3: 000000018ac72001 CR4: 00000000001606a0 Call Trace: ___bpf_prog_run+0x2f2f/0x5c70 kernel/bpf/core.c:908 Code: ff ff 48 89 df e8 b1 f8 21 00 e9 9f fe ff ff 4c 89 e7 e8 a4 f8 21 00 e9 2c fe ff ff 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 <65> 48 8b 04 25 c0 de 01 00 48 85 c0 74 1a 65 8b 15 2b 25 91 53