INFO: task kworker/1:1:24 blocked for more than 120 seconds. Not tainted 4.17.0-rc1+ #8 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. kworker/1:1 D19224 24 2 0x80000000 Workqueue: events cgwb_release_workfn Call Trace: context_switch kernel/sched/core.c:2848 [inline] __schedule+0x801/0x1e30 kernel/sched/core.c:3490 schedule+0xef/0x430 kernel/sched/core.c:3549 bit_wait+0x18/0x90 kernel/sched/wait_bit.c:197 __wait_on_bit+0xb3/0x130 kernel/sched/wait_bit.c:48 out_of_line_wait_on_bit+0x204/0x3a0 kernel/sched/wait_bit.c:63 wait_on_bit include/linux/wait_bit.h:76 [inline] wb_shutdown+0x370/0x470 mm/backing-dev.c:366 cgwb_release_workfn+0x87/0x750 mm/backing-dev.c:509 process_one_work+0xc1e/0x1b50 kernel/workqueue.c:2145 worker_thread+0x1cc/0x1440 kernel/workqueue.c:2279 kthread+0x345/0x410 kernel/kthread.c:238 ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:412 Showing all locks held in the system: 2 locks held by kworker/1:1/24: #0: 000000002dc57831 ((wq_completion)"events"){+.+.}, at: __write_once_size include/linux/compiler.h:215 [inline] #0: 000000002dc57831 ((wq_completion)"events"){+.+.}, at: arch_atomic64_set arch/x86/include/asm/atomic64_64.h:34 [inline] #0: 000000002dc57831 ((wq_completion)"events"){+.+.}, at: atomic64_set include/asm-generic/atomic-instrumented.h:40 [inline] #0: 000000002dc57831 ((wq_completion)"events"){+.+.}, at: atomic_long_set include/asm-generic/atomic-long.h:57 [inline] #0: 000000002dc57831 ((wq_completion)"events"){+.+.}, at: set_work_data kernel/workqueue.c:617 [inline] #0: 000000002dc57831 ((wq_completion)"events"){+.+.}, at: set_work_pool_and_clear_pending kernel/workqueue.c:644 [inline] #0: 000000002dc57831 ((wq_completion)"events"){+.+.}, at: process_one_work+0xaef/0x1b50 kernel/workqueue.c:2116 #1: 0000000084bf34a2 ((work_completion)(&wb->release_work)){+.+.}, at: process_one_work+0xb46/0x1b50 kernel/workqueue.c:2120 2 locks held by khungtaskd/891: #0: 00000000e2887a2d (rcu_read_lock){....}, at: check_hung_uninterruptible_tasks kernel/hung_task.c:175 [inline] #0: 00000000e2887a2d (rcu_read_lock){....}, at: watchdog+0x1ff/0xf60 kernel/hung_task.c:249 #1: 00000000886d09e6 (tasklist_lock){.+.+}, at: debug_show_all_locks+0xde/0x34a kernel/locking/lockdep.c:4470 2 locks held by kworker/0:2/2106: #0: 000000002dc57831 ((wq_completion)"events"){+.+.}, at: __write_once_size include/linux/compiler.h:215 [inline] #0: 000000002dc57831 ((wq_completion)"events"){+.+.}, at: arch_atomic64_set arch/x86/include/asm/atomic64_64.h:34 [inline] #0: 000000002dc57831 ((wq_completion)"events"){+.+.}, at: atomic64_set include/asm-generic/atomic-instrumented.h:40 [inline] #0: 000000002dc57831 ((wq_completion)"events"){+.+.}, at: atomic_long_set include/asm-generic/atomic-long.h:57 [inline] #0: 000000002dc57831 ((wq_completion)"events"){+.+.}, at: set_work_data kernel/workqueue.c:617 [inline] #0: 000000002dc57831 ((wq_completion)"events"){+.+.}, at: set_work_pool_and_clear_pending kernel/workqueue.c:644 [inline] #0: 000000002dc57831 ((wq_completion)"events"){+.+.}, at: process_one_work+0xaef/0x1b50 kernel/workqueue.c:2116 #1: 00000000fec6d0ca ((work_completion)(&wb->release_work)){+.+.}, at: process_one_work+0xb46/0x1b50 kernel/workqueue.c:2120 2 locks held by getty/4425: #0: 000000006a3302db (&tty->ldisc_sem){++++}, at: ldsem_down_read+0x37/0x40 drivers/tty/tty_ldsem.c:365 #1: 0000000091a79cc3 (&ldata->atomic_read_lock){+.+.}, at: n_tty_read+0x321/0x1cc0 drivers/tty/n_tty.c:2131 2 locks held by getty/4426: #0: 00000000e81b5f47 (&tty->ldisc_sem){++++}, at: ldsem_down_read+0x37/0x40 drivers/tty/tty_ldsem.c:365 #1: 00000000782096e0 (&ldata->atomic_read_lock){+.+.}, at: n_tty_read+0x321/0x1cc0 drivers/tty/n_tty.c:2131 2 locks held by getty/4427: #0: 00000000fe4499b7 (&tty->ldisc_sem){++++}, at: ldsem_down_read+0x37/0x40 drivers/tty/tty_ldsem.c:365 #1: 00000000ffd03992 (&ldata->atomic_read_lock){+.+.}, at: n_tty_read+0x321/0x1cc0 drivers/tty/n_tty.c:2131 2 locks held by getty/4428: #0: 0000000026374b78 (&tty->ldisc_sem){++++}, at: ldsem_down_read+0x37/0x40 drivers/tty/tty_ldsem.c:365 #1: 000000002d33d0b2 (&ldata->atomic_read_lock){+.+.}, at: n_tty_read+0x321/0x1cc0 drivers/tty/n_tty.c:2131 2 locks held by getty/4429: #0: 00000000ab8dab0d (&tty->ldisc_sem){++++}, at: ldsem_down_read+0x37/0x40 drivers/tty/tty_ldsem.c:365 #1: 000000006fb9c266 (&ldata->atomic_read_lock){+.+.}, at: n_tty_read+0x321/0x1cc0 drivers/tty/n_tty.c:2131 2 locks held by getty/4430: #0: 00000000eb55bd46 (&tty->ldisc_sem){++++}, at: ldsem_down_read+0x37/0x40 drivers/tty/tty_ldsem.c:365 #1: 00000000c3b46d0c (&ldata->atomic_read_lock){+.+.}, at: n_tty_read+0x321/0x1cc0 drivers/tty/n_tty.c:2131 2 locks held by getty/4431: #0: 00000000d78916eb (&tty->ldisc_sem){++++}, at: ldsem_down_read+0x37/0x40 drivers/tty/tty_ldsem.c:365 #1: 00000000fb69ba89 (&ldata->atomic_read_lock){+.+.}, at: n_tty_read+0x321/0x1cc0 drivers/tty/n_tty.c:2131 2 locks held by kworker/u4:0/28633: #0: 000000002a59db91 ((wq_completion)"events_unbound"){+.+.}, at: __write_once_size include/linux/compiler.h:215 [inline] #0: 000000002a59db91 ((wq_completion)"events_unbound"){+.+.}, at: arch_atomic64_set arch/x86/include/asm/atomic64_64.h:34 [inline] #0: 000000002a59db91 ((wq_completion)"events_unbound"){+.+.}, at: atomic64_set include/asm-generic/atomic-instrumented.h:40 [inline] #0: 000000002a59db91 ((wq_completion)"events_unbound"){+.+.}, at: atomic_long_set include/asm-generic/atomic-long.h:57 [inline] #0: 000000002a59db91 ((wq_completion)"events_unbound"){+.+.}, at: set_work_data kernel/workqueue.c:617 [inline] #0: 000000002a59db91 ((wq_completion)"events_unbound"){+.+.}, at: set_work_pool_and_clear_pending kernel/workqueue.c:644 [inline] #0: 000000002a59db91 ((wq_completion)"events_unbound"){+.+.}, at: process_one_work+0xaef/0x1b50 kernel/workqueue.c:2116 #1: 000000005256c5d3 ((work_completion)(&sub_info->work)){+.+.}, at: process_one_work+0xb46/0x1b50 kernel/workqueue.c:2120 ============================================= NMI backtrace for cpu 1 CPU: 1 PID: 891 Comm: khungtaskd Not tainted 4.17.0-rc1+ #8 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x1b9/0x294 lib/dump_stack.c:113 nmi_cpu_backtrace.cold.4+0x19/0xce lib/nmi_backtrace.c:103 nmi_trigger_cpumask_backtrace+0x151/0x192 lib/nmi_backtrace.c:62 arch_trigger_cpumask_backtrace+0x14/0x20 arch/x86/kernel/apic/hw_nmi.c:38 trigger_all_cpu_backtrace include/linux/nmi.h:138 [inline] check_hung_task kernel/hung_task.c:132 [inline] check_hung_uninterruptible_tasks kernel/hung_task.c:190 [inline] watchdog+0xc10/0xf60 kernel/hung_task.c:249 kthread+0x345/0x410 kernel/kthread.c:238 ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:412 Sending NMI from CPU 1 to CPUs 0: NMI backtrace for cpu 0 CPU: 0 PID: 15629 Comm: udevd Not tainted 4.17.0-rc1+ #8 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 RIP: 0010:__lock_unpin_lock kernel/locking/lockdep.c:3804 [inline] RIP: 0010:lock_unpin_lock+0x1d6/0x490 kernel/locking/lockdep.c:4014 RSP: 0018:ffff8801dae074e8 EFLAGS: 00000097 RAX: 0000000000000003 RBX: 1ffff1003b5c0ea2 RCX: 0000000000000000 RDX: 0000000000000001 RSI: 0000000000007c03 RDI: ffffffff8909cd20 RBP: ffff8801dae07598 R08: ffffed003b5c46c3 R09: ffff8801dae2c598 R10: ffffed003b5c46c2 R11: ffff8801dae23613 R12: ffff8801d31a4340 R13: ffff8801d9a9e240 R14: ffff8801d31a4340 R15: ffff8801dae07728 FS: 00007f89fe29d7a0(0000) GS:ffff8801dae00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 00007f89fe2a4000 CR3: 00000001b8465000 CR4: 00000000001426f0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 Call Trace: rq_unpin_lock kernel/sched/sched.h:1025 [inline] rq_unlock kernel/sched/sched.h:1831 [inline] ttwu_queue kernel/sched/core.c:1841 [inline] try_to_wake_up+0x8a7/0x1190 kernel/sched/core.c:2053 wake_up_process+0x10/0x20 kernel/sched/core.c:2126 process_timeout+0x31/0x40 kernel/time/timer.c:1730 call_timer_fn+0x230/0x940 kernel/time/timer.c:1326 expire_timers kernel/time/timer.c:1363 [inline] __run_timers+0x79e/0xc50 kernel/time/timer.c:1666 run_timer_softirq+0x4c/0x70 kernel/time/timer.c:1692 __do_softirq+0x2e0/0xaf5 kernel/softirq.c:285 invoke_softirq kernel/softirq.c:365 [inline] irq_exit+0x1d1/0x200 kernel/softirq.c:405 exiting_irq arch/x86/include/asm/apic.h:525 [inline] smp_apic_timer_interrupt+0x17e/0x710 arch/x86/kernel/apic/apic.c:1052 apic_timer_interrupt+0xf/0x20 arch/x86/entry/entry_64.S:863 RIP: 0010:arch_local_irq_restore arch/x86/include/asm/paravirt.h:783 [inline] RIP: 0010:lock_release+0x4d4/0xa10 kernel/locking/lockdep.c:3942 RSP: 0018:ffff8801b9b17a60 EFLAGS: 00000286 ORIG_RAX: ffffffffffffff13 RAX: dffffc0000000000 RBX: 1ffff10037362f51 RCX: 1ffff1003a63496d RDX: 1ffffffff11a312d RSI: 0000000000000001 RDI: 0000000000000286 RBP: ffff8801b9b17b90 R08: 1ffff10037362f55 R09: ffff8801b9b178e8 R10: 0000000000000003 R11: 0000000000000000 R12: ffff8801d31a4340 R13: ffffffff88d8e460 R14: ffff8801d31a4340 R15: ffff8801b9b17aa8 rcu_lock_release include/linux/rcupdate.h:251 [inline] rcu_read_unlock_sched include/linux/rcupdate.h:769 [inline] __fd_install+0x2d2/0x880 fs/file.c:607 fd_install+0x4d/0x60 fs/file.c:612 do_sys_open+0x67e/0x740 fs/open.c:1099 __do_sys_open fs/open.c:1111 [inline] __se_sys_open fs/open.c:1106 [inline] __x64_sys_open+0x7e/0xc0 fs/open.c:1106 do_syscall_64+0x1b1/0x800 arch/x86/entry/common.c:287 entry_SYSCALL_64_after_hwframe+0x49/0xbe RIP: 0033:0x7f89fd9a5120 RSP: 002b:00007fff1fbf45d8 EFLAGS: 00000246 ORIG_RAX: 0000000000000002 RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 00007f89fd9a5120 RDX: 0000000000000124 RSI: 0000000000080000 RDI: 00007fff1fbf4680 RBP: 0000000000f8b744 R08: 0000000000f8b744 R09: 00007f89fd9fac20 R10: 7269762f73656369 R11: 0000000000000246 R12: 0000000000f712d0 R13: 0000000000f713c0 R14: 0000000000f71250 R15: 0000000000f76e10 Code: fc ff df 0f b6 14 02 48 89 f8 83 e0 07 83 c0 03 65 4c 8b 34 25 c0 ed 01 00 38 d0 7c 08 84 d2 0f 85 1d 02 00 00 8b 15 da ae ad 07 <85> d2 0f 84 f9 00 00 00 49 8d 86 30 08 00 00 48 89 c2 48 89 85