INFO: task kworker/1:4:2742 blocked for more than 143 seconds.
Not tainted 5.3.0+ #0
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
kworker/1:4 D23376 2742 2 0x80004000
Workqueue: pm hcd_resume_work
Call Trace:
schedule+0xca/0x250 kernel/sched/core.c:4132
schedule_preempt_disabled+0xc/0x20 kernel/sched/core.c:4191
__mutex_lock_common kernel/locking/mutex.c:1033 [inline]
__mutex_lock+0x881/0x1360 kernel/locking/mutex.c:1103
device_lock include/linux/device.h:1462 [inline]
usb_remote_wakeup+0x1f/0xb0 drivers/usb/core/hub.c:3600
process_one_work+0x92b/0x1530 kernel/workqueue.c:2269
worker_thread+0x96/0xe20 kernel/workqueue.c:2415
kthread+0x318/0x420 kernel/kthread.c:255
ret_from_fork+0x24/0x30 arch/x86/entry/entry_64.S:352
Showing all locks held in the system:
1 lock held by khungtaskd/23:
#0: ffffffff86eff3e0 (rcu_read_lock){....}, at: debug_show_all_locks+0x53/0x269 kernel/locking/lockdep.c:5335
1 lock held by rsyslogd/1601:
2 locks held by getty/1691:
#0: ffff8881d6785e10 (&tty->ldisc_sem){++++}, at: tty_ldisc_ref_wait+0x22/0x80 drivers/tty/tty_ldisc.c:272
#1: ffffc900004752e0 (&ldata->atomic_read_lock){+.+.}, at: n_tty_read+0x223/0x1ae0 drivers/tty/n_tty.c:2156
2 locks held by getty/1692:
#0: ffff8881d671cd10 (&tty->ldisc_sem){++++}, at: tty_ldisc_ref_wait+0x22/0x80 drivers/tty/tty_ldisc.c:272
#1: ffffc9000049d2e0 (&ldata->atomic_read_lock){+.+.}, at: n_tty_read+0x223/0x1ae0 drivers/tty/n_tty.c:2156
2 locks held by getty/1693:
#0: ffff8881d9080090 (&tty->ldisc_sem){++++}, at: tty_ldisc_ref_wait+0x22/0x80 drivers/tty/tty_ldisc.c:272
#1: ffffc900004852e0 (&ldata->atomic_read_lock){+.+.}, at: n_tty_read+0x223/0x1ae0 drivers/tty/n_tty.c:2156
2 locks held by getty/1694:
#0: ffff8881d671ef10 (&tty->ldisc_sem){++++}, at: tty_ldisc_ref_wait+0x22/0x80 drivers/tty/tty_ldisc.c:272
#1: ffffc9000048d2e0 (&ldata->atomic_read_lock){+.+.}, at: n_tty_read+0x223/0x1ae0 drivers/tty/n_tty.c:2156
2 locks held by getty/1695:
#0: ffff8881d91bd590 (&tty->ldisc_sem){++++}, at: tty_ldisc_ref_wait+0x22/0x80 drivers/tty/tty_ldisc.c:272
#1: ffffc900004912e0 (&ldata->atomic_read_lock){+.+.}, at: n_tty_read+0x223/0x1ae0 drivers/tty/n_tty.c:2156
2 locks held by getty/1696:
#0: ffff8881d671e690 (&tty->ldisc_sem){++++}, at: tty_ldisc_ref_wait+0x22/0x80 drivers/tty/tty_ldisc.c:272
#1: ffffc900004992e0 (&ldata->atomic_read_lock){+.+.}, at: n_tty_read+0x223/0x1ae0 drivers/tty/n_tty.c:2156
2 locks held by getty/1697:
#0: ffff8881d9083390 (&tty->ldisc_sem){++++}, at: tty_ldisc_ref_wait+0x22/0x80 drivers/tty/tty_ldisc.c:272
#1: ffffc9000046d2e0 (&ldata->atomic_read_lock){+.+.}, at: n_tty_read+0x223/0x1ae0 drivers/tty/n_tty.c:2156
3 locks held by kworker/1:4/2742:
#0: ffff8881da163ca8 ((wq_completion)pm){+.+.}, at: __write_once_size include/linux/compiler.h:226 [inline]
#0: ffff8881da163ca8 ((wq_completion)pm){+.+.}, at: arch_atomic64_set arch/x86/include/asm/atomic64_64.h:34 [inline]
#0: ffff8881da163ca8 ((wq_completion)pm){+.+.}, at: atomic64_set include/asm-generic/atomic-instrumented.h:855 [inline]
#0: ffff8881da163ca8 ((wq_completion)pm){+.+.}, at: atomic_long_set include/asm-generic/atomic-long.h:40 [inline]
#0: ffff8881da163ca8 ((wq_completion)pm){+.+.}, at: set_work_data kernel/workqueue.c:620 [inline]
#0: ffff8881da163ca8 ((wq_completion)pm){+.+.}, at: set_work_pool_and_clear_pending kernel/workqueue.c:647 [inline]
#0: ffff8881da163ca8 ((wq_completion)pm){+.+.}, at: process_one_work+0x827/0x1530 kernel/workqueue.c:2240
#1: ffff8881ae097dd0 ((work_completion)(&hcd->wakeup_work)){+.+.}, at: process_one_work+0x85b/0x1530 kernel/workqueue.c:2244
#2: ffff8881d5c62c80 (&dev->mutex){....}, at: device_lock include/linux/device.h:1462 [inline]
#2: ffff8881d5c62c80 (&dev->mutex){....}, at: usb_remote_wakeup+0x1f/0xb0 drivers/usb/core/hub.c:3600
3 locks held by kworker/1:6/2766:
6 locks held by kworker/0:3/2775:
#0: ffff8881d91ec8a8 ((wq_completion)usb_hub_wq){+.+.}, at: __write_once_size include/linux/compiler.h:226 [inline]
#0: ffff8881d91ec8a8 ((wq_completion)usb_hub_wq){+.+.}, at: arch_atomic64_set arch/x86/include/asm/atomic64_64.h:34 [inline]
#0: ffff8881d91ec8a8 ((wq_completion)usb_hub_wq){+.+.}, at: atomic64_set include/asm-generic/atomic-instrumented.h:855 [inline]
#0: ffff8881d91ec8a8 ((wq_completion)usb_hub_wq){+.+.}, at: atomic_long_set include/asm-generic/atomic-long.h:40 [inline]
#0: ffff8881d91ec8a8 ((wq_completion)usb_hub_wq){+.+.}, at: set_work_data kernel/workqueue.c:620 [inline]
#0: ffff8881d91ec8a8 ((wq_completion)usb_hub_wq){+.+.}, at: set_work_pool_and_clear_pending kernel/workqueue.c:647 [inline]
#0: ffff8881d91ec8a8 ((wq_completion)usb_hub_wq){+.+.}, at: process_one_work+0x827/0x1530 kernel/workqueue.c:2240
#1: ffff8881ad9afdd0 ((work_completion)(&hub->events)){+.+.}, at: process_one_work+0x85b/0x1530 kernel/workqueue.c:2244
#2: ffff8881d5cb5f80 (&dev->mutex){....}, at: device_lock include/linux/device.h:1462 [inline]
#2: ffff8881d5cb5f80 (&dev->mutex){....}, at: hub_event+0x17c/0x3640 drivers/usb/core/hub.c:5387
#3: ffff8881d5cb73a0 (&port_dev->status_lock){+.+.}, at: usb_lock_port drivers/usb/core/hub.c:2996 [inline]
#3: ffff8881d5cb73a0 (&port_dev->status_lock){+.+.}, at: hub_port_connect drivers/usb/core/hub.c:5029 [inline]
#3: ffff8881d5cb73a0 (&port_dev->status_lock){+.+.}, at: hub_port_connect_change drivers/usb/core/hub.c:5213 [inline]
#3: ffff8881d5cb73a0 (&port_dev->status_lock){+.+.}, at: port_event drivers/usb/core/hub.c:5359 [inline]
#3: ffff8881d5cb73a0 (&port_dev->status_lock){+.+.}, at: hub_event+0x196a/0x3640 drivers/usb/core/hub.c:5441
#4: ffff8881d628e5a0 (hcd->address0_mutex){+.+.}, at: hub_port_init+0x1b6/0x2d30 drivers/usb/core/hub.c:4538
#5: ffffffff8741f8a8 (ehci_cf_port_reset_rwsem){.+.+}, at: hub_port_reset+0x17b/0x1460 drivers/usb/core/hub.c:2858
5 locks held by kworker/0:4/2839:
#0: ffff8881d91ec8a8 ((wq_completion)usb_hub_wq){+.+.}, at: __write_once_size include/linux/compiler.h:226 [inline]
#0: ffff8881d91ec8a8 ((wq_completion)usb_hub_wq){+.+.}, at: arch_atomic64_set arch/x86/include/asm/atomic64_64.h:34 [inline]
#0: ffff8881d91ec8a8 ((wq_completion)usb_hub_wq){+.+.}, at: atomic64_set include/asm-generic/atomic-instrumented.h:855 [inline]
#0: ffff8881d91ec8a8 ((wq_completion)usb_hub_wq){+.+.}, at: atomic_long_set include/asm-generic/atomic-long.h:40 [inline]
#0: ffff8881d91ec8a8 ((wq_completion)usb_hub_wq){+.+.}, at: set_work_data kernel/workqueue.c:620 [inline]
#0: ffff8881d91ec8a8 ((wq_completion)usb_hub_wq){+.+.}, at: set_work_pool_and_clear_pending kernel/workqueue.c:647 [inline]
#0: ffff8881d91ec8a8 ((wq_completion)usb_hub_wq){+.+.}, at: process_one_work+0x827/0x1530 kernel/workqueue.c:2240
#1: ffff8881d113fdd0 ((work_completion)(&hub->events)){+.+.}, at: process_one_work+0x85b/0x1530 kernel/workqueue.c:2244
#2: ffff8881d5c62c80 (&dev->mutex){....}, at: device_lock include/linux/device.h:1462 [inline]
#2: ffff8881d5c62c80 (&dev->mutex){....}, at: hub_event+0x17c/0x3640 drivers/usb/core/hub.c:5387
#3: ffff8881cfc8c600 (&dev->mutex){....}, at: device_lock include/linux/device.h:1462 [inline]
#3: ffff8881cfc8c600 (&dev->mutex){....}, at: __device_attach+0x7b/0x360 drivers/base/dd.c:871
#4: ffff8881c1b97890 (&dev->mutex){....}, at: device_lock include/linux/device.h:1462 [inline]
#4: ffff8881c1b97890 (&dev->mutex){....}, at: __device_attach+0x7b/0x360 drivers/base/dd.c:871
4 locks held by kworker/0:5/2846:
#0: ffff8881db231dd8 (&rq->lock){-.-.}, at: rq_lock kernel/sched/sched.h:1211 [inline]
#0: ffff8881db231dd8 (&rq->lock){-.-.}, at: __schedule+0x21c/0x1ac0 kernel/sched/core.c:4017
#1: ffffffff86eff3e0 (rcu_read_lock){....}, at: trace_sched_stat_runtime include/trace/events/sched.h:435 [inline]
#1: ffffffff86eff3e0 (rcu_read_lock){....}, at: update_curr+0x27f/0x810 kernel/sched/fair.c:861
#2: ffff8881d5c66870 (&(&dev->power.lock)->rlock){-.-.}, at: spin_lock_irq include/linux/spinlock.h:363 [inline]
#2: ffff8881d5c66870 (&(&dev->power.lock)->rlock){-.-.}, at: __rpm_callback+0x188/0x3c0 drivers/base/power/runtime.c:380
#3: ffff8881d5c669a0 (&dev->power.wait_queue){..-.}, at: __wake_up_common_lock+0xb4/0x130 kernel/sched/wait.c:122
3 locks held by udevd/7257:
=============================================
NMI backtrace for cpu 1
CPU: 1 PID: 23 Comm: khungtaskd Not tainted 5.3.0+ #0
Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011
Call Trace:
__dump_stack lib/dump_stack.c:77 [inline]
dump_stack+0xca/0x13e lib/dump_stack.c:113
nmi_cpu_backtrace.cold+0x55/0x96 lib/nmi_backtrace.c:101
nmi_trigger_cpumask_backtrace+0x1b0/0x1c7 lib/nmi_backtrace.c:62
trigger_all_cpu_backtrace include/linux/nmi.h:146 [inline]
check_hung_uninterruptible_tasks kernel/hung_task.c:205 [inline]
watchdog+0x9a4/0xe50 kernel/hung_task.c:289
kthread+0x318/0x420 kernel/kthread.c:255
ret_from_fork+0x24/0x30 arch/x86/entry/entry_64.S:352
Sending NMI from CPU 1 to CPUs 0:
NMI backtrace for cpu 0
CPU: 0 PID: 2775 Comm: kworker/0:3 Not tainted 5.3.0+ #0
Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011
Workqueue: usb_hub_wq hub_event
RIP: 0010:timerqueue_del+0x2/0x150 lib/timerqueue.c:64
Code: e8 13 5f ff ff 49 89 c4 e8 bb fc ce fb 4c 89 e0 5d 41 5c c3 e8 af fc ce fb 45 31 e4 5d 4c 89 e0 41 5c c3 0f 1f 44 00 00 41 54 <55> 48 89 fd 53 48 89 f3 48 83 ec 08 e8 8d fc ce fb 48 89 da 48 b8
RSP: 0018:ffff8881db209de8 EFLAGS: 00000006
RAX: ffff8881ad9a0000 RBX: ffff8881c696fdd0 RCX: ffffffff812fc054
RDX: 0000000000010000 RSI: ffff8881c696fdd0 RDI: ffff8881db225bc0
RBP: ffff8881db225b80 R08: ffff8881ad9a0000 R09: ffffed103b64657d
R10: ffffed103b64657c R11: ffff8881db232be3 R12: 0000000000000001
R13: 0000000000000000 R14: ffff8881db225b00 R15: 0000000000000000
FS: 0000000000000000(0000) GS:ffff8881db200000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fdfae922000 CR3: 00000001b2667000 CR4: 00000000001406f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
__remove_hrtimer+0x99/0x1a0 kernel/time/hrtimer.c:995
__run_hrtimer kernel/time/hrtimer.c:1496 [inline]
__hrtimer_run_queues+0x276/0xc50 kernel/time/hrtimer.c:1576
hrtimer_interrupt+0x2e8/0x730 kernel/time/hrtimer.c:1638
local_apic_timer_interrupt arch/x86/kernel/apic/apic.c:1110 [inline]
smp_apic_timer_interrupt+0xf5/0x500 arch/x86/kernel/apic/apic.c:1135
apic_timer_interrupt+0xf/0x20 arch/x86/entry/entry_64.S:830
RIP: 0010:arch_local_irq_restore arch/x86/include/asm/irqflags.h:85 [inline]
RIP: 0010:console_trylock_spinning kernel/printk/printk.c:1751 [inline]
RIP: 0010:vprintk_emit+0x3cd/0x3e0 kernel/printk/printk.c:1995
Code: 00 83 fb ff 75 d6 e9 db fc ff ff e8 ad 87 15 00 e8 c8 b5 1a 00 41 56 9d e9 b1 fd ff ff e8 9b 87 15 00 e8 b6 b5 1a 00 41 56 9d 2a ff ff ff 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 55 48 89
RSP: 0018:ffff8881ad9af530 EFLAGS: 00000216 ORIG_RAX: ffffffffffffff13
RAX: 0000000000000007 RBX: 0000000000000200 RCX: 0000000000000006
RDX: 0000000000000000 RSI: ffff8881ad9a08a0 RDI: ffff8881ad9a084c
RBP: ffff8881ad9af578 R08: ffff8881ad9a0000 R09: fffffbfff11f45ae
R10: fffffbfff11f45ad R11: ffffffff88fa2d6f R12: 0000000000000054
R13: ffff8881da25c800 R14: 0000000000000216 R15: 0000000000000000
dev_vprintk_emit+0x4fc/0x541 drivers/base/core.c:3312
dev_printk_emit+0xba/0xf1 drivers/base/core.c:3323
__dev_printk+0x1db/0x203 drivers/base/core.c:3335
_dev_warn+0xd7/0x109 drivers/base/core.c:3379
usb_parse_endpoint drivers/usb/core/config.c:393 [inline]
usb_parse_interface drivers/usb/core/config.c:520 [inline]
usb_parse_configuration drivers/usb/core/config.c:733 [inline]
usb_get_configuration.cold+0xbad/0x1675 drivers/usb/core/config.c:882
usb_enumerate_device drivers/usb/core/hub.c:2369 [inline]
usb_new_device+0xd3/0x160 drivers/usb/core/hub.c:2505
hub_port_connect drivers/usb/core/hub.c:5098 [inline]
hub_port_connect_change drivers/usb/core/hub.c:5213 [inline]
port_event drivers/usb/core/hub.c:5359 [inline]
hub_event+0x1b5c/0x3640 drivers/usb/core/hub.c:5441
process_one_work+0x92b/0x1530 kernel/workqueue.c:2269
process_scheduled_works kernel/workqueue.c:2331 [inline]
worker_thread+0x7ab/0xe20 kernel/workqueue.c:2417
kthread+0x318/0x420 kernel/kthread.c:255
ret_from_fork+0x24/0x30 arch/x86/entry/entry_64.S:352