ip6_tunnel: ®c9A«÷ª0HÒ`Š xmit: Local address not yet configured! BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=0 stuck for 180s! Showing busy workqueues and worker pools: workqueue events: flags=0x0 pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=2/256 pending: proc_cleanup_work, free_obj_work ====================================================== WARNING: possible circular locking dependency detected 4.14.71+ #8 Not tainted ------------------------------------------------------ swapper/0/0 is trying to acquire lock: (&port_lock_key){-.-.}, at: [] serial8250_console_write+0x6af/0x800 drivers/tty/serial/8250/8250_port.c:3232 but task is already holding lock: (&(&pool->lock)->rlock){-.-.}, at: [] show_workqueue_state.cold.23+0x278/0x1053 kernel/workqueue.c:4495 which lock already depends on the new lock. the existing dependency chain (in reverse order) is: -> #3 (&(&pool->lock)->rlock){-.-.}: __raw_spin_lock include/linux/spinlock_api_smp.h:142 [inline] _raw_spin_lock+0x2a/0x40 kernel/locking/spinlock.c:152 spin_lock include/linux/spinlock.h:317 [inline] __queue_work+0x184/0xe40 kernel/workqueue.c:1419 queue_work_on+0x8f/0xa0 kernel/workqueue.c:1488 queue_work include/linux/workqueue.h:491 [inline] schedule_work include/linux/workqueue.h:549 [inline] put_pwq+0x131/0x180 kernel/workqueue.c:1092 put_pwq_unlocked.part.5+0x30/0x70 kernel/workqueue.c:1109 put_pwq_unlocked kernel/workqueue.c:1103 [inline] destroy_workqueue+0x4dc/0x600 kernel/workqueue.c:4150 ext4_fill_super+0x7342/0xb5e0 fs/ext4/super.c:4425 mount_bdev+0x2ab/0x370 fs/super.c:1149 mount_fs+0x28f/0x31a fs/super.c:1255 vfs_kern_mount.part.9+0xce/0x4a0 fs/namespace.c:1056 vfs_kern_mount fs/namespace.c:1038 [inline] do_new_mount fs/namespace.c:2561 [inline] do_mount+0x404/0x26e0 fs/namespace.c:2891 SYSC_mount fs/namespace.c:3107 [inline] SyS_mount+0xe5/0x100 fs/namespace.c:3084 do_syscall_64+0x19b/0x4b0 arch/x86/entry/common.c:289 entry_SYSCALL_64_after_hwframe+0x42/0xb7 -> #2 (&pool->lock/1){..-.}: __raw_spin_lock include/linux/spinlock_api_smp.h:142 [inline] _raw_spin_lock+0x2a/0x40 kernel/locking/spinlock.c:152 spin_lock include/linux/spinlock.h:317 [inline] __queue_work+0x184/0xe40 kernel/workqueue.c:1419 queue_work_on+0x8f/0xa0 kernel/workqueue.c:1488 pty_write+0x172/0x1c0 drivers/tty/pty.c:124 n_tty_write+0x969/0xda0 drivers/tty/n_tty.c:2342 do_tty_write drivers/tty/tty_io.c:957 [inline] tty_write+0x397/0x810 drivers/tty/tty_io.c:1041 __vfs_write+0xf4/0x5c0 fs/read_write.c:482 vfs_write+0x17f/0x4d0 fs/read_write.c:546 SYSC_write fs/read_write.c:593 [inline] SyS_write+0xc2/0x1a0 fs/read_write.c:585 do_syscall_64+0x19b/0x4b0 arch/x86/entry/common.c:289 entry_SYSCALL_64_after_hwframe+0x42/0xb7 -> #1 (&(&port->lock)->rlock){-.-.}: __raw_spin_lock_irqsave include/linux/spinlock_api_smp.h:110 [inline] _raw_spin_lock_irqsave+0x41/0x50 kernel/locking/spinlock.c:160 tty_port_tty_get+0x1b/0x80 drivers/tty/tty_port.c:287 tty_port_default_wakeup+0x11/0x40 drivers/tty/tty_port.c:46 serial8250_tx_chars+0x417/0xa70 drivers/tty/serial/8250/8250_port.c:1810 serial8250_handle_irq.part.15+0x1b1/0x1f0 drivers/tty/serial/8250/8250_port.c:1883 serial8250_handle_irq drivers/tty/serial/8250/8250_port.c:1869 [inline] serial8250_default_handle_irq+0x9b/0x110 drivers/tty/serial/8250/8250_port.c:1899 serial8250_interrupt+0xea/0x1a0 drivers/tty/serial/8250/8250_core.c:129 __handle_irq_event_percpu+0xf8/0x770 kernel/irq/handle.c:147 handle_irq_event_percpu+0x76/0x150 kernel/irq/handle.c:187 handle_irq_event+0xa1/0x12d kernel/irq/handle.c:204 handle_edge_irq+0x1f0/0x7f0 kernel/irq/chip.c:770 generic_handle_irq_desc include/linux/irqdesc.h:158 [inline] handle_irq+0x242/0x328 arch/x86/kernel/irq_64.c:78 do_IRQ+0x7d/0x1b0 arch/x86/kernel/irq.c:230 ret_from_intr+0x0/0x22 native_safe_halt+0x2/0x10 arch/x86/include/asm/irqflags.h:57 arch_safe_halt arch/x86/include/asm/paravirt.h:94 [inline] default_idle+0x50/0x380 arch/x86/kernel/process.c:501 cpuidle_idle_call kernel/sched/idle.c:159 [inline] do_idle+0x2da/0x3b0 kernel/sched/idle.c:268 cpu_startup_entry+0xc9/0xe0 kernel/sched/idle.c:374 start_kernel+0x6fb/0x739 init/main.c:710 secondary_startup_64+0xa5/0xb0 arch/x86/kernel/head_64.S:239 -> #0 (&port_lock_key){-.-.}: lock_acquire+0x10f/0x380 kernel/locking/lockdep.c:3991 __raw_spin_lock_irqsave include/linux/spinlock_api_smp.h:110 [inline] _raw_spin_lock_irqsave+0x41/0x50 kernel/locking/spinlock.c:160 serial8250_console_write+0x6af/0x800 drivers/tty/serial/8250/8250_port.c:3232 call_console_drivers kernel/printk/printk.c:1574 [inline] console_unlock+0x5b9/0xb50 kernel/printk/printk.c:2242 vprintk_emit+0x112/0x150 kernel/printk/printk.c:1766 vprintk_func+0x58/0x159 kernel/printk/printk_safe.c:401 printk+0xa7/0xcf kernel/printk/printk.c:1838 show_pwq kernel/workqueue.c:4402 [inline] show_workqueue_state.cold.23+0x3d5/0x1053 kernel/workqueue.c:4497 wq_watchdog_timer_fn+0x436/0x4d0 kernel/workqueue.c:5481 call_timer_fn+0x163/0x6a0 kernel/time/timer.c:1279 expire_timers+0x1f3/0x4a0 kernel/time/timer.c:1318 __run_timers kernel/time/timer.c:1634 [inline] run_timer_softirq+0x3a5/0x560 kernel/time/timer.c:1649 __do_softirq+0x215/0x997 kernel/softirq.c:288 invoke_softirq kernel/softirq.c:368 [inline] irq_exit+0x10f/0x150 kernel/softirq.c:409 exiting_irq arch/x86/include/asm/apic.h:648 [inline] smp_apic_timer_interrupt+0x188/0x5f0 arch/x86/kernel/apic/apic.c:1064 apic_timer_interrupt+0x84/0x90 arch/x86/entry/entry_64.S:787 native_safe_halt+0x2/0x10 arch/x86/include/asm/irqflags.h:57 arch_safe_halt arch/x86/include/asm/paravirt.h:94 [inline] default_idle+0x50/0x380 arch/x86/kernel/process.c:501 cpuidle_idle_call kernel/sched/idle.c:159 [inline] do_idle+0x2da/0x3b0 kernel/sched/idle.c:268 cpu_startup_entry+0xc9/0xe0 kernel/sched/idle.c:374 start_kernel+0x6fb/0x739 init/main.c:710 secondary_startup_64+0xa5/0xb0 arch/x86/kernel/head_64.S:239 other info that might help us debug this: Chain exists of: &port_lock_key --> &pool->lock/1 --> &(&pool->lock)->rlock Possible unsafe locking scenario: CPU0 CPU1 ---- ---- lock(&(&pool->lock)->rlock); lock(&pool->lock/1); lock(&(&pool->lock)->rlock); lock(&port_lock_key); *** DEADLOCK *** 4 locks held by swapper/0/0: #0: (kernel/workqueue.c:5419){+.-.}, at: [] lockdep_copy_map include/linux/lockdep.h:174 [inline] #0: (kernel/workqueue.c:5419){+.-.}, at: [] call_timer_fn+0xd7/0x6a0 kernel/time/timer.c:1269 #1: (rcu_read_lock_sched){....}, at: [] show_workqueue_state+0x0/0x100 kernel/workqueue.c:4367 #2: (&(&pool->lock)->rlock){-.-.}, at: [] show_workqueue_state.cold.23+0x278/0x1053 kernel/workqueue.c:4495 #3: (console_lock){+.+.}, at: [] vprintk_emit+0x104/0x150 kernel/printk/printk.c:1765 stack backtrace: CPU: 0 PID: 0 Comm: swapper/0 Not tainted 4.14.71+ #8 Call Trace: __dump_stack lib/dump_stack.c:17 [inline] dump_stack+0xb9/0x11b lib/dump_stack.c:53 print_circular_bug.isra.18.cold.43+0x2d3/0x40c kernel/locking/lockdep.c:1258 check_prev_add kernel/locking/lockdep.c:1901 [inline] check_prevs_add kernel/locking/lockdep.c:2018 [inline] validate_chain kernel/locking/lockdep.c:2460 [inline] __lock_acquire+0x2ff9/0x4320 kernel/locking/lockdep.c:3487 lock_acquire+0x10f/0x380 kernel/locking/lockdep.c:3991 __raw_spin_lock_irqsave include/linux/spinlock_api_smp.h:110 [inline] _raw_spin_lock_irqsave+0x41/0x50 kernel/locking/spinlock.c:160 serial8250_console_write+0x6af/0x800 drivers/tty/serial/8250/8250_port.c:3232 call_console_drivers kernel/printk/printk.c:1574 [inline] console_unlock+0x5b9/0xb50 kernel/printk/printk.c:2242 vprintk_emit+0x112/0x150 kernel/printk/printk.c:1766 vprintk_func+0x58/0x159 kernel/printk/printk_safe.c:401 printk+0xa7/0xcf kernel/printk/printk.c:1838 show_pwq kernel/workqueue.c:4402 [inline] show_workqueue_state.cold.23+0x3d5/0x1053 kernel/workqueue.c:4497 wq_watchdog_timer_fn+0x436/0x4d0 kernel/workqueue.c:5481 call_timer_fn+0x163/0x6a0 kernel/time/timer.c:1279 expire_timers+0x1f3/0x4a0 kernel/time/timer.c:1318 __run_timers kernel/time/timer.c:1634 [inline] run_timer_softirq+0x3a5/0x560 kernel/time/timer.c:1649 __do_softirq+0x215/0x997 kernel/softirq.c:288 invoke_softirq kernel/softirq.c:368 [inline] irq_exit+0x10f/0x150 kernel/softirq.c:409 exiting_irq arch/x86/include/asm/apic.h:648 [inline] smp_apic_timer_interrupt+0x188/0x5f0 arch/x86/kernel/apic/apic.c:1064 apic_timer_interrupt+0x84/0x90 arch/x86/entry/entry_64.S:787 RIP: 0010:native_safe_halt+0x2/0x10 arch/x86/include/asm/irqflags.h:57 RSP: 0018:ffffffffacc07d28 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff10 RAX: 0000000000000007 RBX: ffffffffad028668 RCX: 0000000000000000 RDX: 0000000000000000 RSI: 0000000000000001 RDI: ffffffffacc2ca2c RBP: ffffffffad032aa8 R08: 0000000000000001 R09: 0000000000000000 R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000 R13: 1ffffffff5980fae R14: 0000000000000000 R15: ffffffffad032b64 arch_safe_halt arch/x86/include/asm/paravirt.h:94 [inline] default_idle+0x50/0x380 arch/x86/kernel/process.c:501 cpuidle_idle_call kernel/sched/idle.c:159 [inline] do_idle+0x2da/0x3b0 kernel/sched/idle.c:268 cpu_startup_entry+0xc9/0xe0 kernel/sched/idle.c:374 start_kernel+0x6fb/0x739 init/main.c:710 secondary_startup_64+0xa5/0xb0 arch/x86/kernel/head_64.S:239 pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=2/256 in-flight: 12324:wait_rcu_exp_gp, 12319:linkwatch_event workqueue events_long: flags=0x0 pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 in-flight: 27055:gc_worker workqueue events_power_efficient: flags=0x80 pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 pending: neigh_periodic_work workqueue mm_percpu_wq: flags=0x8 pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 pending: vmstat_update workqueue netns: flags=0xe000a pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/1 in-flight: 9965:cleanup_net delayed: cleanup_net workqueue dm_bufio_cache: flags=0x8 pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 pending: work_fn workqueue ipv6_addrconf: flags=0x40008 pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/1 pending: addrconf_verify_work pool 0: cpus=0 node=0 flags=0x0 nice=0 hung=1s workers=33 idle: 12328 12325 12321 12326 26982 27037 26977 27033 26988 26618 27032 12317 12323 26992 26995 12331 12332 12316 27054 12320 26600 27035 27039 27053 27052 26961 12318 27042 26617 12322 27045 pool 2: cpus=1 node=0 flags=0x0 nice=0 hung=181s workers=5 idle: 26610 27056 27057 27048 pool 4: cpus=0-1 flags=0x4 nice=0 hung=123s workers=5 idle: 27717 9874 9869 27719 INFO: task syz-executor2:12298 blocked for more than 140 seconds. Not tainted 4.14.71+ #8 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. syz-executor2 D28200 12298 26694 0x90000006 Call Trace: schedule+0x7f/0x1b0 kernel/sched/core.c:3490 schedule_preempt_disabled+0x13/0x20 kernel/sched/core.c:3548 __mutex_lock_common kernel/locking/mutex.c:833 [inline] __mutex_lock+0x521/0x1480 kernel/locking/mutex.c:893 tun_detach drivers/net/tun.c:585 [inline] tun_chr_close+0x33/0x50 drivers/net/tun.c:2655 __fput+0x25e/0x6f0 fs/file_table.c:210 task_work_run+0x116/0x190 kernel/task_work.c:113 exit_task_work include/linux/task_work.h:22 [inline] do_exit+0x8f8/0x2800 kernel/exit.c:865 do_group_exit+0x100/0x2e0 kernel/exit.c:968 get_signal+0x4e5/0x1470 kernel/signal.c:2348 do_signal+0x8f/0x1660 arch/x86/kernel/signal.c:809 exit_to_usermode_loop+0x116/0x150 arch/x86/entry/common.c:159 prepare_exit_to_usermode arch/x86/entry/common.c:198 [inline] syscall_return_slowpath arch/x86/entry/common.c:267 [inline] do_syscall_64+0x35d/0x4b0 arch/x86/entry/common.c:294 entry_SYSCALL_64_after_hwframe+0x42/0xb7 RIP: 0033:0x457679 RSP: 002b:00007f3819bc7c78 EFLAGS: 00000246 ORIG_RAX: 000000000000002e RAX: 0000000000000014 RBX: 00007f3819bc86d4 RCX: 0000000000457679 RDX: 0000000000000000 RSI: 00000000200000c0 RDI: 0000000000000006 RBP: 000000000072c180 R08: 0000000000000000 R09: 0000000000000000 R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff R13: 00000000004d5508 R14: 00000000004c38bf R15: 0000000000000004 INFO: task syz-executor0:12292 blocked for more than 140 seconds. Not tainted 4.14.71+ #8 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. syz-executor0 D28232 12292 27110 0x90000006 Call Trace: schedule+0x7f/0x1b0 kernel/sched/core.c:3490 schedule_preempt_disabled+0x13/0x20 kernel/sched/core.c:3548 __mutex_lock_common kernel/locking/mutex.c:833 [inline] __mutex_lock+0x521/0x1480 kernel/locking/mutex.c:893 tun_detach drivers/net/tun.c:585 [inline] tun_chr_close+0x33/0x50 drivers/net/tun.c:2655 __fput+0x25e/0x6f0 fs/file_table.c:210 task_work_run+0x116/0x190 kernel/task_work.c:113 exit_task_work include/linux/task_work.h:22 [inline] do_exit+0x8f8/0x2800 kernel/exit.c:865 do_group_exit+0x100/0x2e0 kernel/exit.c:968 get_signal+0x4e5/0x1470 kernel/signal.c:2348 do_signal+0x8f/0x1660 arch/x86/kernel/signal.c:809 exit_to_usermode_loop+0x116/0x150 arch/x86/entry/common.c:159 prepare_exit_to_usermode arch/x86/entry/common.c:198 [inline] syscall_return_slowpath arch/x86/entry/common.c:267 [inline] do_syscall_64+0x35d/0x4b0 arch/x86/entry/common.c:294 entry_SYSCALL_64_after_hwframe+0x42/0xb7 RIP: 0033:0x457679 RSP: 002b:00007f1d609e4c78 EFLAGS: 00000246 ORIG_RAX: 0000000000000010 RAX: fffffffffffffff2 RBX: 00007f1d609e56d4 RCX: 0000000000457679 RDX: 0000000000400200 RSI: 0000000000008912 RDI: 0000000000000006 RBP: 000000000072bfa0 R08: 0000000000000000 R09: 0000000000000000 R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff R13: 00000000004d06c0 R14: 00000000004c06ca R15: 0000000000000001 INFO: lockdep is turned off. NMI backtrace for cpu 0 CPU: 0 PID: 23 Comm: khungtaskd Not tainted 4.14.71+ #8 Call Trace: __dump_stack lib/dump_stack.c:17 [inline] dump_stack+0xb9/0x11b lib/dump_stack.c:53 nmi_cpu_backtrace.cold.0+0x47/0x85 lib/nmi_backtrace.c:101 nmi_trigger_cpumask_backtrace+0x121/0x146 lib/nmi_backtrace.c:62 trigger_all_cpu_backtrace include/linux/nmi.h:138 [inline] check_hung_uninterruptible_tasks kernel/hung_task.c:196 [inline] watchdog+0x574/0xa70 kernel/hung_task.c:252 kthread+0x348/0x420 kernel/kthread.c:232 ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:402 Sending NMI from CPU 0 to CPUs 1: NMI backtrace for cpu 1 CPU: 1 PID: 12189 Comm: syz-executor1 Not tainted 4.14.71+ #8 task: ffff880162e80000 task.stack: ffff8801be7e0000 RIP: 0033:0x401574 RSP: 002b:00007fae8a51e690 EFLAGS: 00000282 RAX: 00000000cb5af3bc RBX: 0000000000000007 RCX: 0000000000457679 RDX: 0000000000000000 RSI: 00007fae8a51e6c0 RDI: 0000000000000007 RBP: 000000000072bf00 R08: 0000000000000000 R09: 0000000000000000 R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff R13: 00000000004d7950 R14: 00000000004c48c6 R15: 0000000000000000 FS: 00007fae8a51f700(0000) GS:ffff8801dbb00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 000000c420145e80 CR3: 0000000180d3c001 CR4: 00000000001606a0 DR0: 0000000020000000 DR1: 0000000020000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000600