BUG: workqueue lockup - pool cpus=0 node=0 flags=0x0 nice=-20 stuck for 40s! Showing busy workqueues and worker pools: workqueue events: flags=0x0 pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=10/256 pending: defense_work_handler, defense_work_handler, defense_work_handler, defense_work_handler, defense_work_handler, defense_work_handler, defense_work_handler, defense_work_handler, vmstat_shepherd, cache_reap ====================================================== WARNING: possible circular locking dependency detected 4.18.0-rc3+ #128 Not tainted ------------------------------------------------------ ksoftirqd/0/9 is trying to acquire lock: 00000000af0a3ae8 (console_owner){-.-.}, at: log_next kernel/printk/printk.c:496 [inline] 00000000af0a3ae8 (console_owner){-.-.}, at: console_unlock+0x54e/0x10b0 kernel/printk/printk.c:2376 but task is already holding lock: 00000000f83bff9c (&(&pool->lock)->rlock){-.-.}, at: show_workqueue_state.cold.48+0xb16/0x15ec kernel/workqueue.c:4542 which lock already depends on the new lock. the existing dependency chain (in reverse order) is: -> #4 (&(&pool->lock)->rlock){-.-.}: __raw_spin_lock include/linux/spinlock_api_smp.h:142 [inline] _raw_spin_lock+0x2a/0x40 kernel/locking/spinlock.c:144 spin_lock include/linux/spinlock.h:310 [inline] __queue_work+0x352/0x1410 kernel/workqueue.c:1417 queue_work_on+0x19a/0x1e0 kernel/workqueue.c:1486 queue_work include/linux/workqueue.h:512 [inline] schedule_work include/linux/workqueue.h:570 [inline] put_pwq+0x175/0x1c0 kernel/workqueue.c:1090 put_pwq_unlocked.part.29+0x34/0x70 kernel/workqueue.c:1107 put_pwq_unlocked kernel/workqueue.c:1101 [inline] destroy_workqueue+0x880/0x9d0 kernel/workqueue.c:4202 tipc_topsrv_work_stop net/tipc/topsrv.c:641 [inline] tipc_topsrv_stop+0x4b1/0x6ce net/tipc/topsrv.c:700 tipc_exit_net+0x15/0x40 net/tipc/core.c:100 ops_exit_list.isra.7+0xb0/0x160 net/core/net_namespace.c:152 cleanup_net+0x537/0xb50 net/core/net_namespace.c:523 process_one_work+0xc73/0x1ba0 kernel/workqueue.c:2153 worker_thread+0x189/0x13c0 kernel/workqueue.c:2296 kthread+0x345/0x410 kernel/kthread.c:240 ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:412 -> #3 (&pool->lock/1){-.-.}: __raw_spin_lock include/linux/spinlock_api_smp.h:142 [inline] _raw_spin_lock+0x2a/0x40 kernel/locking/spinlock.c:144 spin_lock include/linux/spinlock.h:310 [inline] __queue_work+0x352/0x1410 kernel/workqueue.c:1417 queue_work_on+0x19a/0x1e0 kernel/workqueue.c:1486 queue_work include/linux/workqueue.h:512 [inline] tty_schedule_flip+0x14c/0x1d0 drivers/tty/tty_buffer.c:408 tty_flip_buffer_push+0x15/0x20 drivers/tty/tty_buffer.c:547 pty_write+0x19d/0x1f0 drivers/tty/pty.c:124 n_tty_write+0xc5b/0x11a0 drivers/tty/n_tty.c:2340 do_tty_write drivers/tty/tty_io.c:963 [inline] tty_write+0x45f/0xae0 drivers/tty/tty_io.c:1051 __vfs_write+0x117/0x9f0 fs/read_write.c:485 vfs_write+0x1f8/0x560 fs/read_write.c:549 ksys_write+0x101/0x260 fs/read_write.c:598 __do_sys_write fs/read_write.c:610 [inline] __se_sys_write fs/read_write.c:607 [inline] __x64_sys_write+0x73/0xb0 fs/read_write.c:607 do_syscall_64+0x1b9/0x820 arch/x86/entry/common.c:290 entry_SYSCALL_64_after_hwframe+0x49/0xbe -> #2 (&(&port->lock)->rlock){-.-.}: __raw_spin_lock_irqsave include/linux/spinlock_api_smp.h:110 [inline] _raw_spin_lock_irqsave+0x96/0xc0 kernel/locking/spinlock.c:152 tty_port_tty_get+0x20/0x80 drivers/tty/tty_port.c:288 tty_port_default_wakeup+0x15/0x40 drivers/tty/tty_port.c:47 tty_port_tty_wakeup+0x5d/0x70 drivers/tty/tty_port.c:390 uart_write_wakeup+0x44/0x60 drivers/tty/serial/serial_core.c:103 serial8250_tx_chars+0x4be/0xb60 drivers/tty/serial/8250/8250_port.c:1808 serial8250_handle_irq.part.25+0x1ee/0x280 drivers/tty/serial/8250/8250_port.c:1881 serial8250_handle_irq drivers/tty/serial/8250/8250_port.c:1867 [inline] serial8250_default_handle_irq+0xc8/0x150 drivers/tty/serial/8250/8250_port.c:1897 serial8250_interrupt+0xfa/0x1d0 drivers/tty/serial/8250/8250_core.c:125 __handle_irq_event_percpu+0x1c8/0xaf0 kernel/irq/handle.c:149 handle_irq_event_percpu+0xa0/0x1d0 kernel/irq/handle.c:189 handle_irq_event+0xa7/0x135 kernel/irq/handle.c:206 handle_edge_irq+0x20f/0x870 kernel/irq/chip.c:791 generic_handle_irq_desc include/linux/irqdesc.h:154 [inline] handle_irq+0x18c/0x2e7 arch/x86/kernel/irq_64.c:77 do_IRQ+0x78/0x190 arch/x86/kernel/irq.c:245 ret_from_intr+0x0/0x1e arch_local_irq_restore arch/x86/include/asm/paravirt.h:783 [inline] __raw_spin_unlock_irqrestore include/linux/spinlock_api_smp.h:160 [inline] _raw_spin_unlock_irqrestore+0xa1/0xc0 kernel/locking/spinlock.c:184 spin_unlock_irqrestore include/linux/spinlock.h:365 [inline] uart_write+0x3e7/0x640 drivers/tty/serial/serial_core.c:591 process_output_block drivers/tty/n_tty.c:580 [inline] n_tty_write+0x6c1/0x11a0 drivers/tty/n_tty.c:2317 do_tty_write drivers/tty/tty_io.c:963 [inline] tty_write+0x45f/0xae0 drivers/tty/tty_io.c:1051 redirected_tty_write+0xaf/0xc0 drivers/tty/tty_io.c:1072 __vfs_write+0x117/0x9f0 fs/read_write.c:485 vfs_write+0x1f8/0x560 fs/read_write.c:549 ksys_write+0x101/0x260 fs/read_write.c:598 __do_sys_write fs/read_write.c:610 [inline] __se_sys_write fs/read_write.c:607 [inline] __x64_sys_write+0x73/0xb0 fs/read_write.c:607 do_syscall_64+0x1b9/0x820 arch/x86/entry/common.c:290 entry_SYSCALL_64_after_hwframe+0x49/0xbe -> #1 (&port_lock_key){-.-.}: __raw_spin_lock_irqsave include/linux/spinlock_api_smp.h:110 [inline] _raw_spin_lock_irqsave+0x96/0xc0 kernel/locking/spinlock.c:152 serial8250_console_write+0x8d5/0xb00 drivers/tty/serial/8250/8250_port.c:3230 univ8250_console_write+0x5f/0x70 drivers/tty/serial/8250/8250_core.c:590 call_console_drivers kernel/printk/printk.c:1718 [inline] console_unlock+0xab1/0x10b0 kernel/printk/printk.c:2389 vprintk_emit+0x6c6/0xdf0 kernel/printk/printk.c:1907 vprintk_default+0x28/0x30 kernel/printk/printk.c:1948 vprintk_func+0x7a/0xe7 kernel/printk/printk_safe.c:382 printk+0xa7/0xcf kernel/printk/printk.c:1981 register_console+0x7e7/0xc00 kernel/printk/printk.c:2704 univ8250_console_init+0x3f/0x4b drivers/tty/serial/8250/8250_core.c:685 console_init+0x6e1/0xa54 kernel/printk/printk.c:2788 start_kernel+0x610/0x949 init/main.c:661 x86_64_start_reservations+0x29/0x2b arch/x86/kernel/head64.c:452 x86_64_start_kernel+0x76/0x79 arch/x86/kernel/head64.c:433 secondary_startup_64+0xa5/0xb0 arch/x86/kernel/head_64.S:242 -> #0 (console_owner){-.-.}: lock_acquire+0x1e4/0x540 kernel/locking/lockdep.c:3924 console_lock_spinning_enable kernel/printk/printk.c:1581 [inline] console_unlock+0x5bb/0x10b0 kernel/printk/printk.c:2386 vprintk_emit+0x6c6/0xdf0 kernel/printk/printk.c:1907 vprintk_default+0x28/0x30 kernel/printk/printk.c:1948 vprintk_func+0x7a/0xe7 kernel/printk/printk_safe.c:382 printk+0xa7/0xcf kernel/printk/printk.c:1981 show_pwq kernel/workqueue.c:4449 [inline] show_workqueue_state.cold.48+0xcb8/0x15ec kernel/workqueue.c:4544 wq_watchdog_timer_fn+0x709/0x830 kernel/workqueue.c:5556 call_timer_fn+0x242/0x970 kernel/time/timer.c:1326 expire_timers kernel/time/timer.c:1363 [inline] __run_timers+0x7a6/0xc70 kernel/time/timer.c:1666 run_timer_softirq+0x60/0x70 kernel/time/timer.c:1694 __do_softirq+0x2e8/0xb17 kernel/softirq.c:288 run_ksoftirqd+0x86/0x100 kernel/softirq.c:649 smpboot_thread_fn+0x417/0x870 kernel/smpboot.c:164 kthread+0x345/0x410 kernel/kthread.c:240 ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:412 other info that might help us debug this: Chain exists of: console_owner --> &pool->lock/1 --> &(&pool->lock)->rlock Possible unsafe locking scenario: CPU0 CPU1 ---- ---- lock(&(&pool->lock)->rlock); lock(&pool->lock/1); lock(&(&pool->lock)->rlock); lock(console_owner); *** DEADLOCK *** 4 locks held by ksoftirqd/0/9: #0: 00000000e61dfe19 ((&wq_watchdog_timer)){+.-.}, at: lockdep_copy_map include/linux/lockdep.h:178 [inline] #0: 00000000e61dfe19 ((&wq_watchdog_timer)){+.-.}, at: call_timer_fn+0x1cd/0x970 kernel/time/timer.c:1316 #1: 00000000bcce76c7 (rcu_read_lock_sched){....}, at: show_workqueue_state+0x0/0x1d0 kernel/workqueue.c:4408 #2: 00000000f83bff9c (&(&pool->lock)->rlock){-.-.}, at: show_workqueue_state.cold.48+0xb16/0x15ec kernel/workqueue.c:4542 #3: 00000000990fa23e (console_lock){+.+.}, at: console_trylock_spinning kernel/printk/printk.c:1643 [inline] #3: 00000000990fa23e (console_lock){+.+.}, at: vprintk_emit+0x6ad/0xdf0 kernel/printk/printk.c:1906 stack backtrace: CPU: 0 PID: 9 Comm: ksoftirqd/0 Not tainted 4.18.0-rc3+ #128 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x1c9/0x2b4 lib/dump_stack.c:113 print_circular_bug.isra.36.cold.57+0x1bd/0x27d kernel/locking/lockdep.c:1227 check_prev_add kernel/locking/lockdep.c:1867 [inline] check_prevs_add kernel/locking/lockdep.c:1980 [inline] validate_chain kernel/locking/lockdep.c:2421 [inline] __lock_acquire+0x3449/0x5020 kernel/locking/lockdep.c:3435 lock_acquire+0x1e4/0x540 kernel/locking/lockdep.c:3924 console_lock_spinning_enable kernel/printk/printk.c:1581 [inline] console_unlock+0x5bb/0x10b0 kernel/printk/printk.c:2386 vprintk_emit+0x6c6/0xdf0 kernel/printk/printk.c:1907 vprintk_default+0x28/0x30 kernel/printk/printk.c:1948 vprintk_func+0x7a/0xe7 kernel/printk/printk_safe.c:382 printk+0xa7/0xcf kernel/printk/printk.c:1981 show_pwq kernel/workqueue.c:4449 [inline] show_workqueue_state.cold.48+0xcb8/0x15ec kernel/workqueue.c:4544 wq_watchdog_timer_fn+0x709/0x830 kernel/workqueue.c:5556 call_timer_fn+0x242/0x970 kernel/time/timer.c:1326 expire_timers kernel/time/timer.c:1363 [inline] __run_timers+0x7a6/0xc70 kernel/time/timer.c:1666 run_timer_softirq+0x60/0x70 kernel/time/timer.c:1694 __do_softirq+0x2e8/0xb17 kernel/softirq.c:288 ? check_same_own Lost 11 message(s)! workqueue events_power_efficient: flags=0x80 pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=4/256 pending: gc_worker, do_cache_clean, neigh_periodic_work, neigh_periodic_work workqueue mm_percpu_wq: flags=0x8 pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 pending: vmstat_update workqueue kblockd: flags=0x18 pwq 1: cpus=0 node=0 flags=0x0 nice=-20 active=1/256 pending: blk_mq_requeue_work workqueue ipv6_addrconf: flags=0x40008 pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/1 pending: addrconf_dad_work delayed: addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work FAULT_INJECTION: forcing a failure. name failslab, interval 1, probability 0, space 0, times 1 CPU: 1 PID: 8433 Comm: syz-executor1 Not tainted 4.18.0-rc3+ #128 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x1c9/0x2b4 lib/dump_stack.c:113 fail_dump lib/fault-inject.c:51 [inline] should_fail.cold.4+0xa/0x1a lib/fault-inject.c:149 __should_failslab+0x124/0x180 mm/failslab.c:32 should_failslab+0x9/0x14 mm/slab_common.c:1557 slab_pre_alloc_hook mm/slab.h:423 [inline] slab_alloc_node mm/slab.c:3299 [inline] kmem_cache_alloc_node+0x272/0x780 mm/slab.c:3642 __alloc_skb+0x119/0x790 net/core/skbuff.c:193 FAULT_INJECTION: forcing a failure. name failslab, interval 1, probability 0, space 0, times 1 alloc_skb include/linux/skbuff.h:987 [inline] alloc_skb_with_frags+0x13f/0x770 net/core/skbuff.c:5266 sock_alloc_send_pskb+0x89b/0xb10 net/core/sock.c:2095 sock_alloc_send_skb+0x32/0x40 net/core/sock.c:2112 __ip_append_data.isra.47+0x1600/0x2a90 net/ipv4/ip_output.c:978 ip_make_skb+0x2aa/0x340 net/ipv4/ip_output.c:1502 udp_sendmsg+0x25aa/0x3970 net/ipv4/udp.c:1115 udpv6_sendmsg+0x17b9/0x35f0 net/ipv6/udp.c:1195 inet_sendmsg+0x1a1/0x690 net/ipv4/af_inet.c:798 sock_sendmsg_nosec net/socket.c:641 [inline] sock_sendmsg+0xd5/0x120 net/socket.c:651 ___sys_sendmsg+0x51d/0x930 net/socket.c:2125 __sys_sendmmsg+0x240/0x6f0 net/socket.c:2220 __do_sys_sendmmsg net/socket.c:2249 [inline] __se_sys_sendmmsg net/socket.c:2246 [inline] __x64_sys_sendmmsg+0x9d/0x100 net/socket.c:2246 do_syscall_64+0x1b9/0x820 arch/x86/entry/common.c:290 entry_SYSCALL_64_after_hwframe+0x49/0xbe RIP: 0033:0x455ab9 Code: 1d ba fb ff c3 66 2e 0f 1f 84 00 00 00 00 00 66 90 48 89 f8 48 89 f7 48 89 d6 48 89 ca 4d 89 c2 4d 89 c8 4c 8b 4c 24 08 0f 05 <48> 3d 01 f0 ff ff 0f 83 eb b9 fb ff c3 66 2e 0f 1f 84 00 00 00 00 RSP: 002b:00007f9fb621cc68 EFLAGS: 00000246 ORIG_RAX: 0000000000000133 RAX: ffffffffffffffda RBX: 00007f9fb621d6d4 RCX: 0000000000455ab9 RDX: 04000000000001c1 RSI: 00000000200092c0 RDI: 0000000000000013 RBP: 000000000072bea0 R08: 0000000000000000 R09: 0000000000000000 R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000014 R13: 00000000004c0cd8 R14: 00000000004d0d40 R15: 0000000000000000 CPU: 0 PID: 8444 Comm: syz-executor6 Not tainted 4.18.0-rc3+ #128 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x1c9/0x2b4 lib/dump_stack.c:113 fail_dump lib/fault-inject.c:51 [inline] should_fail.cold.4+0xa/0x1a lib/fault-inject.c:149 __should_failslab+0x124/0x180 mm/failslab.c:32 should_failslab+0x9/0x14 mm/slab_common.c:1557 slab_pre_alloc_hook mm/slab.h:423 [inline] slab_alloc_node mm/slab.c:3299 [inline] kmem_cache_alloc_node+0x272/0x780 mm/slab.c:3642 __alloc_skb+0x119/0x790 net/core/skbuff.c:193 alloc_skb include/linux/skbuff.h:987 [inline] netlink_alloc_large_skb net/netlink/af_netlink.c:1182 [inline] netlink_sendmsg+0xb29/0xfd0 net/netlink/af_netlink.c:1876 sock_sendmsg_nosec net/socket.c:641 [inline] sock_sendmsg+0xd5/0x120 net/socket.c:651 ___sys_sendmsg+0x7fd/0x930 net/socket.c:2125 FAULT_INJECTION: forcing a failure. name failslab, interval 1, probability 0, space 0, times 0 __sys_sendmsg+0x11d/0x290 net/socket.c:2163 __do_sys_sendmsg net/socket.c:2172 [inline] __se_sys_sendmsg net/socket.c:2170 [inline] __x64_sys_sendmsg+0x78/0xb0 net/socket.c:2170 do_syscall_64+0x1b9/0x820 arch/x86/entry/common.c:290 entry_SYSCALL_64_after_hwframe+0x49/0xbe RIP: 0033:0x455ab9 Code: 1d ba fb ff c3 66 2e 0f 1f 84 00 00 00 00 00 66 90 48 89 f8 48 89 f7 48 89 d6 48 89 ca 4d 89 c2 4d 89 c8 4c 8b 4c 24 08 0f 05 <48> 3d 01 f0 ff ff 0f 83 eb b9 fb ff c3 66 2e 0f 1f 84 00 00 00 00 RSP: 002b:00007fdd3b650c68 EFLAGS: 00000246 ORIG_RAX: 000000000000002e RAX: ffffffffffffffda RBX: 00007fdd3b6516d4 RCX: 0000000000455ab9 RDX: 0000000000000000 RSI: 00000000200001c0 RDI: 0000000000000014 RBP: 000000000072bea0 R08: 0000000000000000 R09: 0000000000000000 R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000015 R13: 00000000004c0f2e R14: 00000000004d0fb0 R15: 0000000000000000 CPU: 1 PID: 8466 Comm: syz-executor1 Not tainted 4.18.0-rc3+ #128 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x1c9/0x2b4 lib/dump_stack.c:113 fail_dump lib/fault-inject.c:51 [inline] should_fail.cold.4+0xa/0x1a lib/fault-inject.c:149 __should_failslab+0x124/0x180 mm/failslab.c:32 should_failslab+0x9/0x14 mm/slab_common.c:1557 slab_pre_alloc_hook mm/slab.h:423 [inline] slab_alloc_node mm/slab.c:3299 [inline] kmem_cache_alloc_node_trace+0x26f/0x770 mm/slab.c:3661 __do_kmalloc_node mm/slab.c:3681 [inline] __kmalloc_node_track_caller+0x33/0x70 mm/slab.c:3696 __kmalloc_reserve.isra.40+0x3a/0xe0 net/core/skbuff.c:137 __alloc_skb+0x155/0x790 net/core/skbuff.c:205 alloc_skb include/linux/skbuff.h:987 [inline] alloc_skb_with_frags+0x13f/0x770 net/core/skbuff.c:5266 sock_alloc_send_pskb+0x89b/0xb10 net/core/sock.c:2095 sock_alloc_send_skb+0x32/0x40 net/core/sock.c:2112 __ip_append_data.isra.47+0x1600/0x2a90 net/ipv4/ip_output.c:978 ip_make_skb+0x2aa/0x340 net/ipv4/ip_output.c:1502 udp_sendmsg+0x25aa/0x3970 net/ipv4/udp.c:1115 udpv6_sendmsg+0x17b9/0x35f0 net/ipv6/udp.c:1195 inet_sendmsg+0x1a1/0x690 net/ipv4/af_inet.c:798 sock_sendmsg_nosec net/socket.c:641 [inline] sock_sendmsg+0xd5/0x120 net/socket.c:651 ___sys_sendmsg+0x51d/0x930 net/socket.c:2125 __sys_sendmmsg+0x240/0x6f0 net/socket.c:2220 __do_sys_sendmmsg net/socket.c:2249 [inline] __se_sys_sendmmsg net/socket.c:2246 [inline] __x64_sys_sendmmsg+0x9d/0x100 net/socket.c:2246 do_syscall_64+0x1b9/0x820 arch/x86/entry/common.c:290 entry_SYSCALL_64_after_hwframe+0x49/0xbe RIP: 0033:0x455ab9 Code: 1d ba fb ff c3 66 2e 0f 1f 84 00 00 00 00 00 66 90 48 89 f8 48 89 f7 48 89 d6 48 89 ca 4d 89 c2 4d 89 c8 4c 8b 4c 24 08 0f 05 <48> 3d 01 f0 ff ff 0f 83 eb b9 fb ff c3 66 2e 0f 1f 84 00 00 00 00 RSP: 002b:00007f9fb621cc68 EFLAGS: 00000246 ORIG_RAX: 0000000000000133 RAX: ffffffffffffffda RBX: 00007f9fb621d6d4 RCX: 0000000000455ab9 RDX: 04000000000001c1 RSI: 00000000200092c0 RDI: 0000000000000013 RBP: 000000000072bea0 R08: 0000000000000000 R09: 0000000000000000 R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000014 R13: 00000000004c0cd8 R14: 00000000004d0d40 R15: 0000000000000001