rcu: INFO: rcu_preempt self-detected stall on CPU
rcu: 0-...!: (10500 ticks this GP) idle=126/1/0x4000000000000004 softirq=247909/247909 fqs=0
rcu: (t=10500 jiffies g=231721 q=11336)
rcu: rcu_preempt kthread starved for 10500 jiffies! g231721 f0x0 RCU_GP_WAIT_FQS(5) ->state=0x402 ->cpu=0
rcu: RCU grace-period kthread stack dump:
rcu_preempt I29104 10 2 0x80000000
Call Trace:
context_switch kernel/sched/core.c:2826 [inline]
__schedule+0x866/0x1dc0 kernel/sched/core.c:3474
schedule+0x92/0x1c0 kernel/sched/core.c:3518
schedule_timeout+0x4db/0xfc0 kernel/time/timer.c:1804
rcu_gp_kthread+0xd5c/0x2190 kernel/rcu/tree.c:2202
kthread+0x354/0x420 kernel/kthread.c:246
ret_from_fork+0x24/0x30 arch/x86/entry/entry_64.S:415
NMI backtrace for cpu 0
CPU: 0 PID: 11854 Comm: udevd Not tainted 4.19.71 #0
Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011
Call Trace:
__dump_stack lib/dump_stack.c:77 [inline]
dump_stack+0x172/0x1f0 lib/dump_stack.c:113
nmi_cpu_backtrace.cold+0x63/0xa4 lib/nmi_backtrace.c:101
nmi_trigger_cpumask_backtrace+0x1b0/0x1f8 lib/nmi_backtrace.c:62
arch_trigger_cpumask_backtrace+0x14/0x20 arch/x86/kernel/apic/hw_nmi.c:38
trigger_single_cpu_backtrace include/linux/nmi.h:164 [inline]
rcu_dump_cpu_stacks+0x189/0x1d5 kernel/rcu/tree.c:1340
print_cpu_stall kernel/rcu/tree.c:1478 [inline]
check_cpu_stall kernel/rcu/tree.c:1550 [inline]
__rcu_pending kernel/rcu/tree.c:3293 [inline]
rcu_pending kernel/rcu/tree.c:3336 [inline]
rcu_check_callbacks.cold+0x5e3/0xd90 kernel/rcu/tree.c:2682
update_process_times+0x32/0x80 kernel/time/timer.c:1636
tick_sched_handle+0xa2/0x190 kernel/time/tick-sched.c:164
tick_sched_timer+0x47/0x130 kernel/time/tick-sched.c:1274
__run_hrtimer kernel/time/hrtimer.c:1398 [inline]
__hrtimer_run_queues+0x33b/0xdc0 kernel/time/hrtimer.c:1460
hrtimer_interrupt+0x314/0x770 kernel/time/hrtimer.c:1518
local_apic_timer_interrupt arch/x86/kernel/apic/apic.c:1067 [inline]
smp_apic_timer_interrupt+0x111/0x550 arch/x86/kernel/apic/apic.c:1092
apic_timer_interrupt+0xf/0x20 arch/x86/entry/entry_64.S:893
RIP: 0010:__write_once_size include/linux/compiler.h:220 [inline]
RIP: 0010:__list_del include/linux/list.h:106 [inline]
RIP: 0010:__list_del_entry include/linux/list.h:120 [inline]
RIP: 0010:__list_del_entry include/linux/list.h:115 [inline]
RIP: 0010:list_move_tail include/linux/list.h:182 [inline]
RIP: 0010:hhf_dequeue+0x615/0xa00 net/sched/sch_hhf.c:438
Code: 49 8d 7f 08 4c 8b 73 08 48 89 fa 48 c1 ea 03 42 80 3c 22 00 0f 85 9f 03 00 00 4c 89 f2 4d 89 77 08 48 c1 ea 03 42 80 3c 22 00 <0f> 85 7c 03 00 00 4d 89 3e e8 9d dc dc fb 4d 8d b5 58 03 00 00 4c
RSP: 0018:ffff8880ae806e88 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13
RAX: 1ffff1100aed560f RBX: ffff8880576ab078 RCX: ffffffff858e8986
RDX: 1ffff1100aed5622 RSI: ffffffff858e8990 RDI: ffff8880576ab118
RBP: ffff8880ae806ed8 R08: ffff8880a8ad6500 R09: 0000000000000007
R10: ffff8880a8ad6e98 R11: 000000009c12a06d R12: dffffc0000000000
R13: ffff8880576aadc0 R14: ffff8880576ab110 R15: ffff8880576ab110
dequeue_skb net/sched/sch_generic.c:277 [inline]
qdisc_restart net/sched/sch_generic.c:380 [inline]
__qdisc_run+0x1e7/0x1960 net/sched/sch_generic.c:398
__dev_xmit_skb net/core/dev.c:3501 [inline]
__dev_queue_xmit+0x165c/0x2fe0 net/core/dev.c:3811
dev_queue_xmit+0x18/0x20 net/core/dev.c:3876
br_dev_queue_push_xmit+0x405/0x5d0 net/bridge/br_forward.c:56
NF_HOOK include/linux/netfilter.h:289 [inline]
NF_HOOK include/linux/netfilter.h:283 [inline]
br_forward_finish+0xfa/0x400 net/bridge/br_forward.c:69
NF_HOOK include/linux/netfilter.h:289 [inline]
NF_HOOK include/linux/netfilter.h:283 [inline]
__br_forward+0x647/0xb30 net/bridge/br_forward.c:113
deliver_clone+0x61/0xc0 net/bridge/br_forward.c:129
maybe_deliver net/bridge/br_forward.c:184 [inline]
maybe_deliver net/bridge/br_forward.c:172 [inline]
br_flood+0x4da/0x710 net/bridge/br_forward.c:226
br_dev_xmit+0xeeb/0x1490 net/bridge/br_device.c:103
__netdev_start_xmit include/linux/netdevice.h:4303 [inline]
netdev_start_xmit include/linux/netdevice.h:4312 [inline]
xmit_one net/core/dev.c:3257 [inline]
dev_hard_start_xmit+0x1a5/0x980 net/core/dev.c:3273
__dev_queue_xmit+0x2704/0x2fe0 net/core/dev.c:3843
dev_queue_xmit+0x18/0x20 net/core/dev.c:3876
neigh_resolve_output net/core/neighbour.c:1369 [inline]
neigh_resolve_output+0x5b7/0x980 net/core/neighbour.c:1349
neigh_output include/net/neighbour.h:501 [inline]
ip6_finish_output2+0xb7f/0x2550 net/ipv6/ip6_output.c:120
ip6_finish_output+0x574/0xbe0 net/ipv6/ip6_output.c:154
NF_HOOK_COND include/linux/netfilter.h:278 [inline]
ip6_output+0x235/0x7c0 net/ipv6/ip6_output.c:171
dst_output include/net/dst.h:447 [inline]
NF_HOOK include/linux/netfilter.h:289 [inline]
ndisc_send_skb+0xf3b/0x1460 net/ipv6/ndisc.c:491
ndisc_send_rs+0x134/0x6d0 net/ipv6/ndisc.c:685
addrconf_rs_timer+0x30f/0x680 net/ipv6/addrconf.c:3821
call_timer_fn+0x18d/0x720 kernel/time/timer.c:1326
expire_timers kernel/time/timer.c:1363 [inline]
__run_timers kernel/time/timer.c:1682 [inline]
__run_timers kernel/time/timer.c:1650 [inline]
run_timer_softirq+0x652/0x1700 kernel/time/timer.c:1695
__do_softirq+0x25c/0x921 kernel/softirq.c:292
invoke_softirq kernel/softirq.c:372 [inline]
irq_exit+0x180/0x1d0 kernel/softirq.c:412
exiting_irq arch/x86/include/asm/apic.h:536 [inline]
smp_apic_timer_interrupt+0x13b/0x550 arch/x86/kernel/apic/apic.c:1094
apic_timer_interrupt+0xf/0x20 arch/x86/entry/entry_64.S:893
RIP: 0010:preempt_latency_start kernel/sched/core.c:3175 [inline]
RIP: 0010:preempt_latency_start kernel/sched/core.c:3170 [inline]
RIP: 0010:preempt_count_add+0xe6/0x1b0 kernel/sched/core.c:3198
Code: 85 c0 75 39 65 4c 8b 24 25 40 ee 01 00 49 8d bc 24 a8 11 00 00 48 b8 00 00 00 00 00 fc ff df 48 89 fa 48 c1 ea 03 80 3c 02 00 <0f> 85 ab 00 00 00 49 89 9c 24 a8 11 00 00 5b 41 5c 5d c3 48 8b 45
RSP: 0018:ffff888067537a38 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13
RAX: dffffc0000000000 RBX: ffffffff8335bdaa RCX: 0000000000000000
RDX: 1ffff1101515aed5 RSI: ffffffff8335bda0 RDI: ffff8880a8ad76a8
RBP: ffff888067537a48 R08: ffff8880a8ad6500 R09: 0000000000000000
R10: ffffed1013effc20 R11: ffff88809f7fe103 R12: ffff8880a8ad6500
R13: ffff8880801fa000 R14: 0000000000000000 R15: 000000000009e488
percpu_counter_add_batch+0x2a/0x190 lib/percpu_counter.c:86
percpu_counter_add include/linux/percpu_counter.h:55 [inline]
percpu_counter_inc include/linux/percpu_counter.h:179 [inline]
alloc_empty_file+0xd7/0x170 fs/file_table.c:152
path_openat+0xef/0x45e0 fs/namei.c:3523
do_filp_open+0x1a1/0x280 fs/namei.c:3564
do_sys_open+0x3fe/0x550 fs/open.c:1088
__do_sys_open fs/open.c:1106 [inline]
__se_sys_open fs/open.c:1101 [inline]
__x64_sys_open+0x7e/0xc0 fs/open.c:1101
do_syscall_64+0xfd/0x620 arch/x86/entry/common.c:293
entry_SYSCALL_64_after_hwframe+0x49/0xbe
RIP: 0033:0x7f84712c2120
Code: 48 8b 15 1b 4d 2b 00 f7 d8 64 89 02 83 c8 ff c3 90 90 90 90 90 90 90 90 90 90 83 3d d5 a4 2b 00 00 75 10 b8 02 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 31 c3 48 83 ec 08 e8 5e 8c 01 00 48 89 04 24
RSP: 002b:00007ffed63f2858 EFLAGS: 00000246 ORIG_RAX: 0000000000000002
RAX: ffffffffffffffda RBX: 00000000ffffffff RCX: 00007f84712c2120
RDX: 000000000150c6a0 RSI: 0000000000000002 RDI: 000000000041c571
RBP: 0000000000625500 R08: 0000000000001e6e R09: 0000000000001e6e
R10: 00007f8470fe7240 R11: 0000000000000246 R12: 000000000150be80
R13: 0000000000000007 R14: 00000000014bc030 R15: 0000000000000005
rcu: INFO: rcu_sched detected stalls on CPUs/tasks:
rcu: 0-....: (11012 ticks this GP) idle=126/1/0x4000000000000002 softirq=247652/247909 fqs=5192
rcu: (detected by 1, t=10503 jiffies, g=877, q=2)
Sending NMI from CPU 1 to CPUs 0:
INFO: NMI handler (nmi_cpu_backtrace_handler) took too long to run: 1.237 msecs
NMI backtrace for cpu 0
CPU: 0 PID: 11854 Comm: udevd Not tainted 4.19.71 #0
Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011
RIP: 0010:list_add_tail include/linux/list.h:93 [inline]
RIP: 0010:list_move_tail include/linux/list.h:183 [inline]
RIP: 0010:hhf_dequeue+0x61e/0xa00 net/sched/sch_hhf.c:438
Code: 89 fa 48 c1 ea 03 42 80 3c 22 00 0f 85 9f 03 00 00 4c 89 f2 4d 89 77 08 48 c1 ea 03 42 80 3c 22 00 0f 85 7c 03 00 00 4d 89 3e 9d dc dc fb 4d 8d b5 58 03 00 00 4c 89 f0 48 c1 e8 03 42 80 3c
RSP: 0018:ffff8880ae806e88 EFLAGS: 00000246
RAX: 1ffff1100aed560f RBX: ffff8880576ab078 RCX: ffffffff858e8986
RDX: 1ffff1100aed5622 RSI: ffffffff858e8990 RDI: ffff8880576ab118
RBP: ffff8880ae806ed8 R08: ffff8880a8ad6500 R09: 0000000000000007
R10: ffff8880a8ad6e98 R11: 000000009c12a06d R12: dffffc0000000000
R13: ffff8880576aadc0 R14: ffff8880576ab110 R15: ffff8880576ab110
FS: 00007f8471bba7a0(0000) GS:ffff8880ae800000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000000413720 CR3: 0000000065c84000 CR4: 00000000001406f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
dequeue_skb net/sched/sch_generic.c:277 [inline]
qdisc_restart net/sched/sch_generic.c:380 [inline]
__qdisc_run+0x1e7/0x1960 net/sched/sch_generic.c:398
__dev_xmit_skb net/core/dev.c:3501 [inline]
__dev_queue_xmit+0x165c/0x2fe0 net/core/dev.c:3811
dev_queue_xmit+0x18/0x20 net/core/dev.c:3876
br_dev_queue_push_xmit+0x405/0x5d0 net/bridge/br_forward.c:56
NF_HOOK include/linux/netfilter.h:289 [inline]
NF_HOOK include/linux/netfilter.h:283 [inline]
br_forward_finish+0xfa/0x400 net/bridge/br_forward.c:69
NF_HOOK include/linux/netfilter.h:289 [inline]
NF_HOOK include/linux/netfilter.h:283 [inline]
__br_forward+0x647/0xb30 net/bridge/br_forward.c:113
deliver_clone+0x61/0xc0 net/bridge/br_forward.c:129
maybe_deliver net/bridge/br_forward.c:184 [inline]
maybe_deliver net/bridge/br_forward.c:172 [inline]
br_flood+0x4da/0x710 net/bridge/br_forward.c:226
br_dev_xmit+0xeeb/0x1490 net/bridge/br_device.c:103
__netdev_start_xmit include/linux/netdevice.h:4303 [inline]
netdev_start_xmit include/linux/netdevice.h:4312 [inline]
xmit_one net/core/dev.c:3257 [inline]
dev_hard_start_xmit+0x1a5/0x980 net/core/dev.c:3273
__dev_queue_xmit+0x2704/0x2fe0 net/core/dev.c:3843
dev_queue_xmit+0x18/0x20 net/core/dev.c:3876
neigh_resolve_output net/core/neighbour.c:1369 [inline]
neigh_resolve_output+0x5b7/0x980 net/core/neighbour.c:1349
neigh_output include/net/neighbour.h:501 [inline]
ip6_finish_output2+0xb7f/0x2550 net/ipv6/ip6_output.c:120
ip6_finish_output+0x574/0xbe0 net/ipv6/ip6_output.c:154
NF_HOOK_COND include/linux/netfilter.h:278 [inline]
ip6_output+0x235/0x7c0 net/ipv6/ip6_output.c:171
dst_output include/net/dst.h:447 [inline]
NF_HOOK include/linux/netfilter.h:289 [inline]
ndisc_send_skb+0xf3b/0x1460 net/ipv6/ndisc.c:491
ndisc_send_rs+0x134/0x6d0 net/ipv6/ndisc.c:685
addrconf_rs_timer+0x30f/0x680 net/ipv6/addrconf.c:3821
call_timer_fn+0x18d/0x720 kernel/time/timer.c:1326
expire_timers kernel/time/timer.c:1363 [inline]
__run_timers kernel/time/timer.c:1682 [inline]
__run_timers kernel/time/timer.c:1650 [inline]
run_timer_softirq+0x652/0x1700 kernel/time/timer.c:1695
__do_softirq+0x25c/0x921 kernel/softirq.c:292
invoke_softirq kernel/softirq.c:372 [inline]
irq_exit+0x180/0x1d0 kernel/softirq.c:412
exiting_irq arch/x86/include/asm/apic.h:536 [inline]
smp_apic_timer_interrupt+0x13b/0x550 arch/x86/kernel/apic/apic.c:1094
apic_timer_interrupt+0xf/0x20 arch/x86/entry/entry_64.S:893
RIP: 0010:preempt_latency_start kernel/sched/core.c:3175 [inline]
RIP: 0010:preempt_latency_start kernel/sched/core.c:3170 [inline]
RIP: 0010:preempt_count_add+0xe6/0x1b0 kernel/sched/core.c:3198
Code: 85 c0 75 39 65 4c 8b 24 25 40 ee 01 00 49 8d bc 24 a8 11 00 00 48 b8 00 00 00 00 00 fc ff df 48 89 fa 48 c1 ea 03 80 3c 02 00 <0f> 85 ab 00 00 00 49 89 9c 24 a8 11 00 00 5b 41 5c 5d c3 48 8b 45
RSP: 0018:ffff888067537a38 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13
RAX: dffffc0000000000 RBX: ffffffff8335bdaa RCX: 0000000000000000
RDX: 1ffff1101515aed5 RSI: ffffffff8335bda0 RDI: ffff8880a8ad76a8
RBP: ffff888067537a48 R08: ffff8880a8ad6500 R09: 0000000000000000
R10: ffffed1013effc20 R11: ffff88809f7fe103 R12: ffff8880a8ad6500
R13: ffff8880801fa000 R14: 0000000000000000 R15: 000000000009e488
percpu_counter_add_batch+0x2a/0x190 lib/percpu_counter.c:86
percpu_counter_add include/linux/percpu_counter.h:55 [inline]
percpu_counter_inc include/linux/percpu_counter.h:179 [inline]
alloc_empty_file+0xd7/0x170 fs/file_table.c:152
path_openat+0xef/0x45e0 fs/namei.c:3523
do_filp_open+0x1a1/0x280 fs/namei.c:3564
do_sys_open+0x3fe/0x550 fs/open.c:1088
__do_sys_open fs/open.c:1106 [inline]
__se_sys_open fs/open.c:1101 [inline]
__x64_sys_open+0x7e/0xc0 fs/open.c:1101
do_syscall_64+0xfd/0x620 arch/x86/entry/common.c:293
entry_SYSCALL_64_after_hwframe+0x49/0xbe
RIP: 0033:0x7f84712c2120
Code: 48 8b 15 1b 4d 2b 00 f7 d8 64 89 02 83 c8 ff c3 90 90 90 90 90 90 90 90 90 90 83 3d d5 a4 2b 00 00 75 10 b8 02 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 31 c3 48 83 ec 08 e8 5e 8c 01 00 48 89 04 24
RSP: 002b:00007ffed63f2858 EFLAGS: 00000246 ORIG_RAX: 0000000000000002
RAX: ffffffffffffffda RBX: 00000000ffffffff RCX: 00007f84712c2120
RDX: 000000000150c6a0 RSI: 0000000000000002 RDI: 000000000041c571
RBP: 0000000000625500 R08: 0000000000001e6e R09: 0000000000001e6e
R10: 00007f8470fe7240 R11: 0000000000000246 R12: 000000000150be80
R13: 0000000000000007 R14: 00000000014bc030 R15: 0000000000000005