smpboot_thread_fn+0x68b/0xa00 kernel/smpboot.c:164 kthread+0x35a/0x440 kernel/kthread.c:246 ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:352 CPU: 1 PID: 16 Comm: ksoftirqd/1 Not tainted 4.20.0-rc1+ #137 rcu: INFO: rcu_preempt self-detected stall on CPU Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: rcu: 0-...!: (393 ticks this GP) idle=a26/1/0x4000000000000002 softirq=83407/83407 fqs=0 __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x244/0x39d lib/dump_stack.c:113 warn_alloc.cold.116+0xb7/0x1bd mm/page_alloc.c:3423 __alloc_pages_slowpath+0x26e8/0x2e10 mm/page_alloc.c:4302 rcu: (t=10500 jiffies g=22421 q=1619) rcu: rcu_preempt kthread starved for 10500 jiffies! g22421 f0x0 RCU_GP_WAIT_FQS(5) ->state=0x0 ->cpu=0 rcu: RCU grace-period kthread stack dump: rcu_preempt R running task 23000 10 2 0x80000000 Call Trace: context_switch kernel/sched/core.c:2831 [inline] __schedule+0x8cf/0x21d0 kernel/sched/core.c:3472 schedule+0xfe/0x460 kernel/sched/core.c:3516 schedule_timeout+0x140/0x260 kernel/time/timer.c:1804 __alloc_pages_nodemask+0xa80/0xde0 mm/page_alloc.c:4395 rcu_gp_fqs_loop+0x762/0xa80 kernel/rcu/tree.c:1934 __alloc_pages include/linux/gfp.h:473 [inline] __alloc_pages_node include/linux/gfp.h:486 [inline] kmem_getpages mm/slab.c:1409 [inline] cache_grow_begin+0xa5/0x8c0 mm/slab.c:2677 fallback_alloc+0x204/0x2e0 mm/slab.c:3219 ____cache_alloc_node+0x1c7/0x1e0 mm/slab.c:3287 slab_alloc_node mm/slab.c:3327 [inline] kmem_cache_alloc_node+0xe3/0x730 mm/slab.c:3642 __alloc_skb+0x119/0x770 net/core/skbuff.c:193 alloc_skb include/linux/skbuff.h:997 [inline] ndisc_alloc_skb+0x144/0x340 net/ipv6/ndisc.c:403 ndisc_send_rs+0x331/0x6e0 net/ipv6/ndisc.c:669 addrconf_rs_timer+0x314/0x690 net/ipv6/addrconf.c:3836 call_timer_fn+0x272/0x920 kernel/time/timer.c:1326 rcu_gp_kthread+0x341/0xc70 kernel/rcu/tree.c:2090 expire_timers kernel/time/timer.c:1363 [inline] __run_timers+0x7e5/0xc70 kernel/time/timer.c:1682 run_timer_softirq+0x52/0xb0 kernel/time/timer.c:1695 kthread+0x35a/0x440 kernel/kthread.c:246 __do_softirq+0x308/0xb7e kernel/softirq.c:292 ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:352 NMI backtrace for cpu 0 run_ksoftirqd+0x5e/0x100 kernel/softirq.c:654 smpboot_thread_fn+0x68b/0xa00 kernel/smpboot.c:164 kthread+0x35a/0x440 kernel/kthread.c:246 ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:352 warn_alloc_show_mem: 1 callbacks suppressed CPU: 0 PID: 9 Comm: ksoftirqd/0 Not tainted 4.20.0-rc1+ #137 Mem-Info: active_anon:49717 inactive_anon:131 isolated_anon:0 active_file:14 inactive_file:13 isolated_file:0 unevictable:0 dirty:0 writeback:0 unstable:0 slab_reclaimable:8939 slab_unreclaimable:1471592 mapped:40963 shmem:138 pagetables:604 bounce:0 free:13782 free_pcp:155 free_cma:0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: Node 0 active_anon:198868kB inactive_anon:524kB active_file:56kB inactive_file:52kB unevictable:0kB isolated(anon):0kB isolated(file):0kB mapped:163852kB dirty:0kB writeback:0kB shmem:552kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 184320kB writeback_tmp:0kB unstable:0kB all_unreclaimable? yes Node 0 DMA free:15908kB min:164kB low:204kB high:244kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15908kB mlocked:0kB kernel_stack:0kB pagetables:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x244/0x39d lib/dump_stack.c:113 lowmem_reserve[]: 0 2818 6321 6321 Node 0 DMA32 free:25240kB min:30052kB low:37564kB high:45076kB active_anon:4096kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:3129332kB managed:2888772kB mlocked:0kB kernel_stack:0kB pagetables:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB lowmem_reserve[]: 0 0 3503 3503 Node 0 Normal free:13980kB min:37364kB low:46704kB high:56044kB active_anon:194772kB inactive_anon:524kB active_file:56kB inactive_file:52kB unevictable:0kB writepending:0kB present:4718592kB managed:3587820kB mlocked:0kB kernel_stack:5248kB pagetables:2416kB bounce:0kB free_pcp:620kB local_pcp:312kB free_cma:0kB nmi_cpu_backtrace.cold.2+0x5c/0xa1 lib/nmi_backtrace.c:101 lowmem_reserve[]: 0 0 0 0 Node 0 DMA: 1*4kB (U) 0*8kB 0*16kB 1*32kB (U) 2*64kB (U) 1*128kB (U) 1*256kB (U) 0*512kB 1*1024kB (U) 1*2048kB (M) 3*4096kB (M) = 15908kB nmi_trigger_cpumask_backtrace+0x1e8/0x22a lib/nmi_backtrace.c:62 Node 0 DMA32: 4*4kB (UM) 1*8kB (M) 4*16kB (UM) 2*32kB (M) 4*64kB (UM) 4*128kB (M) 3*256kB (M) 4*512kB (UM) 3*1024kB (UM) 3*2048kB (UM) 3*4096kB (M) = 25240kB arch_trigger_cpumask_backtrace+0x14/0x20 arch/x86/kernel/apic/hw_nmi.c:38 Node 0 Normal: 931*4kB (MEH) 422*8kB (MEH) 166*16kB (ME) 60*32kB (UME) 10*64kB (M) 3*128kB (M) 1*256kB (H) 0*512kB 1*1024kB (H) 0*2048kB 0*4096kB = 13980kB trigger_single_cpu_backtrace include/linux/nmi.h:162 [inline] rcu_dump_cpu_stacks+0x16f/0x1bc kernel/rcu/tree.c:1195 Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB print_cpu_stall.cold.67+0x1f3/0x3c7 kernel/rcu/tree.c:1334 Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB 165 total pagecache pages 0 pages in swap cache Swap cache stats: add 0, delete 0, find 0/0 Free swap = 0kB Total swap = 0kB 1965979 pages RAM 0 pages HighMem/MovableOnly 342854 pages reserved 0 pages cma reserved ICMPv6: ndisc: ndisc_alloc_skb failed to allocate an skb check_cpu_stall kernel/rcu/tree.c:1408 [inline] rcu_pending kernel/rcu/tree.c:2961 [inline] rcu_check_callbacks+0xf3b/0x13f0 kernel/rcu/tree.c:2506 ksoftirqd/1: page allocation failure: order:0, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) ksoftirqd/1 cpuset=/ mems_allowed=0 update_process_times+0x2d/0x70 kernel/time/timer.c:1636 tick_sched_handle+0x9f/0x180 kernel/time/tick-sched.c:164 tick_sched_timer+0x45/0x130 kernel/time/tick-sched.c:1274 __run_hrtimer kernel/time/hrtimer.c:1398 [inline] __hrtimer_run_queues+0x41c/0x10d0 kernel/time/hrtimer.c:1460 hrtimer_interrupt+0x313/0x780 kernel/time/hrtimer.c:1518 local_apic_timer_interrupt arch/x86/kernel/apic/apic.c:1034 [inline] smp_apic_timer_interrupt+0x1a1/0x760 arch/x86/kernel/apic/apic.c:1059 apic_timer_interrupt+0xf/0x20 arch/x86/entry/entry_64.S:804 RIP: 0010:arch_local_irq_restore arch/x86/include/asm/paravirt.h:761 [inline] RIP: 0010:slab_alloc_node mm/slab.c:3329 [inline] RIP: 0010:kmem_cache_alloc_node+0x247/0x730 mm/slab.c:3642 Code: 3f 7e 0f 85 32 ff ff ff e8 75 1d 3e ff e9 28 ff ff ff e8 ec 52 c3 ff 48 83 3d f4 9b 8f 07 00 0f 84 33 01 00 00 4c 89 ff 57 9d <0f> 1f 44 00 00 e9 bf fe ff ff 31 d2 be a5 01 00 00 48 c7 c7 92 52 RSP: 0000:ffff8801d9aff3a0 EFLAGS: 00000286 ORIG_RAX: ffffffffffffff13 RAX: 0000000000000000 RBX: 0000000000480020 RCX: 1ffff1003b35e15c RDX: 0000000000000000 RSI: ffff8801d9af0ae8 RDI: 0000000000000286 RBP: ffff8801d9aff410 R08: ffff8801d9af0ae0 R09: 0000000000000002 R10: 0000000000000000 R11: ffff8801d9af0240 R12: ffff8801d9a1c0c0 R13: ffff8801d9a1c0c0 R14: 0000000000000000 R15: 0000000000000286 __alloc_skb+0x119/0x770 net/core/skbuff.c:193 alloc_skb include/linux/skbuff.h:997 [inline] new_skb+0x26/0x1c0 drivers/block/aoe/aoecmd.c:67 aoecmd_cfg_pkts drivers/block/aoe/aoecmd.c:426 [inline] aoecmd_cfg+0x279/0x7f0 drivers/block/aoe/aoecmd.c:1381 discover_timer+0x50/0x60 drivers/block/aoe/aoemain.c:24 call_timer_fn+0x272/0x920 kernel/time/timer.c:1326 expire_timers kernel/time/timer.c:1363 [inline] __run_timers+0x7e5/0xc70 kernel/time/timer.c:1682 run_timer_softirq+0x52/0xb0 kernel/time/timer.c:1695 __do_softirq+0x308/0xb7e kernel/softirq.c:292 run_ksoftirqd+0x5e/0x100 kernel/softirq.c:654 smpboot_thread_fn+0x68b/0xa00 kernel/smpboot.c:164 kthread+0x35a/0x440 kernel/kthread.c:246 ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:352 CPU: 1 PID: 16 Comm: ksoftirqd/1 Not tainted 4.20.0-rc1+ #137 Sending NMI from CPU 0 to CPUs 1: Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 NMI backtrace for cpu 1 CPU: 1 PID: 16 Comm: ksoftirqd/1 Not tainted 4.20.0-rc1+ #137 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 RIP: 0010:format_decode+0x72/0xaf0 lib/vsprintf.c:2004 Code: 24 89 48 c7 45 88 a0 4e ca 87 c7 00 f1 f1 f1 f1 c7 40 04 00 f2 f2 f2 65 48 8b 04 25 28 00 00 00 48 89 45 d0 31 c0 48 89 7d 98 a9 3b b5 f9 48 8b 45 98 48 89 85 70 ff ff ff 4c 89 e8 48 c1 e8 RSP: 0000:ffff8801d9b56290 EFLAGS: 00000046 RAX: 0000000000000000 RBX: dffffc0000000000 RCX: ffffffff87caef7c RDX: 0000000000000100 RSI: ffff8801d9b563b0 RDI: ffffffff882bec65 RBP: ffff8801d9b56338 R08: ffff8801d9b4a440 R09: fffffbfff1613c6c R10: fffffbfff1613c6c R11: ffffffff8b09e360 R12: ffffffff882bec65 R13: ffff8801d9b563b0 R14: ffff8801d9b56310 R15: ffff8801d9b563f0 FS: 0000000000000000(0000) GS:ffff8801daf00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 000000000040f996 CR3: 00000001d8dc9000 CR4: 00000000001406e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 Call Trace: vsnprintf+0x185/0x1b60 lib/vsprintf.c:2248 sprintf+0xb0/0xe0 lib/vsprintf.c:2498 print_time kernel/printk/printk.c:1228 [inline] print_prefix+0x26a/0x3f0 kernel/printk/printk.c:1251 msg_print_text+0xcc/0x1d0 kernel/printk/printk.c:1278 console_unlock+0x4e6/0x1190 kernel/printk/printk.c:2388 vprintk_emit+0x391/0x990 kernel/printk/printk.c:1922 vprintk_default+0x28/0x30 kernel/printk/printk.c:1964 vprintk_func+0x7e/0x181 kernel/printk/printk_safe.c:398 printk+0xa7/0xcf kernel/printk/printk.c:1997 dump_stack_print_info.cold.1+0x1b/0x20 lib/dump_stack.c:56 __dump_stack lib/dump_stack.c:76 [inline] dump_stack+0x23b/0x39d lib/dump_stack.c:113 warn_alloc.cold.116+0xb7/0x1bd mm/page_alloc.c:3423 __alloc_pages_slowpath+0x26e8/0x2e10 mm/page_alloc.c:4302 __alloc_pages_nodemask+0xa80/0xde0 mm/page_alloc.c:4395 __alloc_pages include/linux/gfp.h:473 [inline] __alloc_pages_node include/linux/gfp.h:486 [inline] kmem_getpages mm/slab.c:1409 [inline] cache_grow_begin+0xa5/0x8c0 mm/slab.c:2677 fallback_alloc+0x204/0x2e0 mm/slab.c:3219 ____cache_alloc_node+0x1c7/0x1e0 mm/slab.c:3287 slab_alloc_node mm/slab.c:3327 [inline] kmem_cache_alloc_node+0xe3/0x730 mm/slab.c:3642 __alloc_skb+0x119/0x770 net/core/skbuff.c:193 alloc_skb include/linux/skbuff.h:997 [inline] ndisc_alloc_skb+0x144/0x340 net/ipv6/ndisc.c:403 ndisc_send_rs+0x331/0x6e0 net/ipv6/ndisc.c:669 addrconf_rs_timer+0x314/0x690 net/ipv6/addrconf.c:3836 call_timer_fn+0x272/0x920 kernel/time/timer.c:1326 expire_timers kernel/time/timer.c:1363 [inline] __run_timers+0x7e5/0xc70 kernel/time/timer.c:1682 run_timer_softirq+0x52/0xb0 kernel/time/timer.c:1695 __do_softirq+0x308/0xb7e kernel/softirq.c:292 run_ksoftirqd+0x5e/0x100 kernel/softirq.c:654 smpboot_thread_fn+0x68b/0xa00 kernel/smpboot.c:164 kthread+0x35a/0x440 kernel/kthread.c:246 ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:352 aoe: skb alloc failure Call Trace: __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x244/0x39d lib/dump_stack.c:113 warn_alloc.cold.116+0xb7/0x1bd mm/page_alloc.c:3423 ksoftirqd/0: page allocation failure: order:0, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) ksoftirqd/0 cpuset=/ mems_allowed=0 __alloc_pages_slowpath+0x26e8/0x2e10 mm/page_alloc.c:4302 __alloc_pages_nodemask+0xa80/0xde0 mm/page_alloc.c:4395 __alloc_pages include/linux/gfp.h:473 [inline] __alloc_pages_node include/linux/gfp.h:486 [inline] kmem_getpages mm/slab.c:1409 [inline] cache_grow_begin+0xa5/0x8c0 mm/slab.c:2677 fallback_alloc+0x204/0x2e0 mm/slab.c:3219 ____cache_alloc_node+0x1c7/0x1e0 mm/slab.c:3287 slab_alloc_node mm/slab.c:3327 [inline] kmem_cache_alloc_node+0xe3/0x730 mm/slab.c:3642 __alloc_skb+0x119/0x770 net/core/skbuff.c:193 alloc_skb include/linux/skbuff.h:997 [inline] ndisc_alloc_skb+0x144/0x340 net/ipv6/ndisc.c:403 ndisc_send_rs+0x331/0x6e0 net/ipv6/ndisc.c:669 addrconf_rs_timer+0x314/0x690 net/ipv6/addrconf.c:3836 call_timer_fn+0x272/0x920 kernel/time/timer.c:1326 expire_timers kernel/time/timer.c:1363 [inline] __run_timers+0x7e5/0xc70 kernel/time/timer.c:1682 run_timer_softirq+0x52/0xb0 kernel/time/timer.c:1695 __do_softirq+0x308/0xb7e kernel/softirq.c:292 run_ksoftirqd+0x5e/0x100 kernel/softirq.c:654 smpboot_thread_fn+0x68b/0xa00 kernel/smpboot.c:164 kthread+0x35a/0x440 kernel/kthread.c:246 ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:352 Mem-Info: CPU: 0 PID: 9 Comm: ksoftirqd/0 Not tainted 4.20.0-rc1+ #137 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 active_anon:49717 inactive_anon:131 isolated_anon:0 active_file:14 inactive_file:13 isolated_file:0 unevictable:0 dirty:0 writeback:0 unstable:0 slab_reclaimable:8939 slab_unreclaimable:1471592 mapped:40963 shmem:138 pagetables:604 bounce:0 free:13782 free_pcp:155 free_cma:0 Call Trace: Node 0 active_anon:198868kB inactive_anon:524kB active_file:56kB inactive_file:52kB unevictable:0kB isolated(anon):0kB isolated(file):0kB mapped:163852kB dirty:0kB writeback:0kB shmem:552kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 184320kB writeback_tmp:0kB unstable:0kB all_unreclaimable? yes __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x244/0x39d lib/dump_stack.c:113 Node 0 DMA free:15908kB min:164kB low:204kB high:244kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15908kB mlocked:0kB kernel_stack:0kB pagetables:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB lowmem_reserve[]: 0 2818 6321 6321 warn_alloc.cold.116+0xb7/0x1bd mm/page_alloc.c:3423 Node 0 DMA32 free:25240kB min:30052kB low:37564kB high:45076kB active_anon:4096kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:3129332kB managed:2888772kB mlocked:0kB kernel_stack:0kB pagetables:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB lowmem_reserve[]: 0 0 3503 3503 Node 0 Normal free:13980kB min:37364kB low:46704kB high:56044kB active_anon:194772kB inactive_anon:524kB active_file:56kB inactive_file:52kB unevictable:0kB writepending:0kB present:4718592kB managed:3587820kB mlocked:0kB kernel_stack:5248kB pagetables:2416kB bounce:0kB free_pcp:620kB local_pcp:312kB free_cma:0kB lowmem_reserve[]: 0 0 0 0 Node 0 DMA: 1*4kB (U) 0*8kB 0*16kB 1*32kB (U) 2*64kB (U) 1*128kB (U) 1*256kB (U) 0*512kB 1*1024kB (U) 1*2048kB (M) 3*4096kB (M) = 15908kB __alloc_pages_slowpath+0x26e8/0x2e10 mm/page_alloc.c:4302 Node 0 DMA32: 4*4kB (UM) 1*8kB (M) 4*16kB (UM) 2*32kB (M) 4*64kB (UM) 4*128kB (M) 3*256kB (M) 4*512kB (UM) 3*1024kB (UM) 3*2048kB (UM) 3*4096kB (M) = 25240kB Node 0 Normal: 931*4kB (MEH) 422*8kB (MEH) 166*16kB (ME) 60*32kB (UME) 10*64kB (M) 3*128kB (M) 1*256kB (H) 0*512kB 1*1024kB (H) 0*2048kB 0*4096kB = 13980kB Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB 165 total pagecache pages 0 pages in swap cache Swap cache stats: add 0, delete 0, find 0/0 Free swap = 0kB Total swap = 0kB 1965979 pages RAM 0 pages HighMem/MovableOnly 342854 pages reserved 0 pages cma reserved INFO: NMI handler (nmi_cpu_backtrace_handler) took too long to run: 1.617 msecs ICMPv6: ndisc: ndisc_alloc_skb failed to allocate an skb ksoftirqd/1: page allocation failure: order:0, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) ksoftirqd/1 cpuset=/ mems_allowed=0 __alloc_pages_nodemask+0xa80/0xde0 mm/page_alloc.c:4395 __alloc_pages include/linux/gfp.h:473 [inline] __alloc_pages_node include/linux/gfp.h:486 [inline] kmem_getpages mm/slab.c:1409 [inline] cache_grow_begin+0xa5/0x8c0 mm/slab.c:2677 fallback_alloc+0x204/0x2e0 mm/slab.c:3219 ____cache_alloc_node+0x1c7/0x1e0 mm/slab.c:3287 slab_alloc_node mm/slab.c:3327 [inline] kmem_cache_alloc_node+0xe3/0x730 mm/slab.c:3642 __alloc_skb+0x119/0x770 net/core/skbuff.c:193 alloc_skb include/linux/skbuff.h:997 [inline] new_skb+0x26/0x1c0 drivers/block/aoe/aoecmd.c:67 aoecmd_cfg_pkts drivers/block/aoe/aoecmd.c:426 [inline] aoecmd_cfg+0x279/0x7f0 drivers/block/aoe/aoecmd.c:1381 discover_timer+0x50/0x60 drivers/block/aoe/aoemain.c:24 call_timer_fn+0x272/0x920 kernel/time/timer.c:1326 expire_timers kernel/time/timer.c:1363 [inline] __run_timers+0x7e5/0xc70 kernel/time/timer.c:1682 run_timer_softirq+0x52/0xb0 kernel/time/timer.c:1695 __do_softirq+0x308/0xb7e kernel/softirq.c:292 run_ksoftirqd+0x5e/0x100 kernel/softirq.c:654 smpboot_thread_fn+0x68b/0xa00 kernel/smpboot.c:164 kthread+0x35a/0x440 kernel/kthread.c:246 ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:352 CPU: 1 PID: 16 Comm: ksoftirqd/1 Not tainted 4.20.0-rc1+ #137 aoe: skb alloc failure Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: ksoftirqd/0: page allocation failure: order:0, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x244/0x39d lib/dump_stack.c:113 ksoftirqd/0 cpuset=/ mems_allowed=0 warn_alloc.cold.116+0xb7/0x1bd mm/page_alloc.c:3423 __alloc_pages_slowpath+0x26e8/0x2e10 mm/page_alloc.c:4302 __alloc_pages_nodemask+0xa80/0xde0 mm/page_alloc.c:4395 __alloc_pages include/linux/gfp.h:473 [inline] __alloc_pages_node include/linux/gfp.h:486 [inline] kmem_getpages mm/slab.c:1409 [inline] cache_grow_begin+0xa5/0x8c0 mm/slab.c:2677 fallback_alloc+0x204/0x2e0 mm/slab.c:3219 ____cache_alloc_node+0x1c7/0x1e0 mm/slab.c:3287 slab_alloc_node mm/slab.c:3327 [inline] kmem_cache_alloc_node+0xe3/0x730 mm/slab.c:3642 __alloc_skb+0x119/0x770 net/core/skbuff.c:193 alloc_skb include/linux/skbuff.h:997 [inline] ndisc_alloc_skb+0x144/0x340 net/ipv6/ndisc.c:403 ndisc_send_rs+0x331/0x6e0 net/ipv6/ndisc.c:669 addrconf_rs_timer+0x314/0x690 net/ipv6/addrconf.c:3836 call_timer_fn+0x272/0x920 kernel/time/timer.c:1326 expire_timers kernel/time/timer.c:1363 [inline] __run_timers+0x7e5/0xc70 kernel/time/timer.c:1682 run_timer_softirq+0x52/0xb0 kernel/time/timer.c:1695 __do_softirq+0x308/0xb7e kernel/softirq.c:292 run_ksoftirqd+0x5e/0x100 kernel/softirq.c:654 smpboot_thread_fn+0x68b/0xa00 kernel/smpboot.c:164 kthread+0x35a/0x440 kernel/kthread.c:246 ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:352 warn_alloc_show_mem: 1 callbacks suppressed CPU: 0 PID: 9 Comm: ksoftirqd/0 Not tainted 4.20.0-rc1+ #137 Mem-Info: active_anon:49717 inactive_anon:131 isolated_anon:0 active_file:14 inactive_file:13 isolated_file:0 unevictable:0 dirty:0 writeback:0 unstable:0 slab_reclaimable:8939 slab_unreclaimable:1471592 mapped:40963 shmem:138 pagetables:604 bounce:0 free:13782 free_pcp:155 free_cma:0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: Node 0 active_anon:198868kB inactive_anon:524kB active_file:56kB inactive_file:52kB unevictable:0kB isolated(anon):0kB isolated(file):0kB mapped:163852kB dirty:0kB writeback:0kB shmem:552kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 184320kB writeback_tmp:0kB unstable:0kB all_unreclaimable? yes __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x244/0x39d lib/dump_stack.c:113 Node 0 DMA free:15908kB min:164kB low:204kB high:244kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15908kB mlocked:0kB kernel_stack:0kB pagetables:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB lowmem_reserve[]: 0 2818 6321 6321 warn_alloc.cold.116+0xb7/0x1bd mm/page_alloc.c:3423 Node 0 DMA32 free:25240kB min:30052kB low:37564kB high:45076kB active_anon:4096kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:3129332kB managed:2888772kB mlocked:0kB kernel_stack:0kB pagetables:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB lowmem_reserve[]: 0 0 3503 3503 Node 0 Normal free:13980kB min:37364kB low:46704kB high:56044kB active_anon:194772kB inactive_anon:524kB active_file:56kB inactive_file:52kB unevictable:0kB writepending:0kB present:4718592kB managed:3587820kB mlocked:0kB kernel_stack:5248kB pagetables:2416kB bounce:0kB free_pcp:620kB local_pcp:312kB free_cma:0kB lowmem_reserve[]: 0 0 0 0 Node 0 DMA: 1*4kB (U) 0*8kB 0*16kB 1*32kB (U) 2*64kB (U) 1*128kB (U) 1*256kB (U) 0*512kB 1*1024kB (U) 1*2048kB (M) 3*4096kB (M) = 15908kB __alloc_pages_slowpath+0x26e8/0x2e10 mm/page_alloc.c:4302 Node 0 DMA32: 4*4kB (UM) 1*8kB (M) 4*16kB (UM) 2*32kB (M) 4*64kB (UM) 4*128kB (M) 3*256kB (M) 4*512kB (UM) 3*1024kB (UM) 3*2048kB (UM) 3*4096kB (M) = 25240kB Node 0 Normal: 931*4kB (MEH) 422*8kB (MEH) 166*16kB (ME) 60*32kB (UME) 10*64kB (M) 3*128kB (M) 1*256kB (H) 0*512kB 1*1024kB (H) 0*2048kB 0*4096kB = 13980kB Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB 165 total pagecache pages 0 pages in swap cache Swap cache stats: add 0, delete 0, find 0/0 Free swap = 0kB Total swap = 0kB 1965979 pages RAM 0 pages HighMem/MovableOnly 342854 pages reserved 0 pages cma reserved ICMPv6: ndisc: ndisc_alloc_skb failed to allocate an skb ksoftirqd/1: page allocation failure: order:0, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) ksoftirqd/1 cpuset=/ mems_allowed=0 __alloc_pages_nodemask+0xa80/0xde0 mm/page_alloc.c:4395 __alloc_pages include/linux/gfp.h:473 [inline] __alloc_pages_node include/linux/gfp.h:486 [inline] kmem_getpages mm/slab.c:1409 [inline] cache_grow_begin+0xa5/0x8c0 mm/slab.c:2677 fallback_alloc+0x204/0x2e0 mm/slab.c:3219 ____cache_alloc_node+0x1c7/0x1e0 mm/slab.c:3287 slab_alloc_node mm/slab.c:3327 [inline] kmem_cache_alloc_node+0xe3/0x730 mm/slab.c:3642 __alloc_skb+0x119/0x770 net/core/skbuff.c:193 alloc_skb include/linux/skbuff.h:997 [inline] new_skb+0x26/0x1c0 drivers/block/aoe/aoecmd.c:67 aoecmd_cfg_pkts drivers/block/aoe/aoecmd.c:426 [inline] aoecmd_cfg+0x279/0x7f0 drivers/block/aoe/aoecmd.c:1381 discover_timer+0x50/0x60 drivers/block/aoe/aoemain.c:24 call_timer_fn+0x272/0x920 kernel/time/timer.c:1326 expire_timers kernel/time/timer.c:1363 [inline] __run_timers+0x7e5/0xc70 kernel/time/timer.c:1682 run_timer_softirq+0x52/0xb0 kernel/time/timer.c:1695 __do_softirq+0x308/0xb7e kernel/softirq.c:292 run_ksoftirqd+0x5e/0x100 kernel/softirq.c:654 smpboot_thread_fn+0x68b/0xa00 kernel/smpboot.c:164 kthread+0x35a/0x440 kernel/kthread.c:246 ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:352 CPU: 1 PID: 16 Comm: ksoftirqd/1 Not tainted 4.20.0-rc1+ #137 aoe: skb alloc failure Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 ksoftirqd/0: page allocation failure: order:0, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) Call Trace: ksoftirqd/0 cpuset=/ mems_allowed=0 __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x244/0x39d lib/dump_stack.c:113 warn_alloc.cold.116+0xb7/0x1bd mm/page_alloc.c:3423 __alloc_pages_slowpath+0x26e8/0x2e10 mm/page_alloc.c:4302 __alloc_pages_nodemask+0xa80/0xde0 mm/page_alloc.c:4395 __alloc_pages include/linux/gfp.h:473 [inline] __alloc_pages_node include/linux/gfp.h:486 [inline] kmem_getpages mm/slab.c:1409 [inline] cache_grow_begin+0xa5/0x8c0 mm/slab.c:2677 fallback_alloc+0x204/0x2e0 mm/slab.c:3219 ____cache_alloc_node+0x1c7/0x1e0 mm/slab.c:3287 slab_alloc_node mm/slab.c:3327 [inline] kmem_cache_alloc_node+0xe3/0x730 mm/slab.c:3642 __alloc_skb+0x119/0x770 net/core/skbuff.c:193 alloc_skb include/linux/skbuff.h:997 [inline] ndisc_alloc_skb+0x144/0x340 net/ipv6/ndisc.c:403 ndisc_send_rs+0x331/0x6e0 net/ipv6/ndisc.c:669 addrconf_rs_timer+0x314/0x690 net/ipv6/addrconf.c:3836 call_timer_fn+0x272/0x920 kernel/time/timer.c:1326 expire_timers kernel/time/timer.c:1363 [inline] __run_timers+0x7e5/0xc70 kernel/time/timer.c:1682 run_timer_softirq+0x52/0xb0 kernel/time/timer.c:1695 __do_softirq+0x308/0xb7e kernel/softirq.c:292 run_ksoftirqd+0x5e/0x100 kernel/softirq.c:654 smpboot_thread_fn+0x68b/0xa00 kernel/smpboot.c:164 kthread+0x35a/0x440 kernel/kthread.c:246 ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:352 CPU: 0 PID: 9 Comm: ksoftirqd/0 Not tainted 4.20.0-rc1+ #137 warn_alloc_show_mem: 1 callbacks suppressed Mem-Info: Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 active_anon:49717 inactive_anon:131 isolated_anon:0 active_file:14 inactive_file:13 isolated_file:0 unevictable:0 dirty:0 writeback:0 unstable:0 slab_reclaimable:8939 slab_unreclaimable:1471592 mapped:40963 shmem:138 pagetables:604 bounce:0 free:13782 free_pcp:155 free_cma:0 Call Trace: Node 0 active_anon:198868kB inactive_anon:524kB active_file:56kB inactive_file:52kB unevictable:0kB isolated(anon):0kB isolated(file):0kB mapped:163852kB dirty:0kB writeback:0kB shmem:552kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 184320kB writeback_tmp:0kB unstable:0kB all_unreclaimable? yes __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x244/0x39d lib/dump_stack.c:113 Node 0 DMA free:15908kB min:164kB low:204kB high:244kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15908kB mlocked:0kB kernel_stack:0kB pagetables:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB lowmem_reserve[]: 0 2818 6321 6321 warn_alloc.cold.116+0xb7/0x1bd mm/page_alloc.c:3423 Node 0 DMA32 free:25240kB min:30052kB low:37564kB high:45076kB active_anon:4096kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:3129332kB managed:2888772kB mlocked:0kB kernel_stack:0kB pagetables:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB lowmem_reserve[]: 0 0 3503 3503 Node 0 Normal free:13980kB min:37364kB low:46704kB high:56044kB active_anon:194772kB inactive_anon:524kB active_file:56kB inactive_file:52kB unevictable:0kB writepending:0kB present:4718592kB managed:3587820kB mlocked:0kB kernel_stack:5248kB pagetables:2416kB bounce:0kB free_pcp:620kB local_pcp:312kB free_cma:0kB lowmem_reserve[]: 0 0 0 0 Node 0 DMA: 1*4kB (U) 0*8kB 0*16kB 1*32kB (U) 2*64kB (U) 1*128kB (U) 1*256kB (U) 0*512kB 1*1024kB (U) 1*2048kB (M) 3*4096kB (M) = 15908kB __alloc_pages_slowpath+0x26e8/0x2e10 mm/page_alloc.c:4302 Node 0 DMA32: 4*4kB (UM) 1*8kB (M) 4*16kB (UM) 2*32kB (M) 4*64kB (UM) 4*128kB (M) 3*256kB (M) 4*512kB (UM) 3*1024kB (UM) 3*2048kB (UM) 3*4096kB (M) = 25240kB Node 0 Normal: 931*4kB (MEH) 422*8kB (MEH) 166*16kB (ME) 60*32kB (UME) 10*64kB (M) 3*128kB (M) 1*256kB (H) 0*512kB 1*1024kB (H) 0*2048kB 0*4096kB = 13980kB Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB 165 total pagecache pages 0 pages in swap cache Swap cache stats: add 0, delete 0, find 0/0 Free swap = 0kB Total swap = 0kB 1965979 pages RAM 0 pages HighMem/MovableOnly 342854 pages reserved 0 pages cma reserved ICMPv6: ndisc: ndisc_alloc_skb failed to allocate an skb ksoftirqd/1: page allocation failure: order:0, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) ksoftirqd/1 cpuset=/ mems_allowed=0 __alloc_pages_nodemask+0xa80/0xde0 mm/page_alloc.c:4395 __alloc_pages include/linux/gfp.h:473 [inline] __alloc_pages_node include/linux/gfp.h:486 [inline] kmem_getpages mm/slab.c:1409 [inline] cache_grow_begin+0xa5/0x8c0 mm/slab.c:2677 fallback_alloc+0x204/0x2e0 mm/slab.c:3219 ____cache_alloc_node+0x1c7/0x1e0 mm/slab.c:3287 slab_alloc_node mm/slab.c:3327 [inline] kmem_cache_alloc_node+0xe3/0x730 mm/slab.c:3642 __alloc_skb+0x119/0x770 net/core/skbuff.c:193 alloc_skb include/linux/skbuff.h:997 [inline] new_skb+0x26/0x1c0 drivers/block/aoe/aoecmd.c:67 aoecmd_cfg_pkts drivers/block/aoe/aoecmd.c:426 [inline] aoecmd_cfg+0x279/0x7f0 drivers/block/aoe/aoecmd.c:1381 discover_timer+0x50/0x60 drivers/block/aoe/aoemain.c:24 call_timer_fn+0x272/0x920 kernel/time/timer.c:1326 expire_timers kernel/time/timer.c:1363 [inline] __run_timers+0x7e5/0xc70 kernel/time/timer.c:1682 run_timer_softirq+0x52/0xb0 kernel/time/timer.c:1695 __do_softirq+0x308/0xb7e kernel/softirq.c:292 run_ksoftirqd+0x5e/0x100 kernel/softirq.c:654 smpboot_thread_fn+0x68b/0xa00 kernel/smpboot.c:164 kthread+0x35a/0x440 kernel/kthread.c:246 ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:352 CPU: 1 PID: 16 Comm: ksoftirqd/1 Not tainted 4.20.0-rc1+ #137 aoe: skb alloc failure Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x244/0x39d lib/dump_stack.c:113 warn_alloc.cold.116+0xb7/0x1bd mm/page_alloc.c:3423 ksoftirqd/0: page allocation failure: order:0, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) ksoftirqd/0 cpuset=/ mems_allowed=0 __alloc_pages_slowpath+0x26e8/0x2e10 mm/page_alloc.c:4302 __alloc_pages_nodemask+0xa80/0xde0 mm/page_alloc.c:4395 __alloc_pages include/linux/gfp.h:473 [inline] __alloc_pages_node include/linux/gfp.h:486 [inline] kmem_getpages mm/slab.c:1409 [inline] cache_grow_begin+0xa5/0x8c0 mm/slab.c:2677 fallback_alloc+0x204/0x2e0 mm/slab.c:3219 ____cache_alloc_node+0x1c7/0x1e0 mm/slab.c:3287 slab_alloc_node mm/slab.c:3327 [inline] kmem_cache_alloc_node+0xe3/0x730 mm/slab.c:3642 __alloc_skb+0x119/0x770 net/core/skbuff.c:193 alloc_skb include/linux/skbuff.h:997 [inline] ndisc_alloc_skb+0x144/0x340 net/ipv6/ndisc.c:403 ndisc_send_rs+0x331/0x6e0 net/ipv6/ndisc.c:669 addrconf_rs_timer+0x314/0x690 net/ipv6/addrconf.c:3836 call_timer_fn+0x272/0x920 kernel/time/timer.c:1326 expire_timers kernel/time/timer.c:1363 [inline] __run_timers+0x7e5/0xc70 kernel/time/timer.c:1682 run_timer_softirq+0x52/0xb0 kernel/time/timer.c:1695 __do_softirq+0x308/0xb7e kernel/softirq.c:292 run_ksoftirqd+0x5e/0x100 kernel/softirq.c:654 smpboot_thread_fn+0x68b/0xa00 kernel/smpboot.c:164 kthread+0x35a/0x440 kernel/kthread.c:246 ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:352 CPU: 0 PID: 9 Comm: ksoftirqd/0 Not tainted 4.20.0-rc1+ #137 warn_alloc_show_mem: 1 callbacks suppressed Mem-Info: Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 active_anon:49717 inactive_anon:131 isolated_anon:0 active_file:14 inactive_file:13 isolated_file:0 unevictable:0 dirty:0 writeback:0 unstable:0 slab_reclaimable:8939 slab_unreclaimable:1471592 mapped:40963 shmem:138 pagetables:604 bounce:0 free:13782 free_pcp:155 free_cma:0 Call Trace: Node 0 active_anon:198868kB inactive_anon:524kB active_file:56kB inactive_file:52kB unevictable:0kB isolated(anon):0kB isolated(file):0kB mapped:163852kB dirty:0kB writeback:0kB shmem:552kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 184320kB writeback_tmp:0kB unstable:0kB all_unreclaimable? yes __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x244/0x39d lib/dump_stack.c:113 Node 0 DMA free:15908kB min:164kB low:204kB high:244kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15908kB mlocked:0kB kernel_stack:0kB pagetables:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB lowmem_reserve[]: 0 2818 6321 6321 warn_alloc.cold.116+0xb7/0x1bd mm/page_alloc.c:3423 Node 0 DMA32 free:25240kB min:30052kB low:37564kB high:45076kB active_anon:4096kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:3129332kB managed:2888772kB mlocked:0kB kernel_stack:0kB pagetables:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB lowmem_reserve[]: 0 0 3503 3503 Node 0 Normal free:13980kB min:37364kB low:46704kB high:56044kB active_anon:194772kB inactive_anon:524kB active_file:56kB inactive_file:52kB unevictable:0kB writepending:0kB present:4718592kB managed:3587820kB mlocked:0kB kernel_stack:5248kB pagetables:2416kB bounce:0kB free_pcp:620kB local_pcp:312kB free_cma:0kB lowmem_reserve[]: 0 0 0 0 Node 0 DMA: 1*4kB (U) 0*8kB 0*16kB 1*32kB (U) 2*64kB (U) 1*128kB (U) 1*256kB (U) 0*512kB 1*1024kB (U) 1*2048kB (M) 3*4096kB (M) = 15908kB __alloc_pages_slowpath+0x26e8/0x2e10 mm/page_alloc.c:4302 Node 0 DMA32: 4*4kB (UM) 1*8kB (M) 4*16kB (UM) 2*32kB (M) 4*64kB (UM) 4*128kB (M) 3*256kB (M) 4*512kB (UM) 3*1024kB (UM) 3*2048kB (UM) 3*4096kB (M) = 25240kB Node 0 Normal: 931*4kB (MEH) 422*8kB (MEH) 166*16kB (ME) 60*32kB (UME) 10*64kB (M) 3*128kB (M) 1*256kB (H) 0*512kB 1*1024kB (H) 0*2048kB 0*4096kB = 13980kB Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB 165 total pagecache pages 0 pages in swap cache Swap cache stats: add 0, delete 0, find 0/0 Free swap = 0kB Total swap = 0kB 1965979 pages RAM 0 pages HighMem/MovableOnly 342854 pages reserved 0 pages cma reserved ICMPv6: ndisc: ndisc_alloc_skb failed to allocate an skb ksoftirqd/1: page allocation failure: order:0, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) ksoftirqd/1 cpuset=/ mems_allowed=0 __alloc_pages_nodemask+0xa80/0xde0 mm/page_alloc.c:4395 __alloc_pages include/linux/gfp.h:473 [inline] __alloc_pages_node include/linux/gfp.h:486 [inline] kmem_getpages mm/slab.c:1409 [inline] cache_grow_begin+0xa5/0x8c0 mm/slab.c:2677 fallback_alloc+0x204/0x2e0 mm/slab.c:3219 ____cache_alloc_node+0x1c7/0x1e0 mm/slab.c:3287 slab_alloc_node mm/slab.c:3327 [inline] kmem_cache_alloc_node+0xe3/0x730 mm/slab.c:3642 __alloc_skb+0x119/0x770 net/core/skbuff.c:193 alloc_skb include/linux/skbuff.h:997 [inline] new_skb+0x26/0x1c0 drivers/block/aoe/aoecmd.c:67 aoecmd_cfg_pkts drivers/block/aoe/aoecmd.c:426 [inline] aoecmd_cfg+0x279/0x7f0 drivers/block/aoe/aoecmd.c:1381 discover_timer+0x50/0x60 drivers/block/aoe/aoemain.c:24 call_timer_fn+0x272/0x920 kernel/time/timer.c:1326 expire_timers kernel/time/timer.c:1363 [inline] __run_timers+0x7e5/0xc70 kernel/time/timer.c:1682 run_timer_softirq+0x52/0xb0 kernel/time/timer.c:1695 __do_softirq+0x308/0xb7e kernel/softirq.c:292 run_ksoftirqd+0x5e/0x100 kernel/softirq.c:654 smpboot_thread_fn+0x68b/0xa00 kernel/smpboot.c:164 kthread+0x35a/0x440 kernel/kthread.c:246 ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:352 CPU: 1 PID: 16 Comm: ksoftirqd/1 Not tainted 4.20.0-rc1+ #137 aoe: skb alloc failure Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x244/0x39d lib/dump_stack.c:113 ksoftirqd/0: page allocation failure: order:0, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) warn_alloc.cold.116+0xb7/0x1bd mm/page_alloc.c:3423 ksoftirqd/0 cpuset=/ mems_allowed=0 __alloc_pages_slowpath+0x26e8/0x2e10 mm/page_alloc.c:4302 __alloc_pages_nodemask+0xa80/0xde0 mm/page_alloc.c:4395 __alloc_pages include/linux/gfp.h:473 [inline] __alloc_pages_node include/linux/gfp.h:486 [inline] kmem_getpages mm/slab.c:1409 [inline] cache_grow_begin+0xa5/0x8c0 mm/slab.c:2677 fallback_alloc+0x204/0x2e0 mm/slab.c:3219 ____cache_alloc_node+0x1c7/0x1e0 mm/slab.c:3287 slab_alloc_node mm/slab.c:3327 [inline] kmem_cache_alloc_node+0xe3/0x730 mm/slab.c:3642 __alloc_skb+0x119/0x770 net/core/skbuff.c:193 alloc_skb include/linux/skbuff.h:997 [inline] ndisc_alloc_skb+0x144/0x340 net/ipv6/ndisc.c:403 ndisc_send_rs+0x331/0x6e0 net/ipv6/ndisc.c:669 addrconf_rs_timer+0x314/0x690 net/ipv6/addrconf.c:3836 call_timer_fn+0x272/0x920 kernel/time/timer.c:1326 expire_timers kernel/time/timer.c:1363 [inline] __run_timers+0x7e5/0xc70 kernel/time/timer.c:1682 run_timer_softirq+0x52/0xb0 kernel/time/timer.c:1695 __do_softirq+0x308/0xb7e kernel/softirq.c:292 run_ksoftirqd+0x5e/0x100 kernel/softirq.c:654 smpboot_thread_fn+0x68b/0xa00 kernel/smpboot.c:164 kthread+0x35a/0x440 kernel/kthread.c:246 ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:352 warn_alloc_show_mem: 1 callbacks suppressed CPU: 0 PID: 9 Comm: ksoftirqd/0 Not tainted 4.20.0-rc1+ #137 Mem-Info: active_anon:49717 inactive_anon:131 isolated_anon:0 active_file:14 inactive_file:13 isolated_file:0 unevictable:0 dirty:0 writeback:0 unstable:0 slab_reclaimable:8939 slab_unreclaimable:1471592 mapped:40963 shmem:138 pagetables:604 bounce:0 free:13782 free_pcp:155 free_cma:0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Node 0 active_anon:198868kB inactive_anon:524kB active_file:56kB inactive_file:52kB unevictable:0kB isolated(anon):0kB isolated(file):0kB mapped:163852kB dirty:0kB writeback:0kB shmem:552kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 184320kB writeback_tmp:0kB unstable:0kB all_unreclaimable? yes Call Trace: Node 0 DMA free:15908kB min:164kB low:204kB high:244kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15908kB mlocked:0kB kernel_stack:0kB pagetables:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x244/0x39d lib/dump_stack.c:113 lowmem_reserve[]: 0 2818 6321 6321 Node 0 DMA32 free:25240kB min:30052kB low:37564kB high:45076kB active_anon:4096kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:3129332kB managed:2888772kB mlocked:0kB kernel_stack:0kB pagetables:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB warn_alloc.cold.116+0xb7/0x1bd mm/page_alloc.c:3423 lowmem_reserve[]: 0 0 3503 3503 Node 0 Normal free:13980kB min:37364kB low:46704kB high:56044kB active_anon:194772kB inactive_anon:524kB active_file:56kB inactive_file:52kB unevictable:0kB writepending:0kB present:4718592kB managed:3587820kB mlocked:0kB kernel_stack:5248kB pagetables:2416kB bounce:0kB free_pcp:620kB local_pcp:312kB free_cma:0kB lowmem_reserve[]: 0 0 0 0 Node 0 DMA: 1*4kB (U) 0*8kB 0*16kB 1*32kB (U) 2*64kB (U) 1*128kB (U) 1*256kB (U) 0*512kB 1*1024kB (U) 1*2048kB (M) 3*4096kB (M) = 15908kB Node 0 DMA32: 4*4kB (UM) 1*8kB (M) 4*16kB (UM) 2*32kB (M) 4*64kB (UM) 4*128kB (M) 3*256kB (M) 4*512kB (UM) 3*1024kB (UM) 3*2048kB (UM) 3*4096kB (M) = 25240kB __alloc_pages_slowpath+0x26e8/0x2e10 mm/page_alloc.c:4302 Node 0 Normal: 931*4kB (MEH) 422*8kB (MEH) 166*16kB (ME) 60*32kB (UME) 10*64kB (M) 3*128kB (M) 1*256kB (H) 0*512kB 1*1024kB (H) 0*2048kB 0*4096kB = 13980kB Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB 165 total pagecache pages 0 pages in swap cache Swap cache stats: add 0, delete 0, find 0/0 Free swap = 0kB Total swap = 0kB 1965979 pages RAM 0 pages HighMem/MovableOnly 342854 pages reserved 0 pages cma reserved ICMPv6: ndisc: ndisc_alloc_skb failed to allocate an skb ksoftirqd/1: page allocation failure: order:0, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) ksoftirqd/1 cpuset=/ mems_allowed=0 __alloc_pages_nodemask+0xa80/0xde0 mm/page_alloc.c:4395 __alloc_pages include/linux/gfp.h:473 [inline] __alloc_pages_node include/linux/gfp.h:486 [inline] kmem_getpages mm/slab.c:1409 [inline] cache_grow_begin+0xa5/0x8c0 mm/slab.c:2677 fallback_alloc+0x204/0x2e0 mm/slab.c:3219 ____cache_alloc_node+0x1c7/0x1e0 mm/slab.c:3287 slab_alloc_node mm/slab.c:3327 [inline] kmem_cache_alloc_node+0xe3/0x730 mm/slab.c:3642 __alloc_skb+0x119/0x770 net/core/skbuff.c:193 alloc_skb include/linux/skbuff.h:997 [inline] new_skb+0x26/0x1c0 drivers/block/aoe/aoecmd.c:67 aoecmd_cfg_pkts drivers/block/aoe/aoecmd.c:426 [inline] aoecmd_cfg+0x279/0x7f0 drivers/block/aoe/aoecmd.c:1381 discover_timer+0x50/0x60 drivers/block/aoe/aoemain.c:24 call_timer_fn+0x272/0x920 kernel/time/timer.c:1326 expire_timers kernel/time/timer.c:1363 [inline] __run_timers+0x7e5/0xc70 kernel/time/timer.c:1682 run_timer_softirq+0x52/0xb0 kernel/time/timer.c:1695 __do_softirq+0x308/0xb7e kernel/softirq.c:292 run_ksoftirqd+0x5e/0x100 kernel/softirq.c:654 smpboot_thread_fn+0x68b/0xa00 kernel/smpboot.c:164 kthread+0x35a/0x440 kernel/kthread.c:246 ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:352 CPU: 1 PID: 16 Comm: ksoftirqd/1 Not tainted 4.20.0-rc1+ #137 aoe: skb alloc failure Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x244/0x39d lib/dump_stack.c:113 ksoftirqd/0: page allocation failure: order:0, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) warn_alloc.cold.116+0xb7/0x1bd mm/page_alloc.c:3423 ksoftirqd/0 cpuset=/ mems_allowed=0 __alloc_pages_slowpath+0x26e8/0x2e10 mm/page_alloc.c:4302 __alloc_pages_nodemask+0xa80/0xde0 mm/page_alloc.c:4395 __alloc_pages include/linux/gfp.h:473 [inline] __alloc_pages_node include/linux/gfp.h:486 [inline] kmem_getpages mm/slab.c:1409 [inline] cache_grow_begin+0xa5/0x8c0 mm/slab.c:2677 fallback_alloc+0x204/0x2e0 mm/slab.c:3219 ____cache_alloc_node+0x1c7/0x1e0 mm/slab.c:3287 slab_alloc_node mm/slab.c:3327 [inline] kmem_cache_alloc_node+0xe3/0x730 mm/slab.c:3642 __alloc_skb+0x119/0x770 net/core/skbuff.c:193 alloc_skb include/linux/skbuff.h:997 [inline] ndisc_alloc_skb+0x144/0x340 net/ipv6/ndisc.c:403 ndisc_send_rs+0x331/0x6e0 net/ipv6/ndisc.c:669 addrconf_rs_timer+0x314/0x690 net/ipv6/addrconf.c:3836 call_timer_fn+0x272/0x920 kernel/time/timer.c:1326 expire_timers kernel/time/timer.c:1363 [inline] __run_timers+0x7e5/0xc70 kernel/time/timer.c:1682