RDX: 0000000000000fff RSI: 00007f3ff3d485a0 RDI: 0000000000000004 RBP: 0000000000000000 R08: 0000000000d97260 R09: 0000000000000000 R10: 7269762f73656369 R11: 0000000000000293 R12: 000000000065e420 R13: 00007f3ff25149c0 R14: 00007f3ff55b9040 R15: 0000000000000003 CPU: 1 PID: 16 Comm: ksoftirqd/1 Not tainted 4.20.0-rc1+ #137 rcu: INFO: rcu_preempt detected stalls on CPUs/tasks: Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: rcu: (detected by 0, t=10504 jiffies, g=71941, q=6) __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x244/0x39d lib/dump_stack.c:113 rcu: All QSes seen, last rcu_preempt kthread activity 10479 (4295003885-4294993406), jiffies_till_next_fqs=1, root ->qsmask 0x0 rsyslogd R running task 11424 5520 1 0x80000008 warn_alloc.cold.116+0xb7/0x1bd mm/page_alloc.c:3423 Call Trace: sched_show_task.cold.84+0x2b6/0x30a kernel/sched/core.c:5293 __alloc_pages_slowpath+0x26e8/0x2e10 mm/page_alloc.c:4302 print_other_cpu_stall.cold.66+0x7dd/0x8bb kernel/rcu/tree.c:1286 check_cpu_stall kernel/rcu/tree.c:1415 [inline] rcu_pending kernel/rcu/tree.c:2961 [inline] rcu_check_callbacks+0xfd7/0x13f0 kernel/rcu/tree.c:2506 __alloc_pages_nodemask+0xa80/0xde0 mm/page_alloc.c:4395 __alloc_pages include/linux/gfp.h:473 [inline] __alloc_pages_node include/linux/gfp.h:486 [inline] kmem_getpages mm/slab.c:1409 [inline] cache_grow_begin+0xa5/0x8c0 mm/slab.c:2677 fallback_alloc+0x204/0x2e0 mm/slab.c:3219 ____cache_alloc_node+0x1c7/0x1e0 mm/slab.c:3287 slab_alloc_node mm/slab.c:3327 [inline] kmem_cache_alloc_node+0xe3/0x730 mm/slab.c:3642 __alloc_skb+0x119/0x770 net/core/skbuff.c:193 alloc_skb include/linux/skbuff.h:997 [inline] ndisc_alloc_skb+0x144/0x340 net/ipv6/ndisc.c:403 ndisc_send_rs+0x331/0x6e0 net/ipv6/ndisc.c:669 addrconf_rs_timer+0x314/0x690 net/ipv6/addrconf.c:3836 update_process_times+0x2d/0x70 kernel/time/timer.c:1636 tick_sched_handle+0x9f/0x180 kernel/time/tick-sched.c:164 call_timer_fn+0x272/0x920 kernel/time/timer.c:1326 tick_sched_timer+0x45/0x130 kernel/time/tick-sched.c:1274 __run_hrtimer kernel/time/hrtimer.c:1398 [inline] __hrtimer_run_queues+0x41c/0x10d0 kernel/time/hrtimer.c:1460 expire_timers kernel/time/timer.c:1363 [inline] __run_timers+0x7e5/0xc70 kernel/time/timer.c:1682 hrtimer_interrupt+0x313/0x780 kernel/time/hrtimer.c:1518 local_apic_timer_interrupt arch/x86/kernel/apic/apic.c:1034 [inline] smp_apic_timer_interrupt+0x1a1/0x760 arch/x86/kernel/apic/apic.c:1059 apic_timer_interrupt+0xf/0x20 arch/x86/entry/entry_64.S:804 RIP: 0010:arch_local_irq_restore arch/x86/include/asm/paravirt.h:761 [inline] RIP: 0010:slab_alloc_node mm/slab.c:3329 [inline] RIP: 0010:kmem_cache_alloc_node+0x247/0x730 mm/slab.c:3642 Code: 3f 7e 0f 85 32 ff ff ff e8 75 1d 3e ff e9 28 ff ff ff e8 ec 52 c3 ff 48 83 3d f4 9b 8f 07 00 0f 84 33 01 00 00 4c 89 ff 57 9d <0f> 1f 44 00 00 e9 bf fe ff ff 31 d2 be a5 01 00 00 48 c7 c7 92 52 RSP: 0000:ffff8801dae073d8 EFLAGS: 00000286 ORIG_RAX: ffffffffffffff13 RAX: 0000000000000000 RBX: 0000000000480020 RCX: 1ffff10037751936 run_timer_softirq+0x52/0xb0 kernel/time/timer.c:1695 RDX: 0000000000000000 RSI: ffff8801bba8c9b8 RDI: 0000000000000286 RBP: ffff8801dae07448 R08: ffff8801bba8c9b0 R09: 0000000000000002 R10: 0000000000000000 R11: ffff8801bba8c0c0 R12: ffff8801d9bee600 __do_softirq+0x308/0xb7e kernel/softirq.c:292 R13: ffff8801d9bee600 R14: 0000000000000000 R15: 0000000000000286 __alloc_skb+0x119/0x770 net/core/skbuff.c:193 run_ksoftirqd+0x5e/0x100 kernel/softirq.c:654 smpboot_thread_fn+0x68b/0xa00 kernel/smpboot.c:164 alloc_skb include/linux/skbuff.h:997 [inline] new_skb+0x26/0x1c0 drivers/block/aoe/aoecmd.c:67 aoecmd_cfg_pkts drivers/block/aoe/aoecmd.c:426 [inline] aoecmd_cfg+0x279/0x7f0 drivers/block/aoe/aoecmd.c:1381 kthread+0x35a/0x440 kernel/kthread.c:246 discover_timer+0x50/0x60 drivers/block/aoe/aoemain.c:24 call_timer_fn+0x272/0x920 kernel/time/timer.c:1326 ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:352 warn_alloc_show_mem: 1 callbacks suppressed Mem-Info: active_anon:58077 inactive_anon:3958 isolated_anon:0 active_file:12 inactive_file:14 isolated_file:0 unevictable:0 dirty:0 writeback:0 unstable:0 slab_reclaimable:11484 slab_unreclaimable:1464692 mapped:16386 shmem:4014 pagetables:510 bounce:0 free:13720 free_pcp:483 free_cma:0 Node 0 active_anon:232308kB inactive_anon:15832kB active_file:48kB inactive_file:56kB unevictable:0kB isolated(anon):0kB isolated(file):0kB mapped:65544kB dirty:0kB writeback:0kB shmem:16056kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 217088kB writeback_tmp:0kB unstable:0kB all_unreclaimable? yes Node 0 DMA free:15908kB min:164kB low:204kB high:244kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15908kB mlocked:0kB kernel_stack:0kB pagetables:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB lowmem_reserve[]: 0 2818 6321 6321 Node 0 DMA32 free:25040kB min:30052kB low:37564kB high:45076kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:3129332kB managed:2888772kB mlocked:0kB kernel_stack:0kB pagetables:0kB bounce:0kB free_pcp:248kB local_pcp:0kB free_cma:0kB lowmem_reserve[]: 0 0 3503 3503 Node 0 Normal free:13932kB min:37364kB low:46704kB high:56044kB active_anon:232308kB inactive_anon:15832kB active_file:48kB inactive_file:56kB unevictable:0kB writepending:0kB present:4718592kB managed:3587820kB mlocked:0kB kernel_stack:5184kB pagetables:2040kB bounce:0kB free_pcp:1684kB local_pcp:356kB free_cma:0kB lowmem_reserve[]: 0 0 0 0 Node 0 DMA: 1*4kB (U) 0*8kB 0*16kB 1*32kB (U) 2*64kB (U) 1*128kB (U) 1*256kB (U) 0*512kB 1*1024kB (U) 1*2048kB (M) 3*4096kB (M) = 15908kB Node 0 DMA32: 4*4kB (UM) 2*8kB (UM) 3*16kB (M) 2*32kB (M) 3*64kB (M) 5*128kB (UM) 4*256kB (UM) 3*512kB (M) 3*1024kB (UM) 3*2048kB (UM) 3*4096kB (M) = 25040kB Node 0 Normal: 1007*4kB (UMEH) 460*8kB (ME) 157*16kB (ME) 51*32kB (UM) 13*64kB (UM) 0*128kB 1*256kB (U) 0*512kB 1*1024kB (U) 0*2048kB 0*4096kB = 13964kB expire_timers kernel/time/timer.c:1363 [inline] __run_timers+0x7e5/0xc70 kernel/time/timer.c:1682 Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB 4040 total pagecache pages 0 pages in swap cache Swap cache stats: add 0, delete 0, find 0/0 Free swap = 0kB Total swap = 0kB 1965979 pages RAM 0 pages HighMem/MovableOnly 342854 pages reserved 0 pages cma reserved ICMPv6: ndisc: ndisc_alloc_skb failed to allocate an skb ksoftirqd/1: page allocation failure: order:0, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) ksoftirqd/1 cpuset=/ mems_allowed=0 CPU: 1 PID: 16 Comm: ksoftirqd/1 Not tainted 4.20.0-rc1+ #137 run_timer_softirq+0x52/0xb0 kernel/time/timer.c:1695 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: __do_softirq+0x308/0xb7e kernel/softirq.c:292 __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x244/0x39d lib/dump_stack.c:113 warn_alloc.cold.116+0xb7/0x1bd mm/page_alloc.c:3423 __alloc_pages_slowpath+0x26e8/0x2e10 mm/page_alloc.c:4302 invoke_softirq kernel/softirq.c:373 [inline] irq_exit+0x17f/0x1c0 kernel/softirq.c:413 exiting_irq arch/x86/include/asm/apic.h:536 [inline] smp_apic_timer_interrupt+0x1cb/0x760 arch/x86/kernel/apic/apic.c:1061 apic_timer_interrupt+0xf/0x20 arch/x86/entry/entry_64.S:804 RIP: 0010:arch_local_irq_enable arch/x86/include/asm/paravirt.h:771 [inline] RIP: 0010:__raw_spin_unlock_irq include/linux/spinlock_api_smp.h:168 [inline] RIP: 0010:_raw_spin_unlock_irq+0x56/0x80 kernel/locking/spinlock.c:192 Code: 51 89 48 b8 00 00 00 00 00 fc ff df 48 89 fa 48 c1 ea 03 80 3c 02 00 75 32 48 83 3d 93 cf 82 01 00 74 1f fb 66 0f 1f 44 00 00 01 00 00 00 e8 b0 87 86 f9 65 8b 05 89 c1 32 78 85 c0 74 05 5b RSP: 0000:ffff8801bb976610 EFLAGS: 00000286 ORIG_RAX: ffffffffffffff13 RAX: dffffc0000000000 RBX: ffff88021fffdb80 RCX: 1ffff10037751931 RDX: 1ffffffff12a3f78 RSI: 0000000000000000 RDI: ffffffff8951fbc0 RBP: ffff8801bb976618 R08: ffff8801bba8c988 R09: 0000000000000006 R10: 0000000000000000 R11: ffff8801bba8c0c0 R12: ffffffff8951fbb0 R13: ffff8801bb9767f8 R14: 0000000000000001 R15: ffff8801bb976ec8 __alloc_pages_nodemask+0xa80/0xde0 mm/page_alloc.c:4395 spin_unlock_irq include/linux/spinlock.h:379 [inline] shrink_inactive_list+0xa3a/0x1c60 mm/vmscan.c:1984 __alloc_pages include/linux/gfp.h:473 [inline] __alloc_pages_node include/linux/gfp.h:486 [inline] kmem_getpages mm/slab.c:1409 [inline] cache_grow_begin+0xa5/0x8c0 mm/slab.c:2677 shrink_list mm/vmscan.c:2275 [inline] shrink_node_memcg+0x78b/0x18f0 mm/vmscan.c:2540 fallback_alloc+0x204/0x2e0 mm/slab.c:3219 ____cache_alloc_node+0x1c7/0x1e0 mm/slab.c:3287 slab_alloc_node mm/slab.c:3327 [inline] kmem_cache_alloc_node+0xe3/0x730 mm/slab.c:3642 __alloc_skb+0x119/0x770 net/core/skbuff.c:193 shrink_node+0x3bc/0x16b0 mm/vmscan.c:2755 alloc_skb include/linux/skbuff.h:997 [inline] ndisc_alloc_skb+0x144/0x340 net/ipv6/ndisc.c:403 ndisc_send_rs+0x331/0x6e0 net/ipv6/ndisc.c:669 addrconf_rs_timer+0x314/0x690 net/ipv6/addrconf.c:3836 shrink_zones mm/vmscan.c:2987 [inline] do_try_to_free_pages+0x3e7/0x1290 mm/vmscan.c:3049 call_timer_fn+0x272/0x920 kernel/time/timer.c:1326 try_to_free_pages+0x4d0/0xb90 mm/vmscan.c:3264 __perform_reclaim mm/page_alloc.c:3773 [inline] __alloc_pages_direct_reclaim mm/page_alloc.c:3795 [inline] __alloc_pages_slowpath+0xa14/0x2e10 mm/page_alloc.c:4196 expire_timers kernel/time/timer.c:1363 [inline] __run_timers+0x7e5/0xc70 kernel/time/timer.c:1682 __alloc_pages_nodemask+0xa80/0xde0 mm/page_alloc.c:4395 run_timer_softirq+0x52/0xb0 kernel/time/timer.c:1695 __do_softirq+0x308/0xb7e kernel/softirq.c:292 __alloc_pages include/linux/gfp.h:473 [inline] __alloc_pages_node include/linux/gfp.h:486 [inline] kmem_getpages mm/slab.c:1409 [inline] cache_grow_begin+0xa5/0x8c0 mm/slab.c:2677 fallback_alloc+0x204/0x2e0 mm/slab.c:3219 run_ksoftirqd+0x5e/0x100 kernel/softirq.c:654 ____cache_alloc_node+0x1c7/0x1e0 mm/slab.c:3287 smpboot_thread_fn+0x68b/0xa00 kernel/smpboot.c:164 __do_cache_alloc mm/slab.c:3356 [inline] slab_alloc mm/slab.c:3384 [inline] kmem_cache_alloc_trace+0x214/0x750 mm/slab.c:3618 kmalloc include/linux/slab.h:546 [inline] syslog_print kernel/printk/printk.c:1301 [inline] do_syslog+0xb9b/0x1690 kernel/printk/printk.c:1469 kthread+0x35a/0x440 kernel/kthread.c:246 kmsg_read+0x8f/0xc0 fs/proc/kmsg.c:40 ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:352 proc_reg_read+0x2a3/0x3d0 fs/proc/inode.c:231 ICMPv6: ndisc: ndisc_alloc_skb failed to allocate an skb ksoftirqd/1: page allocation failure: order:0, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) ksoftirqd/1 cpuset=/ mems_allowed=0 __vfs_read+0x117/0x9b0 fs/read_write.c:416 CPU: 1 PID: 16 Comm: ksoftirqd/1 Not tainted 4.20.0-rc1+ #137 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x244/0x39d lib/dump_stack.c:113 warn_alloc.cold.116+0xb7/0x1bd mm/page_alloc.c:3423 vfs_read+0x17f/0x3c0 fs/read_write.c:452 ksys_read+0x101/0x260 fs/read_write.c:578 __alloc_pages_slowpath+0x26e8/0x2e10 mm/page_alloc.c:4302 __do_sys_read fs/read_write.c:588 [inline] __se_sys_read fs/read_write.c:586 [inline] __x64_sys_read+0x73/0xb0 fs/read_write.c:586 do_syscall_64+0x1b9/0x820 arch/x86/entry/common.c:290 entry_SYSCALL_64_after_hwframe+0x49/0xbe RIP: 0033:0x7f3ff4f741fd Code: Bad RIP value. RSP: 002b:00007f3ff2513e30 EFLAGS: 00000293 ORIG_RAX: 0000000000000000 RAX: ffffffffffffffda RBX: 0000000000dac670 RCX: 00007f3ff4f741fd RDX: 0000000000000fff RSI: 00007f3ff3d485a0 RDI: 0000000000000004 RBP: 0000000000000000 R08: 0000000000d97260 R09: 0000000000000000 R10: 7269762f73656369 R11: 0000000000000293 R12: 000000000065e420 R13: 00007f3ff25149c0 R14: 00007f3ff55b9040 R15: 0000000000000003 rcu: rcu_preempt kthread starved for 10479 jiffies! g71941 f0x2 RCU_GP_WAIT_FQS(5) ->state=0x0 ->cpu=0 rcu: RCU grace-period kthread stack dump: rcu_preempt R running task 21384 10 2 0x80000000 Call Trace: context_switch kernel/sched/core.c:2831 [inline] __schedule+0x8cf/0x21d0 kernel/sched/core.c:3472 __alloc_pages_nodemask+0xa80/0xde0 mm/page_alloc.c:4395 schedule+0xfe/0x460 kernel/sched/core.c:3516 __alloc_pages include/linux/gfp.h:473 [inline] __alloc_pages_node include/linux/gfp.h:486 [inline] kmem_getpages mm/slab.c:1409 [inline] cache_grow_begin+0xa5/0x8c0 mm/slab.c:2677 fallback_alloc+0x204/0x2e0 mm/slab.c:3219 schedule_timeout+0x140/0x260 kernel/time/timer.c:1804 ____cache_alloc_node+0x1c7/0x1e0 mm/slab.c:3287 slab_alloc_node mm/slab.c:3327 [inline] kmem_cache_alloc_node+0xe3/0x730 mm/slab.c:3642 rcu_gp_fqs_loop+0x762/0xa80 kernel/rcu/tree.c:1934 __alloc_skb+0x119/0x770 net/core/skbuff.c:193 alloc_skb include/linux/skbuff.h:997 [inline] ndisc_alloc_skb+0x144/0x340 net/ipv6/ndisc.c:403 ndisc_send_rs+0x331/0x6e0 net/ipv6/ndisc.c:669 addrconf_rs_timer+0x314/0x690 net/ipv6/addrconf.c:3836 call_timer_fn+0x272/0x920 kernel/time/timer.c:1326 rcu_gp_kthread+0x341/0xc70 kernel/rcu/tree.c:2090 expire_timers kernel/time/timer.c:1363 [inline] __run_timers+0x7e5/0xc70 kernel/time/timer.c:1682 run_timer_softirq+0x52/0xb0 kernel/time/timer.c:1695 __do_softirq+0x308/0xb7e kernel/softirq.c:292 kthread+0x35a/0x440 kernel/kthread.c:246 ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:352 run_ksoftirqd+0x5e/0x100 kernel/softirq.c:654 aoe: skb alloc failure smpboot_thread_fn+0x68b/0xa00 kernel/smpboot.c:164 rsyslogd: page allocation failure: order:0, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) rsyslogd cpuset=/ mems_allowed=0 kthread+0x35a/0x440 kernel/kthread.c:246 ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:352 warn_alloc_show_mem: 1 callbacks suppressed CPU: 0 PID: 5520 Comm: rsyslogd Not tainted 4.20.0-rc1+ #137 Mem-Info: active_anon:58077 inactive_anon:3958 isolated_anon:0 active_file:12 inactive_file:14 isolated_file:0 unevictable:0 dirty:0 writeback:0 unstable:0 slab_reclaimable:11484 slab_unreclaimable:1464692 mapped:16386 shmem:4014 pagetables:510 bounce:0 free:13720 free_pcp:483 free_cma:0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Node 0 active_anon:232308kB inactive_anon:15832kB active_file:48kB inactive_file:56kB unevictable:0kB isolated(anon):0kB isolated(file):0kB mapped:65544kB dirty:0kB writeback:0kB shmem:16056kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 217088kB writeback_tmp:0kB unstable:0kB all_unreclaimable? yes Call Trace: Node 0 DMA free:15908kB min:164kB low:204kB high:244kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15908kB mlocked:0kB kernel_stack:0kB pagetables:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB lowmem_reserve[]: 0 2818 6321 6321 __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x244/0x39d lib/dump_stack.c:113 Node 0 DMA32 free:25040kB min:30052kB low:37564kB high:45076kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:3129332kB managed:2888772kB mlocked:0kB kernel_stack:0kB pagetables:0kB bounce:0kB free_pcp:248kB local_pcp:0kB free_cma:0kB lowmem_reserve[]: 0 0 3503 3503 warn_alloc.cold.116+0xb7/0x1bd mm/page_alloc.c:3423 Node 0 Normal free:13932kB min:37364kB low:46704kB high:56044kB active_anon:232308kB inactive_anon:15832kB active_file:48kB inactive_file:56kB unevictable:0kB writepending:0kB present:4718592kB managed:3587820kB mlocked:0kB kernel_stack:5184kB pagetables:2040kB bounce:0kB free_pcp:1684kB local_pcp:356kB free_cma:0kB lowmem_reserve[]: 0 0 0 0 Node 0 DMA: 1*4kB (U) 0*8kB 0*16kB 1*32kB (U) 2*64kB (U) 1*128kB (U) 1*256kB (U) 0*512kB 1*1024kB (U) 1*2048kB (M) 3*4096kB (M) = 15908kB Node 0 DMA32: 4*4kB (UM) 2*8kB (UM) 3*16kB (M) 2*32kB (M) 3*64kB (M) 5*128kB (UM) 4*256kB (UM) 3*512kB (M) 3*1024kB (UM) 3*2048kB (UM) 3*4096kB (M) = 25040kB Node 0 Normal: 1007*4kB (UMEH) 460*8kB (ME) 157*16kB (ME) 51*32kB (UM) 13*64kB (UM) 0*128kB 1*256kB (U) 0*512kB 1*1024kB (U) 0*2048kB 0*4096kB = 13964kB __alloc_pages_slowpath+0x26e8/0x2e10 mm/page_alloc.c:4302 Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB 4040 total pagecache pages 0 pages in swap cache Swap cache stats: add 0, delete 0, find 0/0 Free swap = 0kB Total swap = 0kB 1965979 pages RAM 0 pages HighMem/MovableOnly 342854 pages reserved 0 pages cma reserved ICMPv6: ndisc: ndisc_alloc_skb failed to allocate an skb ksoftirqd/1: page allocation failure: order:0, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) ksoftirqd/1 cpuset=/ mems_allowed=0 __alloc_pages_nodemask+0xa80/0xde0 mm/page_alloc.c:4395 __alloc_pages include/linux/gfp.h:473 [inline] __alloc_pages_node include/linux/gfp.h:486 [inline] kmem_getpages mm/slab.c:1409 [inline] cache_grow_begin+0xa5/0x8c0 mm/slab.c:2677 fallback_alloc+0x204/0x2e0 mm/slab.c:3219 ____cache_alloc_node+0x1c7/0x1e0 mm/slab.c:3287 slab_alloc_node mm/slab.c:3327 [inline] kmem_cache_alloc_node+0xe3/0x730 mm/slab.c:3642 __alloc_skb+0x119/0x770 net/core/skbuff.c:193 alloc_skb include/linux/skbuff.h:997 [inline] new_skb+0x26/0x1c0 drivers/block/aoe/aoecmd.c:67 aoecmd_cfg_pkts drivers/block/aoe/aoecmd.c:426 [inline] aoecmd_cfg+0x279/0x7f0 drivers/block/aoe/aoecmd.c:1381 discover_timer+0x50/0x60 drivers/block/aoe/aoemain.c:24 call_timer_fn+0x272/0x920 kernel/time/timer.c:1326 expire_timers kernel/time/timer.c:1363 [inline] __run_timers+0x7e5/0xc70 kernel/time/timer.c:1682 run_timer_softirq+0x52/0xb0 kernel/time/timer.c:1695 __do_softirq+0x308/0xb7e kernel/softirq.c:292 invoke_softirq kernel/softirq.c:373 [inline] irq_exit+0x17f/0x1c0 kernel/softirq.c:413 exiting_irq arch/x86/include/asm/apic.h:536 [inline] smp_apic_timer_interrupt+0x1cb/0x760 arch/x86/kernel/apic/apic.c:1061 apic_timer_interrupt+0xf/0x20 arch/x86/entry/entry_64.S:804 RIP: 0010:arch_local_irq_enable arch/x86/include/asm/paravirt.h:771 [inline] RIP: 0010:__raw_spin_unlock_irq include/linux/spinlock_api_smp.h:168 [inline] RIP: 0010:_raw_spin_unlock_irq+0x56/0x80 kernel/locking/spinlock.c:192 Code: 51 89 48 b8 00 00 00 00 00 fc ff df 48 89 fa 48 c1 ea 03 80 3c 02 00 75 32 48 83 3d 93 cf 82 01 00 74 1f fb 66 0f 1f 44 00 00 01 00 00 00 e8 b0 87 86 f9 65 8b 05 89 c1 32 78 85 c0 74 05 5b RSP: 0000:ffff8801bb976610 EFLAGS: 00000286 ORIG_RAX: ffffffffffffff13 RAX: dffffc0000000000 RBX: ffff88021fffdb80 RCX: 1ffff10037751931 RDX: 1ffffffff12a3f78 RSI: 0000000000000000 RDI: ffffffff8951fbc0 RBP: ffff8801bb976618 R08: ffff8801bba8c988 R09: 0000000000000006 R10: 0000000000000000 R11: ffff8801bba8c0c0 R12: ffffffff8951fbb0 R13: ffff8801bb9767f8 R14: 0000000000000001 R15: ffff8801bb976ec8 spin_unlock_irq include/linux/spinlock.h:379 [inline] shrink_inactive_list+0xa3a/0x1c60 mm/vmscan.c:1984 shrink_list mm/vmscan.c:2275 [inline] shrink_node_memcg+0x78b/0x18f0 mm/vmscan.c:2540 shrink_node+0x3bc/0x16b0 mm/vmscan.c:2755 shrink_zones mm/vmscan.c:2987 [inline] do_try_to_free_pages+0x3e7/0x1290 mm/vmscan.c:3049 try_to_free_pages+0x4d0/0xb90 mm/vmscan.c:3264 __perform_reclaim mm/page_alloc.c:3773 [inline] __alloc_pages_direct_reclaim mm/page_alloc.c:3795 [inline] __alloc_pages_slowpath+0xa14/0x2e10 mm/page_alloc.c:4196 __alloc_pages_nodemask+0xa80/0xde0 mm/page_alloc.c:4395 __alloc_pages include/linux/gfp.h:473 [inline] __alloc_pages_node include/linux/gfp.h:486 [inline] kmem_getpages mm/slab.c:1409 [inline] cache_grow_begin+0xa5/0x8c0 mm/slab.c:2677 fallback_alloc+0x204/0x2e0 mm/slab.c:3219 ____cache_alloc_node+0x1c7/0x1e0 mm/slab.c:3287 __do_cache_alloc mm/slab.c:3356 [inline] slab_alloc mm/slab.c:3384 [inline] kmem_cache_alloc_trace+0x214/0x750 mm/slab.c:3618 kmalloc include/linux/slab.h:546 [inline] syslog_print kernel/printk/printk.c:1301 [inline] do_syslog+0xb9b/0x1690 kernel/printk/printk.c:1469 kmsg_read+0x8f/0xc0 fs/proc/kmsg.c:40 proc_reg_read+0x2a3/0x3d0 fs/proc/inode.c:231 __vfs_read+0x117/0x9b0 fs/read_write.c:416 vfs_read+0x17f/0x3c0 fs/read_write.c:452 ksys_read+0x101/0x260 fs/read_write.c:578 __do_sys_read fs/read_write.c:588 [inline] __se_sys_read fs/read_write.c:586 [inline] __x64_sys_read+0x73/0xb0 fs/read_write.c:586 do_syscall_64+0x1b9/0x820 arch/x86/entry/common.c:290 entry_SYSCALL_64_after_hwframe+0x49/0xbe RIP: 0033:0x7f3ff4f741fd Code: Bad RIP value. RSP: 002b:00007f3ff2513e30 EFLAGS: 00000293 ORIG_RAX: 0000000000000000 RAX: ffffffffffffffda RBX: 0000000000dac670 RCX: 00007f3ff4f741fd RDX: 0000000000000fff RSI: 00007f3ff3d485a0 RDI: 0000000000000004 RBP: 0000000000000000 R08: 0000000000d97260 R09: 0000000000000000 R10: 7269762f73656369 R11: 0000000000000293 R12: 000000000065e420 R13: 00007f3ff25149c0 R14: 00007f3ff55b9040 R15: 0000000000000003 CPU: 1 PID: 16 Comm: ksoftirqd/1 Not tainted 4.20.0-rc1+ #137 aoe: skb alloc failure Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x244/0x39d lib/dump_stack.c:113 rsyslogd: page allocation failure: order:0, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) warn_alloc.cold.116+0xb7/0x1bd mm/page_alloc.c:3423 rsyslogd cpuset=/ mems_allowed=0 __alloc_pages_slowpath+0x26e8/0x2e10 mm/page_alloc.c:4302 __alloc_pages_nodemask+0xa80/0xde0 mm/page_alloc.c:4395 __alloc_pages include/linux/gfp.h:473 [inline] __alloc_pages_node include/linux/gfp.h:486 [inline] kmem_getpages mm/slab.c:1409 [inline] cache_grow_begin+0xa5/0x8c0 mm/slab.c:2677 fallback_alloc+0x204/0x2e0 mm/slab.c:3219 ____cache_alloc_node+0x1c7/0x1e0 mm/slab.c:3287 slab_alloc_node mm/slab.c:3327 [inline] kmem_cache_alloc_node+0xe3/0x730 mm/slab.c:3642 __alloc_skb+0x119/0x770 net/core/skbuff.c:193 alloc_skb include/linux/skbuff.h:997 [inline] ndisc_alloc_skb+0x144/0x340 net/ipv6/ndisc.c:403 ndisc_send_rs+0x331/0x6e0 net/ipv6/ndisc.c:669 addrconf_rs_timer+0x314/0x690 net/ipv6/addrconf.c:3836 call_timer_fn+0x272/0x920 kernel/time/timer.c:1326 expire_timers kernel/time/timer.c:1363 [inline] __run_timers+0x7e5/0xc70 kernel/time/timer.c:1682 run_timer_softirq+0x52/0xb0 kernel/time/timer.c:1695 __do_softirq+0x308/0xb7e kernel/softirq.c:292 run_ksoftirqd+0x5e/0x100 kernel/softirq.c:654 smpboot_thread_fn+0x68b/0xa00 kernel/smpboot.c:164 kthread+0x35a/0x440 kernel/kthread.c:246 ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:352 CPU: 0 PID: 5520 Comm: rsyslogd Not tainted 4.20.0-rc1+ #137 warn_alloc_show_mem: 1 callbacks suppressed Mem-Info: Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 active_anon:58077 inactive_anon:3958 isolated_anon:0 active_file:12 inactive_file:14 isolated_file:0 unevictable:0 dirty:0 writeback:0 unstable:0 slab_reclaimable:11484 slab_unreclaimable:1464692 mapped:16386 shmem:4014 pagetables:510 bounce:0 free:13720 free_pcp:483 free_cma:0 Call Trace: Node 0 active_anon:232308kB inactive_anon:15832kB active_file:48kB inactive_file:56kB unevictable:0kB isolated(anon):0kB isolated(file):0kB mapped:65544kB dirty:0kB writeback:0kB shmem:16056kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 217088kB writeback_tmp:0kB unstable:0kB all_unreclaimable? yes Node 0 DMA free:15908kB min:164kB low:204kB high:244kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15908kB mlocked:0kB kernel_stack:0kB pagetables:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x244/0x39d lib/dump_stack.c:113 lowmem_reserve[]: 0 2818 6321 6321 Node 0 DMA32 free:25040kB min:30052kB low:37564kB high:45076kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:3129332kB managed:2888772kB mlocked:0kB kernel_stack:0kB pagetables:0kB bounce:0kB free_pcp:248kB local_pcp:0kB free_cma:0kB warn_alloc.cold.116+0xb7/0x1bd mm/page_alloc.c:3423 lowmem_reserve[]: 0 0 3503 3503 Node 0 Normal free:13932kB min:37364kB low:46704kB high:56044kB active_anon:232308kB inactive_anon:15832kB active_file:48kB inactive_file:56kB unevictable:0kB writepending:0kB present:4718592kB managed:3587820kB mlocked:0kB kernel_stack:5184kB pagetables:2040kB bounce:0kB free_pcp:1684kB local_pcp:356kB free_cma:0kB lowmem_reserve[]: 0 0 0 0 Node 0 DMA: 1*4kB (U) 0*8kB 0*16kB 1*32kB (U) 2*64kB (U) 1*128kB (U) 1*256kB (U) 0*512kB 1*1024kB (U) 1*2048kB (M) 3*4096kB (M) = 15908kB Node 0 DMA32: 4*4kB (UM) 2*8kB (UM) 3*16kB (M) 2*32kB (M) 3*64kB (M) 5*128kB (UM) 4*256kB (UM) 3*512kB (M) 3*1024kB (UM) 3*2048kB (UM) 3*4096kB (M) = 25040kB __alloc_pages_slowpath+0x26e8/0x2e10 mm/page_alloc.c:4302 Node 0 Normal: 1007*4kB (UMEH) 460*8kB (ME) 157*16kB (ME) 51*32kB (UM) 13*64kB (UM) 0*128kB 1*256kB (U) 0*512kB 1*1024kB (U) 0*2048kB 0*4096kB = 13964kB Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB 4040 total pagecache pages 0 pages in swap cache Swap cache stats: add 0, delete 0, find 0/0 Free swap = 0kB Total swap = 0kB 1965979 pages RAM 0 pages HighMem/MovableOnly 342854 pages reserved 0 pages cma reserved ICMPv6: ndisc: ndisc_alloc_skb failed to allocate an skb ksoftirqd/1: page allocation failure: order:0, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) ksoftirqd/1 cpuset=/ mems_allowed=0 __alloc_pages_nodemask+0xa80/0xde0 mm/page_alloc.c:4395 __alloc_pages include/linux/gfp.h:473 [inline] __alloc_pages_node include/linux/gfp.h:486 [inline] kmem_getpages mm/slab.c:1409 [inline] cache_grow_begin+0xa5/0x8c0 mm/slab.c:2677 fallback_alloc+0x204/0x2e0 mm/slab.c:3219 ____cache_alloc_node+0x1c7/0x1e0 mm/slab.c:3287 slab_alloc_node mm/slab.c:3327 [inline] kmem_cache_alloc_node+0xe3/0x730 mm/slab.c:3642 __alloc_skb+0x119/0x770 net/core/skbuff.c:193 alloc_skb include/linux/skbuff.h:997 [inline] new_skb+0x26/0x1c0 drivers/block/aoe/aoecmd.c:67 aoecmd_cfg_pkts drivers/block/aoe/aoecmd.c:426 [inline] aoecmd_cfg+0x279/0x7f0 drivers/block/aoe/aoecmd.c:1381 discover_timer+0x50/0x60 drivers/block/aoe/aoemain.c:24 call_timer_fn+0x272/0x920 kernel/time/timer.c:1326 expire_timers kernel/time/timer.c:1363 [inline] __run_timers+0x7e5/0xc70 kernel/time/timer.c:1682 run_timer_softirq+0x52/0xb0 kernel/time/timer.c:1695 __do_softirq+0x308/0xb7e kernel/softirq.c:292 invoke_softirq kernel/softirq.c:373 [inline] irq_exit+0x17f/0x1c0 kernel/softirq.c:413 exiting_irq arch/x86/include/asm/apic.h:536 [inline] smp_apic_timer_interrupt+0x1cb/0x760 arch/x86/kernel/apic/apic.c:1061 apic_timer_interrupt+0xf/0x20 arch/x86/entry/entry_64.S:804 RIP: 0010:arch_local_irq_enable arch/x86/include/asm/paravirt.h:771 [inline] RIP: 0010:__raw_spin_unlock_irq include/linux/spinlock_api_smp.h:168 [inline] RIP: 0010:_raw_spin_unlock_irq+0x56/0x80 kernel/locking/spinlock.c:192 Code: 51 89 48 b8 00 00 00 00 00 fc ff df 48 89 fa 48 c1 ea 03 80 3c 02 00 75 32 48 83 3d 93 cf 82 01 00 74 1f fb 66 0f 1f 44 00 00 01 00 00 00 e8 b0 87 86 f9 65 8b 05 89 c1 32 78 85 c0 74 05 5b RSP: 0000:ffff8801bb976610 EFLAGS: 00000286 ORIG_RAX: ffffffffffffff13 RAX: dffffc0000000000 RBX: ffff88021fffdb80 RCX: 1ffff10037751931 RDX: 1ffffffff12a3f78 RSI: 0000000000000000 RDI: ffffffff8951fbc0 RBP: ffff8801bb976618 R08: ffff8801bba8c988 R09: 0000000000000006 R10: 0000000000000000 R11: ffff8801bba8c0c0 R12: ffffffff8951fbb0 R13: ffff8801bb9767f8 R14: 0000000000000001 R15: ffff8801bb976ec8 spin_unlock_irq include/linux/spinlock.h:379 [inline] shrink_inactive_list+0xa3a/0x1c60 mm/vmscan.c:1984 shrink_list mm/vmscan.c:2275 [inline] shrink_node_memcg+0x78b/0x18f0 mm/vmscan.c:2540 shrink_node+0x3bc/0x16b0 mm/vmscan.c:2755 shrink_zones mm/vmscan.c:2987 [inline] do_try_to_free_pages+0x3e7/0x1290 mm/vmscan.c:3049 try_to_free_pages+0x4d0/0xb90 mm/vmscan.c:3264 __perform_reclaim mm/page_alloc.c:3773 [inline] __alloc_pages_direct_reclaim mm/page_alloc.c:3795 [inline] __alloc_pages_slowpath+0xa14/0x2e10 mm/page_alloc.c:4196 __alloc_pages_nodemask+0xa80/0xde0 mm/page_alloc.c:4395 __alloc_pages include/linux/gfp.h:473 [inline] __alloc_pages_node include/linux/gfp.h:486 [inline] kmem_getpages mm/slab.c:1409 [inline] cache_grow_begin+0xa5/0x8c0 mm/slab.c:2677 fallback_alloc+0x204/0x2e0 mm/slab.c:3219 ____cache_alloc_node+0x1c7/0x1e0 mm/slab.c:3287 __do_cache_alloc mm/slab.c:3356 [inline] slab_alloc mm/slab.c:3384 [inline] kmem_cache_alloc_trace+0x214/0x750 mm/slab.c:3618 kmalloc include/linux/slab.h:546 [inline] syslog_print kernel/printk/printk.c:1301 [inline] do_syslog+0xb9b/0x1690 kernel/printk/printk.c:1469 kmsg_read+0x8f/0xc0 fs/proc/kmsg.c:40 proc_reg_read+0x2a3/0x3d0 fs/proc/inode.c:231 __vfs_read+0x117/0x9b0 fs/read_write.c:416 vfs_read+0x17f/0x3c0 fs/read_write.c:452 ksys_read+0x101/0x260 fs/read_write.c:578 __do_sys_read fs/read_write.c:588 [inline] __se_sys_read fs/read_write.c:586 [inline] __x64_sys_read+0x73/0xb0 fs/read_write.c:586 do_syscall_64+0x1b9/0x820 arch/x86/entry/common.c:290 entry_SYSCALL_64_after_hwframe+0x49/0xbe RIP: 0033:0x7f3ff4f741fd Code: Bad RIP value. RSP: 002b:00007f3ff2513e30 EFLAGS: 00000293 ORIG_RAX: 0000000000000000 RAX: ffffffffffffffda RBX: 0000000000dac670 RCX: 00007f3ff4f741fd RDX: 0000000000000fff RSI: 00007f3ff3d485a0 RDI: 0000000000000004 RBP: 0000000000000000 R08: 0000000000d97260 R09: 0000000000000000 R10: 7269762f73656369 R11: 0000000000000293 R12: 000000000065e420 R13: 00007f3ff25149c0 R14: 00007f3ff55b9040 R15: 0000000000000003 CPU: 1 PID: 16 Comm: ksoftirqd/1 Not tainted 4.20.0-rc1+ #137 aoe: skb alloc failure Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x244/0x39d lib/dump_stack.c:113 rsyslogd: page allocation failure: order:0, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) warn_alloc.cold.116+0xb7/0x1bd mm/page_alloc.c:3423 rsyslogd cpuset=/ mems_allowed=0 __alloc_pages_slowpath+0x26e8/0x2e10 mm/page_alloc.c:4302 __alloc_pages_nodemask+0xa80/0xde0 mm/page_alloc.c:4395 __alloc_pages include/linux/gfp.h:473 [inline] __alloc_pages_node include/linux/gfp.h:486 [inline] kmem_getpages mm/slab.c:1409 [inline] cache_grow_begin+0xa5/0x8c0 mm/slab.c:2677 fallback_alloc+0x204/0x2e0 mm/slab.c:3219 ____cache_alloc_node+0x1c7/0x1e0 mm/slab.c:3287 slab_alloc_node mm/slab.c:3327 [inline] kmem_cache_alloc_node+0xe3/0x730 mm/slab.c:3642 __alloc_skb+0x119/0x770 net/core/skbuff.c:193 alloc_skb include/linux/skbuff.h:997 [inline] ndisc_alloc_skb+0x144/0x340 net/ipv6/ndisc.c:403 ndisc_send_rs+0x331/0x6e0 net/ipv6/ndisc.c:669 addrconf_rs_timer+0x314/0x690 net/ipv6/addrconf.c:3836 call_timer_fn+0x272/0x920 kernel/time/timer.c:1326 expire_timers kernel/time/timer.c:1363 [inline] __run_timers+0x7e5/0xc70 kernel/time/timer.c:1682 run_timer_softirq+0x52/0xb0 kernel/time/timer.c:1695 __do_softirq+0x308/0xb7e kernel/softirq.c:292 run_ksoftirqd+0x5e/0x100 kernel/softirq.c:654 smpboot_thread_fn+0x68b/0xa00 kernel/smpboot.c:164 kthread+0x35a/0x440 kernel/kthread.c:246 ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:352 warn_alloc_show_mem: 1 callbacks suppressed CPU: 0 PID: 5520 Comm: rsyslogd Not tainted 4.20.0-rc1+ #137 Mem-Info: active_anon:58077 inactive_anon:3958 isolated_anon:0 active_file:12 inactive_file:14 isolated_file:0 unevictable:0 dirty:0 writeback:0 unstable:0 slab_reclaimable:11484 slab_unreclaimable:1464692 mapped:16386 shmem:4014 pagetables:510 bounce:0 free:13720 free_pcp:483 free_cma:0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: Node 0 active_anon:232308kB inactive_anon:15832kB active_file:48kB inactive_file:56kB unevictable:0kB isolated(anon):0kB isolated(file):0kB mapped:65544kB dirty:0kB writeback:0kB shmem:16056kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 217088kB writeback_tmp:0kB unstable:0kB all_unreclaimable? yes Node 0 DMA free:15908kB min:164kB low:204kB high:244kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15908kB mlocked:0kB kernel_stack:0kB pagetables:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x244/0x39d lib/dump_stack.c:113 lowmem_reserve[]: 0 2818 6321 6321 Node 0 DMA32 free:25040kB min:30052kB low:37564kB high:45076kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:3129332kB managed:2888772kB mlocked:0kB kernel_stack:0kB pagetables:0kB bounce:0kB free_pcp:248kB local_pcp:0kB free_cma:0kB warn_alloc.cold.116+0xb7/0x1bd mm/page_alloc.c:3423 lowmem_reserve[]: 0 0 3503 3503 Node 0 Normal free:13932kB min:37364kB low:46704kB high:56044kB active_anon:232308kB inactive_anon:15832kB active_file:48kB inactive_file:56kB unevictable:0kB writepending:0kB present:4718592kB managed:3587820kB mlocked:0kB kernel_stack:5184kB pagetables:2040kB bounce:0kB free_pcp:1684kB local_pcp:356kB free_cma:0kB lowmem_reserve[]: 0 0 0 0 Node 0 DMA: 1*4kB (U) 0*8kB 0*16kB 1*32kB (U) 2*64kB (U) 1*128kB (U) 1*256kB (U) 0*512kB 1*1024kB (U) 1*2048kB (M) 3*4096kB (M) = 15908kB Node 0 DMA32: 4*4kB (UM) 2*8kB (UM) 3*16kB (M) 2*32kB (M) 3*64kB (M) 5*128kB (UM) 4*256kB (UM) 3*512kB (M) 3*1024kB (UM) 3*2048kB (UM) 3*4096kB (M) = 25040kB __alloc_pages_slowpath+0x26e8/0x2e10 mm/page_alloc.c:4302 Node 0 Normal: 1007*4kB (UMEH) 460*8kB (ME) 157*16kB (ME) 51*32kB (UM) 13*64kB (UM) 0*128kB 1*256kB (U) 0*512kB 1*1024kB (U) 0*2048kB 0*4096kB = 13964kB Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB 4040 total pagecache pages 0 pages in swap cache Swap cache stats: add 0, delete 0, find 0/0 Free swap = 0kB Total swap = 0kB 1965979 pages RAM 0 pages HighMem/MovableOnly 342854 pages reserved 0 pages cma reserved ICMPv6: ndisc: ndisc_alloc_skb failed to allocate an skb ksoftirqd/1: page allocation failure: order:0, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) ksoftirqd/1 cpuset=/ mems_allowed=0 __alloc_pages_nodemask+0xa80/0xde0 mm/page_alloc.c:4395 __alloc_pages include/linux/gfp.h:473 [inline] __alloc_pages_node include/linux/gfp.h:486 [inline] kmem_getpages mm/slab.c:1409 [inline] cache_grow_begin+0xa5/0x8c0 mm/slab.c:2677 fallback_alloc+0x204/0x2e0 mm/slab.c:3219 ____cache_alloc_node+0x1c7/0x1e0 mm/slab.c:3287 slab_alloc_node mm/slab.c:3327 [inline] kmem_cache_alloc_node+0xe3/0x730 mm/slab.c:3642 __alloc_skb+0x119/0x770 net/core/skbuff.c:193 alloc_skb include/linux/skbuff.h:997 [inline] new_skb+0x26/0x1c0 drivers/block/aoe/aoecmd.c:67 aoecmd_cfg_pkts drivers/block/aoe/aoecmd.c:426 [inline] aoecmd_cfg+0x279/0x7f0 drivers/block/aoe/aoecmd.c:1381 discover_timer+0x50/0x60 drivers/block/aoe/aoemain.c:24 call_timer_fn+0x272/0x920 kernel/time/timer.c:1326 expire_timers kernel/time/timer.c:1363 [inline] __run_timers+0x7e5/0xc70 kernel/time/timer.c:1682 run_timer_softirq+0x52/0xb0 kernel/time/timer.c:1695 __do_softirq+0x308/0xb7e kernel/softirq.c:292 invoke_softirq kernel/softirq.c:373 [inline] irq_exit+0x17f/0x1c0 kernel/softirq.c:413 exiting_irq arch/x86/include/asm/apic.h:536 [inline] smp_apic_timer_interrupt+0x1cb/0x760 arch/x86/kernel/apic/apic.c:1061 apic_timer_interrupt+0xf/0x20 arch/x86/entry/entry_64.S:804 RIP: 0010:arch_local_irq_enable arch/x86/include/asm/paravirt.h:771 [inline] RIP: 0010:__raw_spin_unlock_irq include/linux/spinlock_api_smp.h:168 [inline] RIP: 0010:_raw_spin_unlock_irq+0x56/0x80 kernel/locking/spinlock.c:192 Code: 51 89 48 b8 00 00 00 00 00 fc ff df 48 89 fa 48 c1 ea 03 80 3c 02 00 75 32 48 83 3d 93 cf 82 01 00 74 1f fb 66 0f 1f 44 00 00 01 00 00 00 e8 b0 87 86 f9 65 8b 05 89 c1 32 78 85 c0 74 05 5b RSP: 0000:ffff8801bb976610 EFLAGS: 00000286 ORIG_RAX: ffffffffffffff13 RAX: dffffc0000000000 RBX: ffff88021fffdb80 RCX: 1ffff10037751931 RDX: 1ffffffff12a3f78 RSI: 0000000000000000 RDI: ffffffff8951fbc0 RBP: ffff8801bb976618 R08: ffff8801bba8c988 R09: 0000000000000006 R10: 0000000000000000 R11: ffff8801bba8c0c0 R12: ffffffff8951fbb0 R13: ffff8801bb9767f8 R14: 0000000000000001 R15: ffff8801bb976ec8 spin_unlock_irq include/linux/spinlock.h:379 [inline] shrink_inactive_list+0xa3a/0x1c60 mm/vmscan.c:1984 shrink_list mm/vmscan.c:2275 [inline] shrink_node_memcg+0x78b/0x18f0 mm/vmscan.c:2540 shrink_node+0x3bc/0x16b0 mm/vmscan.c:2755 shrink_zones mm/vmscan.c:2987 [inline] do_try_to_free_pages+0x3e7/0x1290 mm/vmscan.c:3049 try_to_free_pages+0x4d0/0xb90 mm/vmscan.c:3264 __perform_reclaim mm/page_alloc.c:3773 [inline] __alloc_pages_direct_reclaim mm/page_alloc.c:3795 [inline] __alloc_pages_slowpath+0xa14/0x2e10 mm/page_alloc.c:4196 __alloc_pages_nodemask+0xa80/0xde0 mm/page_alloc.c:4395 __alloc_pages include/linux/gfp.h:473 [inline] __alloc_pages_node include/linux/gfp.h:486 [inline] kmem_getpages mm/slab.c:1409 [inline] cache_grow_begin+0xa5/0x8c0 mm/slab.c:2677 fallback_alloc+0x204/0x2e0 mm/slab.c:3219 ____cache_alloc_node+0x1c7/0x1e0 mm/slab.c:3287 __do_cache_alloc mm/slab.c:3356 [inline] slab_alloc mm/slab.c:3384 [inline] kmem_cache_alloc_trace+0x214/0x750 mm/slab.c:3618 kmalloc include/linux/slab.h:546 [inline] syslog_print kernel/printk/printk.c:1301 [inline] do_syslog+0xb9b/0x1690 kernel/printk/printk.c:1469 kmsg_read+0x8f/0xc0 fs/proc/kmsg.c:40 proc_reg_read+0x2a3/0x3d0 fs/proc/inode.c:231 __vfs_read+0x117/0x9b0 fs/read_write.c:416 vfs_read+0x17f/0x3c0 fs/read_write.c:452 ksys_read+0x101/0x260 fs/read_write.c:578 __do_sys_read fs/read_write.c:588 [inline] __se_sys_read fs/read_write.c:586 [inline] __x64_sys_read+0x73/0xb0 fs/read_write.c:586 do_syscall_64+0x1b9/0x820 arch/x86/entry/common.c:290 entry_SYSCALL_64_after_hwframe+0x49/0xbe RIP: 0033:0x7f3ff4f741fd Code: Bad RIP value. RSP: 002b:00007f3ff2513e30 EFLAGS: 00000293 ORIG_RAX: 0000000000000000 RAX: ffffffffffffffda RBX: 0000000000dac670 RCX: 00007f3ff4f741fd RDX: 0000000000000fff RSI: 00007f3ff3d485a0 RDI: 0000000000000004 RBP: 0000000000000000 R08: 0000000000d97260 R09: 0000000000000000 R10: 7269762f73656369 R11: 0000000000000293 R12: 000000000065e420 R13: 00007f3ff25149c0 R14: 00007f3ff55b9040 R15: 0000000000000003 CPU: 1 PID: 16 Comm: ksoftirqd/1 Not tainted 4.20.0-rc1+ #137 aoe: skb alloc failure Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x244/0x39d lib/dump_stack.c:113 rsyslogd: page allocation failure: order:0, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) warn_alloc.cold.116+0xb7/0x1bd mm/page_alloc.c:3423 rsyslogd cpuset=/ mems_allowed=0 __alloc_pages_slowpath+0x26e8/0x2e10 mm/page_alloc.c:4302 __alloc_pages_nodemask+0xa80/0xde0 mm/page_alloc.c:4395 __alloc_pages include/linux/gfp.h:473 [inline] __alloc_pages_node include/linux/gfp.h:486 [inline] kmem_getpages mm/slab.c:1409 [inline] cache_grow_begin+0xa5/0x8c0 mm/slab.c:2677 fallback_alloc+0x204/0x2e0 mm/slab.c:3219 ____cache_alloc_node+0x1c7/0x1e0 mm/slab.c:3287 slab_alloc_node mm/slab.c:3327 [inline] kmem_cache_alloc_node+0xe3/0x730 mm/slab.c:3642 __alloc_skb+0x119/0x770 net/core/skbuff.c:193 alloc_skb include/linux/skbuff.h:997 [inline] ndisc_alloc_skb+0x144/0x340 net/ipv6/ndisc.c:403 ndisc_send_rs+0x331/0x6e0 net/ipv6/ndisc.c:669 addrconf_rs_timer+0x314/0x690 net/ipv6/addrconf.c:3836 call_timer_fn+0x272/0x920 kernel/time/timer.c:1326 expire_timers kernel/time/timer.c:1363 [inline] __run_timers+0x7e5/0xc70 kernel/time/timer.c:1682 run_timer_softirq+0x52/0xb0 kernel/time/timer.c:1695 __do_softirq+0x308/0xb7e kernel/softirq.c:292 run_ksoftirqd+0x5e/0x100 kernel/softirq.c:654 smpboot_thread_fn+0x68b/0xa00 kernel/smpboot.c:164 kthread+0x35a/0x440 kernel/kthread.c:246 ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:352 warn_alloc_show_mem: 1 callbacks suppressed CPU: 0 PID: 5520 Comm: rsyslogd Not tainted 4.20.0-rc1+ #137 Mem-Info: active_anon:58077 inactive_anon:3958 isolated_anon:0 active_file:12 inactive_file:14 isolated_file:0 unevictable:0 dirty:0 writeback:0 unstable:0 slab_reclaimable:11484 slab_unreclaimable:1464692 mapped:16386 shmem:4014 pagetables:510 bounce:0 free:13720 free_pcp:483 free_cma:0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: Node 0 active_anon:232308kB inactive_anon:15832kB active_file:48kB inactive_file:56kB unevictable:0kB isolated(anon):0kB isolated(file):0kB mapped:65544kB dirty:0kB writeback:0kB shmem:16056kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 217088kB writeback_tmp:0kB unstable:0kB all_unreclaimable? yes Node 0 DMA free:15908kB min:164kB low:204kB high:244kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15908kB mlocked:0kB kernel_stack:0kB pagetables:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x244/0x39d lib/dump_stack.c:113 lowmem_reserve[]: 0 2818 6321 6321 Node 0 DMA32 free:25040kB min:30052kB low:37564kB high:45076kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:3129332kB managed:2888772kB mlocked:0kB kernel_stack:0kB pagetables:0kB bounce:0kB free_pcp:248kB local_pcp:0kB free_cma:0kB warn_alloc.cold.116+0xb7/0x1bd mm/page_alloc.c:3423 lowmem_reserve[]: 0 0 3503 3503 Node 0 Normal free:13932kB min:37364kB low:46704kB high:56044kB active_anon:232308kB inactive_anon:15832kB active_file:48kB inactive_file:56kB unevictable:0kB writepending:0kB present:4718592kB managed:3587820kB mlocked:0kB kernel_stack:5184kB pagetables:2040kB bounce:0kB free_pcp:1684kB local_pcp:356kB free_cma:0kB lowmem_reserve[]: 0 0 0 0 Node 0 DMA: 1*4kB (U) 0*8kB 0*16kB 1*32kB (U) 2*64kB (U) 1*128kB (U) 1*256kB (U) 0*512kB 1*1024kB (U) 1*2048kB (M) 3*4096kB (M) = 15908kB Node 0 DMA32: 4*4kB (UM) 2*8kB (UM) 3*16kB (M) 2*32kB (M) 3*64kB (M) 5*128kB (UM) 4*256kB (UM) 3*512kB (M) 3*1024kB (UM) 3*2048kB (UM) 3*4096kB (M) = 25040kB __alloc_pages_slowpath+0x26e8/0x2e10 mm/page_alloc.c:4302 Node 0 Normal: 1007*4kB (UMEH) 460*8kB (ME) 157*16kB (ME) 51*32kB (UM) 13*64kB (UM) 0*128kB 1*256kB (U) 0*512kB 1*1024kB (U) 0*2048kB 0*4096kB = 13964kB Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB 4040 total pagecache pages 0 pages in swap cache Swap cache stats: add 0, delete 0, find 0/0 Free swap = 0kB Total swap = 0kB 1965979 pages RAM 0 pages HighMem/MovableOnly 342854 pages reserved 0 pages cma reserved ICMPv6: ndisc: ndisc_alloc_skb failed to allocate an skb ksoftirqd/1: page allocation failure: order:0, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) ksoftirqd/1 cpuset=/ mems_allowed=0 __alloc_pages_nodemask+0xa80/0xde0 mm/page_alloc.c:4395 __alloc_pages include/linux/gfp.h:473 [inline] __alloc_pages_node include/linux/gfp.h:486 [inline] kmem_getpages mm/slab.c:1409 [inline] cache_grow_begin+0xa5/0x8c0 mm/slab.c:2677 fallback_alloc+0x204/0x2e0 mm/slab.c:3219 ____cache_alloc_node+0x1c7/0x1e0 mm/slab.c:3287 slab_alloc_node mm/slab.c:3327 [inline] kmem_cache_alloc_node+0xe3/0x730 mm/slab.c:3642 __alloc_skb+0x119/0x770 net/core/skbuff.c:193 alloc_skb include/linux/skbuff.h:997 [inline] new_skb+0x26/0x1c0 drivers/block/aoe/aoecmd.c:67 aoecmd_cfg_pkts drivers/block/aoe/aoecmd.c:426 [inline] aoecmd_cfg+0x279/0x7f0 drivers/block/aoe/aoecmd.c:1381 discover_timer+0x50/0x60 drivers/block/aoe/aoemain.c:24 call_timer_fn+0x272/0x920 kernel/time/timer.c:1326 expire_timers kernel/time/timer.c:1363 [inline] __run_timers+0x7e5/0xc70 kernel/time/timer.c:1682