diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 1301ba7b2c9a..6c669de1b269 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -265,7 +265,7 @@ static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma) void hugetlb_vma_lock_read(struct vm_area_struct *vma) { if (__vma_shareable_lock(vma)) { - struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; + struct hugetlb_vma_lock *vma_lock = READ_ONCE(vma->vm_private_data); down_read(&vma_lock->rw_sema); } else if (__vma_private_lock(vma)) { @@ -278,7 +278,7 @@ void hugetlb_vma_lock_read(struct vm_area_struct *vma) void hugetlb_vma_unlock_read(struct vm_area_struct *vma) { if (__vma_shareable_lock(vma)) { - struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; + struct hugetlb_vma_lock *vma_lock = READ_ONCE(vma->vm_private_data); up_read(&vma_lock->rw_sema); } else if (__vma_private_lock(vma)) { @@ -291,7 +291,7 @@ void hugetlb_vma_unlock_read(struct vm_area_struct *vma) void hugetlb_vma_lock_write(struct vm_area_struct *vma) { if (__vma_shareable_lock(vma)) { - struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; + struct hugetlb_vma_lock *vma_lock = READ_ONCE(vma->vm_private_data); down_write(&vma_lock->rw_sema); } else if (__vma_private_lock(vma)) { @@ -304,7 +304,7 @@ void hugetlb_vma_lock_write(struct vm_area_struct *vma) void hugetlb_vma_unlock_write(struct vm_area_struct *vma) { if (__vma_shareable_lock(vma)) { - struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; + struct hugetlb_vma_lock *vma_lock = READ_ONCE(vma->vm_private_data); up_write(&vma_lock->rw_sema); } else if (__vma_private_lock(vma)) { @@ -318,7 +318,7 @@ int hugetlb_vma_trylock_write(struct vm_area_struct *vma) { if (__vma_shareable_lock(vma)) { - struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; + struct hugetlb_vma_lock *vma_lock = READ_ONCE(vma->vm_private_data); return down_write_trylock(&vma_lock->rw_sema); } else if (__vma_private_lock(vma)) { @@ -333,7 +333,7 @@ int hugetlb_vma_trylock_write(struct vm_area_struct *vma) void hugetlb_vma_assert_locked(struct vm_area_struct *vma) { if (__vma_shareable_lock(vma)) { - struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; + struct hugetlb_vma_lock *vma_lock = READ_ONCE(vma->vm_private_data); lockdep_assert_held(&vma_lock->rw_sema); } else if (__vma_private_lock(vma)) { @@ -361,7 +361,7 @@ static void __hugetlb_vma_unlock_write_put(struct hugetlb_vma_lock *vma_lock) * Semaphore synchronizes access to vma_lock->vma field. */ vma_lock->vma = NULL; - vma->vm_private_data = NULL; + WRITE_ONCE(vma->vm_private_data, NULL); up_write(&vma_lock->rw_sema); kref_put(&vma_lock->refs, hugetlb_vma_lock_release); } @@ -369,7 +369,7 @@ static void __hugetlb_vma_unlock_write_put(struct hugetlb_vma_lock *vma_lock) static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma) { if (__vma_shareable_lock(vma)) { - struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; + struct hugetlb_vma_lock *vma_lock = READ_ONCE(vma->vm_private_data); __hugetlb_vma_unlock_write_put(vma_lock); } else if (__vma_private_lock(vma)) { @@ -388,8 +388,8 @@ static void hugetlb_vma_lock_free(struct vm_area_struct *vma) if (!vma || !__vma_shareable_lock(vma)) return; - if (vma->vm_private_data) { - struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; + if (READ_ONCE(vma->vm_private_data)) { + struct hugetlb_vma_lock *vma_lock = READ_ONCE(vma->vm_private_data); down_write(&vma_lock->rw_sema); __hugetlb_vma_unlock_write_put(vma_lock); @@ -405,7 +405,7 @@ static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma) return; /* Should never get here with non-NULL vm_private_data */ - if (vma->vm_private_data) + if (READ_ONCE(vma->vm_private_data)) return; vma_lock = kmalloc(sizeof(*vma_lock), GFP_KERNEL); @@ -427,7 +427,7 @@ static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma) kref_init(&vma_lock->refs); init_rwsem(&vma_lock->rw_sema); vma_lock->vma = vma; - vma->vm_private_data = vma_lock; + WRITE_ONCE(vma->vm_private_data, vma_lock); } /* Helper that removes a struct file_region from the resv_map cache and returns @@ -1058,13 +1058,13 @@ __weak unsigned long vma_mmu_pagesize(struct vm_area_struct *vma) */ static unsigned long get_vma_private_data(struct vm_area_struct *vma) { - return (unsigned long)vma->vm_private_data; + return (unsigned long)READ_ONCE(vma->vm_private_data); } static void set_vma_private_data(struct vm_area_struct *vma, unsigned long value) { - vma->vm_private_data = (void *)value; + WRITE_ONCE(vma->vm_private_data, (void *)value); } static void @@ -1205,12 +1205,12 @@ void hugetlb_dup_vma_private(struct vm_area_struct *vma) * not guaranteed to succeed, even if read-only. */ if (vma->vm_flags & VM_MAYSHARE) { - struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; + struct hugetlb_vma_lock *vma_lock = READ_ONCE(vma->vm_private_data); if (vma_lock && vma_lock->vma != vma) - vma->vm_private_data = NULL; + WRITE_ONCE(vma->vm_private_data, NULL); } else - vma->vm_private_data = NULL; + WRITE_ONCE(vma->vm_private_data, NULL); } /* @@ -4853,11 +4853,11 @@ static void hugetlb_vm_op_open(struct vm_area_struct *vma) * for this vma. */ if (vma->vm_flags & VM_MAYSHARE) { - struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; + struct hugetlb_vma_lock *vma_lock = READ_ONCE(vma->vm_private_data); if (vma_lock) { if (vma_lock->vma != vma) { - vma->vm_private_data = NULL; + WRITE_ONCE(vma->vm_private_data, NULL); hugetlb_vma_lock_alloc(vma); } else pr_warn("HugeTLB: vma_lock already exists in %s.\n", __func__); @@ -4894,6 +4894,7 @@ static void hugetlb_vm_op_close(struct vm_area_struct *vma) hugetlb_acct_memory(h, -gbl_reserve); } + hugetlb_dup_vma_private(vma); kref_put(&resv->refs, resv_map_release); } @@ -6924,7 +6925,7 @@ static unsigned long page_table_shareable(struct vm_area_struct *svma, if (pmd_index(addr) != pmd_index(saddr) || vm_flags != svm_flags || !range_in_vma(svma, sbase, s_end) || - !svma->vm_private_data) + !READ_ONCE(svma->vm_private_data)) return 0; return saddr; @@ -6944,7 +6945,7 @@ bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr) */ if (!(vma->vm_flags & VM_MAYSHARE)) return false; - if (!vma->vm_private_data) /* vma lock required for sharing */ + if (!READ_ONCE(vma->vm_private_data)) /* vma lock required for sharing */ return false; if (!range_in_vma(vma, start, end)) return false;