diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index e070b8593b37..e6157d4ae2dd 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -5502,8 +5502,6 @@ static inline vm_fault_t hugetlb_handle_userfault(struct vm_area_struct *vma,
 	mutex_unlock(&hugetlb_fault_mutex_table[hash]);
 	i_mmap_unlock_read(mapping);
 	ret = handle_userfault(&vmf, reason);
-	i_mmap_lock_read(mapping);
-	mutex_lock(&hugetlb_fault_mutex_table[hash]);
 
 	return ret;
 }
@@ -5554,7 +5552,7 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm,
 			ret = hugetlb_handle_userfault(vma, mapping, idx,
 						       flags, haddr, address,
 						       VM_UFFD_MISSING);
-			goto out;
+			goto userfault_out;
 		}
 
 		page = alloc_huge_page(vma, haddr, 0);
@@ -5618,7 +5616,7 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm,
 			ret = hugetlb_handle_userfault(vma, mapping, idx,
 						       flags, haddr, address,
 						       VM_UFFD_MINOR);
-			goto out;
+			goto userfault_out;
 		}
 	}
 
@@ -5676,6 +5674,11 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm,
 
 	unlock_page(page);
 out:
+	hash = hugetlb_fault_mutex_hash(mapping, idx);
+	mutex_unlock(&hugetlb_fault_mutex_table[hash]);
+	i_mmap_unlock_read(mapping);
+
+userfault_out:
 	return ret;
 
 backout:
@@ -5777,7 +5780,7 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
 	if (huge_pte_none_mostly(entry)) {
 		ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep,
 				      entry, flags);
-		goto out_mutex;
+		goto out_unlocked;
 	}
 
 	ret = 0;
@@ -5878,6 +5881,8 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
 out_mutex:
 	mutex_unlock(&hugetlb_fault_mutex_table[hash]);
 	i_mmap_unlock_read(mapping);
+
+out_unlocked:	
 	/*
 	 * Generally it's safe to hold refcount during waiting page lock. But
 	 * here we just wait to defer the next page fault to avoid busy loop and