diff --git a/Documentation/admin-guide/mm/pagemap.rst b/Documentation/admin-guide/mm/pagemap.rst index fe17cf210426..c8f380271cad 100644 --- a/Documentation/admin-guide/mm/pagemap.rst +++ b/Documentation/admin-guide/mm/pagemap.rst @@ -227,92 +227,3 @@ Before Linux 3.11 pagemap bits 55-60 were used for "page-shift" (which is always 12 at most architectures). Since Linux 3.11 their meaning changes after first clear of soft-dirty bits. Since Linux 4.2 they are used for flags unconditionally. - -Pagemap Scan IOCTL -================== - -The ``PAGEMAP_SCAN`` IOCTL on the pagemap file can be used to get or optionally -clear the info about page table entries. The following operations are supported -in this IOCTL: - -- Scan the address range and get the memory ranges matching the provided criteria. - This is performed when the output buffer is specified. -- Write-protect the pages. The ``PM_SCAN_WP_MATCHING`` is used to write-protect - the pages of interest. The ``PM_SCAN_CHECK_WPASYNC`` aborts the operation if - non-Async Write Protected pages are found. The ``PM_SCAN_WP_MATCHING`` can be - used with or without ``PM_SCAN_CHECK_WPASYNC``. -- Both of those operations can be combined into one atomic operation where we can - get and write protect the pages as well. - -Following flags about pages are currently supported: - -- ``PAGE_IS_WPALLOWED`` - Page has async-write-protection enabled -- ``PAGE_IS_WRITTEN`` - Page has been written to from the time it was write protected -- ``PAGE_IS_FILE`` - Page is file backed -- ``PAGE_IS_PRESENT`` - Page is present in the memory -- ``PAGE_IS_SWAPPED`` - Page is in swapped -- ``PAGE_IS_PFNZERO`` - Page has zero PFN -- ``PAGE_IS_HUGE`` - Page is THP or Hugetlb backed - -The ``struct pm_scan_arg`` is used as the argument of the IOCTL. - - 1. The size of the ``struct pm_scan_arg`` must be specified in the ``size`` - field. This field will be helpful in recognizing the structure if extensions - are done later. - 2. The flags can be specified in the ``flags`` field. The ``PM_SCAN_WP_MATCHING`` - and ``PM_SCAN_CHECK_WPASYNC`` are the only added flags at this time. The get - operation is optionally performed depending upon if the output buffer is - provided or not. - 3. The range is specified through ``start`` and ``end``. - 4. The walk can abort before visiting the complete range such as the user buffer - can get full etc. The walk ending address is specified in``end_walk``. - 5. The output buffer of ``struct page_region`` array and size is specified in - ``vec`` and ``vec_len``. - 6. The optional maximum requested pages are specified in the ``max_pages``. - 7. The masks are specified in ``category_mask``, ``category_anyof_mask``, - ``category_inverted`` and ``return_mask``. - -Find pages which have been written and WP them as well:: - - struct pm_scan_arg arg = { - .size = sizeof(arg), - .flags = PM_SCAN_CHECK_WPASYNC | PM_SCAN_CHECK_WPASYNC, - .. - .category_mask = PAGE_IS_WRITTEN, - .return_mask = PAGE_IS_WRITTEN, - }; - -Find pages which have been written, are file backed, not swapped and either -present or huge:: - - struct pm_scan_arg arg = { - .size = sizeof(arg), - .flags = 0, - .. - .category_mask = PAGE_IS_WRITTEN | PAGE_IS_SWAPPED, - .category_inverted = PAGE_IS_SWAPPED, - .category_anyof_mask = PAGE_IS_PRESENT | PAGE_IS_HUGE, - .return_mask = PAGE_IS_WRITTEN | PAGE_IS_SWAPPED | - PAGE_IS_PRESENT | PAGE_IS_HUGE, - }; - -The ``PAGE_IS_WRITTEN`` flag can be considered as a better-performing alternative -of soft-dirty flag. It doesn't get affected by VMA merging of the kernel and hence -the user can find the true soft-dirty pages in case of normal pages. (There may -still be extra dirty pages reported for THP or Hugetlb pages.) - -"PAGE_IS_WRITTEN" category is used with uffd write protect-enabled ranges to -implement memory dirty tracking in userspace: - - 1. The userfaultfd file descriptor is created with ``userfaultfd`` syscall. - 2. The ``UFFD_FEATURE_WP_UNPOPULATED`` and ``UFFD_FEATURE_WP_ASYNC`` features - are set by ``UFFDIO_API`` IOCTL. - 3. The memory range is registered with ``UFFDIO_REGISTER_MODE_WP`` mode - through ``UFFDIO_REGISTER`` IOCTL. - 4. Then any part of the registered memory or the whole memory region must - be write protected using ``PAGEMAP_SCAN`` IOCTL with flag ``PM_SCAN_WP_MATCHING`` - or the ``UFFDIO_WRITEPROTECT`` IOCTL can be used. Both of these perform the - same operation. The former is better in terms of performance. - 5. Now the ``PAGEMAP_SCAN`` IOCTL can be used to either just find pages which - have been written to since they were last marked and/or optionally write protect - the pages as well. diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index ef2eb12906da..ff2eae6be537 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -20,8 +20,6 @@ #include #include #include -#include -#include #include #include @@ -1758,737 +1756,11 @@ static int pagemap_release(struct inode *inode, struct file *file) return 0; } -#define PM_SCAN_CATEGORIES (PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | \ - PAGE_IS_FILE | PAGE_IS_PRESENT | \ - PAGE_IS_SWAPPED | PAGE_IS_PFNZERO | \ - PAGE_IS_HUGE) -#define PM_SCAN_FLAGS (PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC) - -struct pagemap_scan_private { - struct pm_scan_arg arg; - unsigned long masks_of_interest, cur_vma_category; - struct page_region *vec_buf; - unsigned long vec_buf_len, vec_buf_index, found_pages; - struct page_region __user *vec_out; -}; - -static unsigned long pagemap_page_category(struct pagemap_scan_private *p, - struct vm_area_struct *vma, - unsigned long addr, pte_t pte) -{ - unsigned long categories = 0; - - if (pte_present(pte)) { - struct page *page; - - categories |= PAGE_IS_PRESENT; - if (!pte_uffd_wp(pte)) - categories |= PAGE_IS_WRITTEN; - - if (p->masks_of_interest & PAGE_IS_FILE) { - page = vm_normal_page(vma, addr, pte); - if (page && !PageAnon(page)) - categories |= PAGE_IS_FILE; - } - - if (is_zero_pfn(pte_pfn(pte))) - categories |= PAGE_IS_PFNZERO; - } else if (is_swap_pte(pte)) { - swp_entry_t swp; - - categories |= PAGE_IS_SWAPPED; - if (!pte_swp_uffd_wp_any(pte)) - categories |= PAGE_IS_WRITTEN; - - if (p->masks_of_interest & PAGE_IS_FILE) { - swp = pte_to_swp_entry(pte); - if (is_pfn_swap_entry(swp) && - !PageAnon(pfn_swap_entry_to_page(swp))) - categories |= PAGE_IS_FILE; - } - } - - return categories; -} - -static void make_uffd_wp_pte(struct vm_area_struct *vma, - unsigned long addr, pte_t *pte) -{ - pte_t ptent = ptep_get(pte); - - if (pte_present(ptent)) { - pte_t old_pte; - - old_pte = ptep_modify_prot_start(vma, addr, pte); - ptent = pte_mkuffd_wp(ptent); - ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent); - } else if (is_swap_pte(ptent)) { - ptent = pte_swp_mkuffd_wp(ptent); - set_pte_at(vma->vm_mm, addr, pte, ptent); - } else { - set_pte_at(vma->vm_mm, addr, pte, - make_pte_marker(PTE_MARKER_UFFD_WP)); - } -} - -#ifdef CONFIG_TRANSPARENT_HUGEPAGE -static unsigned long pagemap_thp_category(struct pagemap_scan_private *p, - struct vm_area_struct *vma, - unsigned long addr, pmd_t pmd) -{ - unsigned long categories = PAGE_IS_HUGE; - - if (pmd_present(pmd)) { - struct page *page; - - categories |= PAGE_IS_PRESENT; - if (!pmd_uffd_wp(pmd)) - categories |= PAGE_IS_WRITTEN; - - if (p->masks_of_interest & PAGE_IS_FILE) { - page = vm_normal_page_pmd(vma, addr, pmd); - if (page && !PageAnon(page)) - categories |= PAGE_IS_FILE; - } - - if (is_zero_pfn(pmd_pfn(pmd))) - categories |= PAGE_IS_PFNZERO; - } else if (is_swap_pmd(pmd)) { - swp_entry_t swp; - - categories |= PAGE_IS_SWAPPED; - if (!pmd_swp_uffd_wp(pmd)) - categories |= PAGE_IS_WRITTEN; - - if (p->masks_of_interest & PAGE_IS_FILE) { - swp = pmd_to_swp_entry(pmd); - if (is_pfn_swap_entry(swp) && - !PageAnon(pfn_swap_entry_to_page(swp))) - categories |= PAGE_IS_FILE; - } - } - - return categories; -} - -static void make_uffd_wp_pmd(struct vm_area_struct *vma, - unsigned long addr, pmd_t *pmdp) -{ - pmd_t old, pmd = *pmdp; - - if (pmd_present(pmd)) { - old = pmdp_invalidate_ad(vma, addr, pmdp); - pmd = pmd_mkuffd_wp(old); - set_pmd_at(vma->vm_mm, addr, pmdp, pmd); - } else if (is_migration_entry(pmd_to_swp_entry(pmd))) { - pmd = pmd_swp_mkuffd_wp(pmd); - set_pmd_at(vma->vm_mm, addr, pmdp, pmd); - } -} -#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ - -#ifdef CONFIG_HUGETLB_PAGE -static unsigned long pagemap_hugetlb_category(pte_t pte) -{ - unsigned long categories = PAGE_IS_HUGE; - - /* - * According to pagemap_hugetlb_range(), file-backed HugeTLB - * page cannot be swapped. So PAGE_IS_FILE is not checked for - * swapped pages. - */ - if (pte_present(pte)) { - categories |= PAGE_IS_PRESENT; - if (!huge_pte_uffd_wp(pte)) - categories |= PAGE_IS_WRITTEN; - if (!PageAnon(pte_page(pte))) - categories |= PAGE_IS_FILE; - if (is_zero_pfn(pte_pfn(pte))) - categories |= PAGE_IS_PFNZERO; - } else if (is_swap_pte(pte)) { - categories |= PAGE_IS_SWAPPED; - if (!pte_swp_uffd_wp_any(pte)) - categories |= PAGE_IS_WRITTEN; - } - - return categories; -} - -static void make_uffd_wp_huge_pte(struct vm_area_struct *vma, - unsigned long addr, pte_t *ptep, - pte_t ptent) -{ - unsigned long psize; - - if (is_hugetlb_entry_hwpoisoned(ptent) || is_pte_marker(ptent)) - return; - - psize = huge_page_size(hstate_vma(vma)); - - if (is_hugetlb_entry_migration(ptent)) - set_huge_pte_at(vma->vm_mm, addr, ptep, - pte_swp_mkuffd_wp(ptent), psize); - else if (!huge_pte_none(ptent)) - huge_ptep_modify_prot_commit(vma, addr, ptep, ptent, - huge_pte_mkuffd_wp(ptent)); - else - set_huge_pte_at(vma->vm_mm, addr, ptep, - make_pte_marker(PTE_MARKER_UFFD_WP), psize); -} -#endif /* CONFIG_HUGETLB_PAGE */ - -#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE) -static void pagemap_scan_backout_range(struct pagemap_scan_private *p, - unsigned long addr, unsigned long end) -{ - struct page_region *cur_buf = &p->vec_buf[p->vec_buf_index]; - - if (cur_buf->start != addr) - cur_buf->end = addr; - else - cur_buf->start = cur_buf->end = 0; - - p->found_pages -= (end - addr) / PAGE_SIZE; -} -#endif - -static bool pagemap_scan_is_interesting_page(unsigned long categories, - const struct pagemap_scan_private *p) -{ - categories ^= p->arg.category_inverted; - if ((categories & p->arg.category_mask) != p->arg.category_mask) - return false; - if (p->arg.category_anyof_mask && !(categories & p->arg.category_anyof_mask)) - return false; - - return true; -} - -static bool pagemap_scan_is_interesting_vma(unsigned long categories, - const struct pagemap_scan_private *p) -{ - unsigned long required = p->arg.category_mask & PAGE_IS_WPALLOWED; - - categories ^= p->arg.category_inverted; - if ((categories & required) != required) - return false; - - return true; -} - -static int pagemap_scan_test_walk(unsigned long start, unsigned long end, - struct mm_walk *walk) -{ - struct pagemap_scan_private *p = walk->private; - struct vm_area_struct *vma = walk->vma; - unsigned long vma_category = 0; - - if (userfaultfd_wp_async(vma) && userfaultfd_wp_use_markers(vma)) - vma_category |= PAGE_IS_WPALLOWED; - else if (p->arg.flags & PM_SCAN_CHECK_WPASYNC) - return -EPERM; - - if (vma->vm_flags & VM_PFNMAP) - return 1; - - if (!pagemap_scan_is_interesting_vma(vma_category, p)) - return 1; - - p->cur_vma_category = vma_category; - - return 0; -} - -static bool pagemap_scan_push_range(unsigned long categories, - struct pagemap_scan_private *p, - unsigned long addr, unsigned long end) -{ - struct page_region *cur_buf = &p->vec_buf[p->vec_buf_index]; - - /* - * When there is no output buffer provided at all, the sentinel values - * won't match here. There is no other way for `cur_buf->end` to be - * non-zero other than it being non-empty. - */ - if (addr == cur_buf->end && categories == cur_buf->categories) { - cur_buf->end = end; - return true; - } - - if (cur_buf->end) { - if (p->vec_buf_index >= p->vec_buf_len - 1) - return false; - - cur_buf = &p->vec_buf[++p->vec_buf_index]; - } - - cur_buf->start = addr; - cur_buf->end = end; - cur_buf->categories = categories; - - return true; -} - -static int pagemap_scan_output(unsigned long categories, - struct pagemap_scan_private *p, - unsigned long addr, unsigned long *end) -{ - unsigned long n_pages, total_pages; - int ret = 0; - - if (!p->vec_buf) - return 0; - - categories &= p->arg.return_mask; - - n_pages = (*end - addr) / PAGE_SIZE; - if (check_add_overflow(p->found_pages, n_pages, &total_pages) || - total_pages > p->arg.max_pages) { - size_t n_too_much = total_pages - p->arg.max_pages; - *end -= n_too_much * PAGE_SIZE; - n_pages -= n_too_much; - ret = -ENOSPC; - } - - if (!pagemap_scan_push_range(categories, p, addr, *end)) { - *end = addr; - n_pages = 0; - ret = -ENOSPC; - } - - p->found_pages += n_pages; - if (ret) - p->arg.walk_end = *end; - - return ret; -} - -static int pagemap_scan_thp_entry(pmd_t *pmd, unsigned long start, - unsigned long end, struct mm_walk *walk) -{ -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - struct pagemap_scan_private *p = walk->private; - struct vm_area_struct *vma = walk->vma; - unsigned long categories; - spinlock_t *ptl; - int ret = 0; - - ptl = pmd_trans_huge_lock(pmd, vma); - if (!ptl) - return -ENOENT; - - categories = p->cur_vma_category | - pagemap_thp_category(p, vma, start, *pmd); - - if (!pagemap_scan_is_interesting_page(categories, p)) - goto out_unlock; - - ret = pagemap_scan_output(categories, p, start, &end); - if (start == end) - goto out_unlock; - - if (~p->arg.flags & PM_SCAN_WP_MATCHING) - goto out_unlock; - if (~categories & PAGE_IS_WRITTEN) - goto out_unlock; - - /* - * Break huge page into small pages if the WP operation - * needs to be performed on a portion of the huge page. - */ - if (end != start + HPAGE_SIZE) { - spin_unlock(ptl); - split_huge_pmd(vma, pmd, start); - pagemap_scan_backout_range(p, start, end); - /* Report as if there was no THP */ - return -ENOENT; - } - - make_uffd_wp_pmd(vma, start, pmd); - flush_tlb_range(vma, start, end); -out_unlock: - spin_unlock(ptl); - return ret; -#else /* !CONFIG_TRANSPARENT_HUGEPAGE */ - return -ENOENT; -#endif -} - -static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, - unsigned long end, struct mm_walk *walk) -{ - struct pagemap_scan_private *p = walk->private; - struct vm_area_struct *vma = walk->vma; - unsigned long addr, flush_end = 0; - pte_t *pte, *start_pte; - spinlock_t *ptl; - int ret; - - arch_enter_lazy_mmu_mode(); - - ret = pagemap_scan_thp_entry(pmd, start, end, walk); - if (ret != -ENOENT) { - arch_leave_lazy_mmu_mode(); - return ret; - } - - ret = 0; - start_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, start, &ptl); - if (!pte) { - arch_leave_lazy_mmu_mode(); - walk->action = ACTION_AGAIN; - return 0; - } - - if (!p->vec_out) { - /* Fast path for performing exclusive WP */ - for (addr = start; addr != end; pte++, addr += PAGE_SIZE) { - if (pte_uffd_wp(ptep_get(pte))) - continue; - make_uffd_wp_pte(vma, addr, pte); - if (!flush_end) - start = addr; - flush_end = addr + PAGE_SIZE; - } - goto flush_and_return; - } - - if (!p->arg.category_anyof_mask && !p->arg.category_inverted && - p->arg.category_mask == PAGE_IS_WRITTEN && - p->arg.return_mask == PAGE_IS_WRITTEN) { - for (addr = start; addr < end; pte++, addr += PAGE_SIZE) { - unsigned long next = addr + PAGE_SIZE; - - if (pte_uffd_wp(ptep_get(pte))) - continue; - ret = pagemap_scan_output(p->cur_vma_category | PAGE_IS_WRITTEN, - p, addr, &next); - if (next == addr) - break; - if (~p->arg.flags & PM_SCAN_WP_MATCHING) - continue; - make_uffd_wp_pte(vma, addr, pte); - if (!flush_end) - start = addr; - flush_end = next; - } - goto flush_and_return; - } - - for (addr = start; addr != end; pte++, addr += PAGE_SIZE) { - unsigned long categories = p->cur_vma_category | - pagemap_page_category(p, vma, addr, ptep_get(pte)); - unsigned long next = addr + PAGE_SIZE; - - if (!pagemap_scan_is_interesting_page(categories, p)) - continue; - - ret = pagemap_scan_output(categories, p, addr, &next); - if (next == addr) - break; - - if (~p->arg.flags & PM_SCAN_WP_MATCHING) - continue; - if (~categories & PAGE_IS_WRITTEN) - continue; - - make_uffd_wp_pte(vma, addr, pte); - if (!flush_end) - start = addr; - flush_end = next; - } - -flush_and_return: - if (flush_end) - flush_tlb_range(vma, start, addr); - - pte_unmap_unlock(start_pte, ptl); - arch_leave_lazy_mmu_mode(); - - cond_resched(); - return ret; -} - -#ifdef CONFIG_HUGETLB_PAGE -static int pagemap_scan_hugetlb_entry(pte_t *ptep, unsigned long hmask, - unsigned long start, unsigned long end, - struct mm_walk *walk) -{ - struct pagemap_scan_private *p = walk->private; - struct vm_area_struct *vma = walk->vma; - unsigned long categories; - spinlock_t *ptl; - int ret = 0; - pte_t pte; - - if (~p->arg.flags & PM_SCAN_WP_MATCHING) { - /* Go the short route when not write-protecting pages. */ - - pte = huge_ptep_get(ptep); - categories = p->cur_vma_category | pagemap_hugetlb_category(pte); - - if (!pagemap_scan_is_interesting_page(categories, p)) - return 0; - - return pagemap_scan_output(categories, p, start, &end); - } - - i_mmap_lock_write(vma->vm_file->f_mapping); - ptl = huge_pte_lock(hstate_vma(vma), vma->vm_mm, ptep); - - pte = huge_ptep_get(ptep); - categories = p->cur_vma_category | pagemap_hugetlb_category(pte); - - if (!pagemap_scan_is_interesting_page(categories, p)) - goto out_unlock; - - ret = pagemap_scan_output(categories, p, start, &end); - if (start == end) - goto out_unlock; - - if (~categories & PAGE_IS_WRITTEN) - goto out_unlock; - - if (end != start + HPAGE_SIZE) { - /* Partial HugeTLB page WP isn't possible. */ - pagemap_scan_backout_range(p, start, end); - p->arg.walk_end = start; - ret = 0; - goto out_unlock; - } - - make_uffd_wp_huge_pte(vma, start, ptep, pte); - flush_hugetlb_tlb_range(vma, start, end); - -out_unlock: - spin_unlock(ptl); - i_mmap_unlock_write(vma->vm_file->f_mapping); - - return ret; -} -#else -#define pagemap_scan_hugetlb_entry NULL -#endif - -static int pagemap_scan_pte_hole(unsigned long addr, unsigned long end, - int depth, struct mm_walk *walk) -{ - struct pagemap_scan_private *p = walk->private; - struct vm_area_struct *vma = walk->vma; - int ret, err; - - if (!vma || !pagemap_scan_is_interesting_page(p->cur_vma_category, p)) - return 0; - - ret = pagemap_scan_output(p->cur_vma_category, p, addr, &end); - if (addr == end) - return ret; - - if (~p->arg.flags & PM_SCAN_WP_MATCHING) - return ret; - - err = uffd_wp_range(vma, addr, end - addr, true); - if (err < 0) - ret = err; - - return ret; -} - -static const struct mm_walk_ops pagemap_scan_ops = { - .test_walk = pagemap_scan_test_walk, - .pmd_entry = pagemap_scan_pmd_entry, - .pte_hole = pagemap_scan_pte_hole, - .hugetlb_entry = pagemap_scan_hugetlb_entry, -}; - -static int pagemap_scan_get_args(struct pm_scan_arg *arg, - unsigned long uarg) -{ - if (copy_from_user(arg, (void __user *)uarg, sizeof(*arg))) - return -EFAULT; - - if (arg->size != sizeof(struct pm_scan_arg)) - return -EINVAL; - - /* Validate requested features */ - if (arg->flags & ~PM_SCAN_FLAGS) - return -EINVAL; - if ((arg->category_inverted | arg->category_mask | - arg->category_anyof_mask | arg->return_mask) & ~PM_SCAN_CATEGORIES) - return -EINVAL; - - arg->start = untagged_addr((unsigned long)arg->start); - arg->end = untagged_addr((unsigned long)arg->end); - arg->vec = untagged_addr((unsigned long)arg->vec); - - /* Validate memory pointers */ - if (!IS_ALIGNED(arg->start, PAGE_SIZE)) - return -EINVAL; - if (!access_ok((void __user *)(long)arg->start, arg->end - arg->start)) - return -EFAULT; - if (!arg->vec && arg->vec_len) - return -EINVAL; - if (arg->vec && !access_ok((void __user *)(long)arg->vec, - arg->vec_len * sizeof(struct page_region))) - return -EFAULT; - - /* Fixup default values */ - arg->end = ALIGN(arg->end, PAGE_SIZE); - arg->walk_end = 0; - if (!arg->max_pages) - arg->max_pages = ULONG_MAX; - - return 0; -} - -static int pagemap_scan_writeback_args(struct pm_scan_arg *arg, - unsigned long uargl) -{ - struct pm_scan_arg __user *uarg = (void __user *)uargl; - - if (copy_to_user(&uarg->walk_end, &arg->walk_end, sizeof(arg->walk_end))) - return -EFAULT; - - return 0; -} - -static int pagemap_scan_init_bounce_buffer(struct pagemap_scan_private *p) -{ - if (!p->arg.vec_len) - return 0; - - p->vec_buf_len = min_t(size_t, PAGEMAP_WALK_SIZE >> PAGE_SHIFT, - p->arg.vec_len); - p->vec_buf = kmalloc_array(p->vec_buf_len, sizeof(*p->vec_buf), - GFP_KERNEL); - if (!p->vec_buf) - return -ENOMEM; - - p->vec_buf->start = p->vec_buf->end = 0; - p->vec_out = (struct page_region __user *)(long)p->arg.vec; - - return 0; -} - -static long pagemap_scan_flush_buffer(struct pagemap_scan_private *p) -{ - const struct page_region *buf = p->vec_buf; - long n = p->vec_buf_index; - - if (!p->vec_buf) - return 0; - - if (buf[n].end != buf[n].start) - n++; - - if (!n) - return 0; - - if (copy_to_user(p->vec_out, buf, n * sizeof(*buf))) - return -EFAULT; - - p->arg.vec_len -= n; - p->vec_out += n; - - p->vec_buf_index = 0; - p->vec_buf_len = min_t(size_t, p->vec_buf_len, p->arg.vec_len); - p->vec_buf->start = p->vec_buf->end = 0; - - return n; -} - -static long do_pagemap_scan(struct mm_struct *mm, unsigned long uarg) -{ - struct mmu_notifier_range range; - struct pagemap_scan_private p = {0}; - unsigned long walk_start; - size_t n_ranges_out = 0; - int ret; - - ret = pagemap_scan_get_args(&p.arg, uarg); - if (ret) - return ret; - - p.masks_of_interest = p.arg.category_mask | p.arg.category_anyof_mask | - p.arg.return_mask; - ret = pagemap_scan_init_bounce_buffer(&p); - if (ret) - return ret; - - /* Protection change for the range is going to happen. */ - if (p.arg.flags & PM_SCAN_WP_MATCHING) { - mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_VMA, 0, - mm, p.arg.start, p.arg.end); - mmu_notifier_invalidate_range_start(&range); - } - - for (walk_start = p.arg.start; walk_start < p.arg.end; - walk_start = p.arg.walk_end) { - long n_out; - - if (fatal_signal_pending(current)) { - ret = -EINTR; - break; - } - - ret = mmap_read_lock_killable(mm); - if (ret) - break; - ret = walk_page_range(mm, walk_start, p.arg.end, - &pagemap_scan_ops, &p); - mmap_read_unlock(mm); - - n_out = pagemap_scan_flush_buffer(&p); - if (n_out < 0) - ret = n_out; - else - n_ranges_out += n_out; - - if (ret != -ENOSPC) - break; - - if (p.arg.vec_len == 0 || p.found_pages == p.arg.max_pages) - break; - } - - /* ENOSPC signifies early stop (buffer full) from the walk. */ - if (!ret || ret == -ENOSPC) - ret = n_ranges_out; - - /* The walk_end isn't set when ret is zero */ - if (!p.arg.walk_end) - p.arg.walk_end = p.arg.end; - if (pagemap_scan_writeback_args(&p.arg, uarg)) - ret = -EFAULT; - - if (p.arg.flags & PM_SCAN_WP_MATCHING) - mmu_notifier_invalidate_range_end(&range); - - kfree(p.vec_buf); - return ret; -} - -static long do_pagemap_cmd(struct file *file, unsigned int cmd, - unsigned long arg) -{ - struct mm_struct *mm = file->private_data; - - switch (cmd) { - case PAGEMAP_SCAN: - return do_pagemap_scan(mm, arg); - - default: - return -EINVAL; - } -} - const struct file_operations proc_pagemap_operations = { .llseek = mem_lseek, /* borrow this */ .read = pagemap_read, .open = pagemap_open, .release = pagemap_release, - .unlocked_ioctl = do_pagemap_cmd, - .compat_ioctl = do_pagemap_cmd, }; #endif /* CONFIG_PROC_PAGE_MONITOR */ diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index d3acecc5db4b..6d06e70314b9 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -280,7 +280,6 @@ long hugetlb_change_protection(struct vm_area_struct *vma, unsigned long cp_flags); bool is_hugetlb_entry_migration(pte_t pte); -bool is_hugetlb_entry_hwpoisoned(pte_t pte); void hugetlb_unshare_all_pmds(struct vm_area_struct *vma); #else /* !CONFIG_HUGETLB_PAGE */ diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index f2dc19f40d05..c98df391bfd8 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -221,13 +221,6 @@ static inline vm_fault_t handle_userfault(struct vm_fault *vmf, return VM_FAULT_SIGBUS; } -static inline long uffd_wp_range(struct vm_area_struct *vma, - unsigned long start, unsigned long len, - bool enable_wp) -{ - return false; -} - static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, struct vm_userfaultfd_ctx vm_ctx) { diff --git a/include/uapi/linux/fs.h b/include/uapi/linux/fs.h index da43810b7485..b7b56871029c 100644 --- a/include/uapi/linux/fs.h +++ b/include/uapi/linux/fs.h @@ -305,63 +305,4 @@ typedef int __bitwise __kernel_rwf_t; #define RWF_SUPPORTED (RWF_HIPRI | RWF_DSYNC | RWF_SYNC | RWF_NOWAIT |\ RWF_APPEND) -/* Pagemap ioctl */ -#define PAGEMAP_SCAN _IOWR('f', 16, struct pm_scan_arg) - -/* Bitmasks provided in pm_scan_args masks and reported in page_region.categories. */ -#define PAGE_IS_WPALLOWED (1 << 0) -#define PAGE_IS_WRITTEN (1 << 1) -#define PAGE_IS_FILE (1 << 2) -#define PAGE_IS_PRESENT (1 << 3) -#define PAGE_IS_SWAPPED (1 << 4) -#define PAGE_IS_PFNZERO (1 << 5) -#define PAGE_IS_HUGE (1 << 6) - -/* - * struct page_region - Page region with flags - * @start: Start of the region - * @end: End of the region (exclusive) - * @categories: PAGE_IS_* category bitmask for the region - */ -struct page_region { - __u64 start; - __u64 end; - __u64 categories; -}; - -/* Flags for PAGEMAP_SCAN ioctl */ -#define PM_SCAN_WP_MATCHING (1 << 0) /* Write protect the pages matched. */ -#define PM_SCAN_CHECK_WPASYNC (1 << 1) /* Abort the scan when a non-WP-enabled page is found. */ - -/* - * struct pm_scan_arg - Pagemap ioctl argument - * @size: Size of the structure - * @flags: Flags for the IOCTL - * @start: Starting address of the region - * @end: Ending address of the region - * @walk_end Address where the scan stopped (written by kernel). - * walk_end == end (address tags cleared) informs that the scan completed on entire range. - * @vec: Address of page_region struct array for output - * @vec_len: Length of the page_region struct array - * @max_pages: Optional limit for number of returned pages (0 = disabled) - * @category_inverted: PAGE_IS_* categories which values match if 0 instead of 1 - * @category_mask: Skip pages for which any category doesn't match - * @category_anyof_mask: Skip pages for which no category matches - * @return_mask: PAGE_IS_* categories that are to be reported in `page_region`s returned - */ -struct pm_scan_arg { - __u64 size; - __u64 flags; - __u64 start; - __u64 end; - __u64 walk_end; - __u64 vec; - __u64 vec_len; - __u64 max_pages; - __u64 category_inverted; - __u64 category_mask; - __u64 category_anyof_mask; - __u64 return_mask; -}; - #endif /* _UAPI_LINUX_FS_H */ diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 1169ef2f2176..08d7e753c9dc 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -5258,7 +5258,7 @@ bool is_hugetlb_entry_migration(pte_t pte) return false; } -bool is_hugetlb_entry_hwpoisoned(pte_t pte) +static bool is_hugetlb_entry_hwpoisoned(pte_t pte) { swp_entry_t swp; @@ -6482,8 +6482,7 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, } entry = huge_pte_clear_uffd_wp(entry); - set_huge_pte_at(mm, haddr, ptep, entry, - huge_page_size(hstate_vma(vma))); + set_huge_pte_at(mm, haddr, ptep, entry); /* Fallthrough to CoW */ } diff --git a/tools/include/uapi/linux/fs.h b/tools/include/uapi/linux/fs.h index da43810b7485..b7b56871029c 100644 --- a/tools/include/uapi/linux/fs.h +++ b/tools/include/uapi/linux/fs.h @@ -305,63 +305,4 @@ typedef int __bitwise __kernel_rwf_t; #define RWF_SUPPORTED (RWF_HIPRI | RWF_DSYNC | RWF_SYNC | RWF_NOWAIT |\ RWF_APPEND) -/* Pagemap ioctl */ -#define PAGEMAP_SCAN _IOWR('f', 16, struct pm_scan_arg) - -/* Bitmasks provided in pm_scan_args masks and reported in page_region.categories. */ -#define PAGE_IS_WPALLOWED (1 << 0) -#define PAGE_IS_WRITTEN (1 << 1) -#define PAGE_IS_FILE (1 << 2) -#define PAGE_IS_PRESENT (1 << 3) -#define PAGE_IS_SWAPPED (1 << 4) -#define PAGE_IS_PFNZERO (1 << 5) -#define PAGE_IS_HUGE (1 << 6) - -/* - * struct page_region - Page region with flags - * @start: Start of the region - * @end: End of the region (exclusive) - * @categories: PAGE_IS_* category bitmask for the region - */ -struct page_region { - __u64 start; - __u64 end; - __u64 categories; -}; - -/* Flags for PAGEMAP_SCAN ioctl */ -#define PM_SCAN_WP_MATCHING (1 << 0) /* Write protect the pages matched. */ -#define PM_SCAN_CHECK_WPASYNC (1 << 1) /* Abort the scan when a non-WP-enabled page is found. */ - -/* - * struct pm_scan_arg - Pagemap ioctl argument - * @size: Size of the structure - * @flags: Flags for the IOCTL - * @start: Starting address of the region - * @end: Ending address of the region - * @walk_end Address where the scan stopped (written by kernel). - * walk_end == end (address tags cleared) informs that the scan completed on entire range. - * @vec: Address of page_region struct array for output - * @vec_len: Length of the page_region struct array - * @max_pages: Optional limit for number of returned pages (0 = disabled) - * @category_inverted: PAGE_IS_* categories which values match if 0 instead of 1 - * @category_mask: Skip pages for which any category doesn't match - * @category_anyof_mask: Skip pages for which no category matches - * @return_mask: PAGE_IS_* categories that are to be reported in `page_region`s returned - */ -struct pm_scan_arg { - __u64 size; - __u64 flags; - __u64 start; - __u64 end; - __u64 walk_end; - __u64 vec; - __u64 vec_len; - __u64 max_pages; - __u64 category_inverted; - __u64 category_mask; - __u64 category_anyof_mask; - __u64 return_mask; -}; - #endif /* _UAPI_LINUX_FS_H */ diff --git a/tools/testing/selftests/mm/.gitignore b/tools/testing/selftests/mm/.gitignore index 4ff10ea61461..1d73b721ad26 100644 --- a/tools/testing/selftests/mm/.gitignore +++ b/tools/testing/selftests/mm/.gitignore @@ -18,8 +18,6 @@ mremap_dontunmap mremap_test on-fault-limit transhuge-stress -pagemap_ioctl -*.tmp* protection_keys protection_keys_32 protection_keys_64 diff --git a/tools/testing/selftests/mm/Makefile b/tools/testing/selftests/mm/Makefile index 78dfec8bc676..e71ec9910c62 100644 --- a/tools/testing/selftests/mm/Makefile +++ b/tools/testing/selftests/mm/Makefile @@ -33,7 +33,7 @@ endif MAKEFLAGS += --no-builtin-rules CFLAGS = -Wall -I $(top_srcdir) $(EXTRA_CFLAGS) $(KHDR_INCLUDES) -LDLIBS = -lrt -lpthread -lm +LDLIBS = -lrt -lpthread TEST_GEN_FILES = cow TEST_GEN_FILES += compaction_test @@ -60,7 +60,6 @@ TEST_GEN_FILES += mrelease_test TEST_GEN_FILES += mremap_dontunmap TEST_GEN_FILES += mremap_test TEST_GEN_FILES += on-fault-limit -TEST_GEN_PROGS += pagemap_ioctl TEST_GEN_FILES += thuge-gen TEST_GEN_FILES += transhuge-stress TEST_GEN_FILES += uffd-stress diff --git a/tools/testing/selftests/mm/config b/tools/testing/selftests/mm/config index 4309916f629e..be087c4bc396 100644 --- a/tools/testing/selftests/mm/config +++ b/tools/testing/selftests/mm/config @@ -1,6 +1,5 @@ CONFIG_SYSVIPC=y CONFIG_USERFAULTFD=y -CONFIG_PTE_MARKER_UFFD_WP=y CONFIG_TEST_VMALLOC=m CONFIG_DEVICE_PRIVATE=y CONFIG_TEST_HMM=m diff --git a/tools/testing/selftests/mm/pagemap_ioctl.c b/tools/testing/selftests/mm/pagemap_ioctl.c index befab43719ba..000000000000 --- a/tools/testing/selftests/mm/pagemap_ioctl.c +++ /dev/null @@ -1,1661 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include "vm_util.h" -#include "../kselftest.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define PAGEMAP_BITS_ALL (PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | \ - PAGE_IS_FILE | PAGE_IS_PRESENT | \ - PAGE_IS_SWAPPED | PAGE_IS_PFNZERO | \ - PAGE_IS_HUGE) -#define PAGEMAP_NON_WRITTEN_BITS (PAGE_IS_WPALLOWED | PAGE_IS_FILE | \ - PAGE_IS_PRESENT | PAGE_IS_SWAPPED | \ - PAGE_IS_PFNZERO | PAGE_IS_HUGE) - -#define TEST_ITERATIONS 100 -#define PAGEMAP "/proc/self/pagemap" -int pagemap_fd; -int uffd; -int page_size; -int hpage_size; - -#define LEN(region) ((region.end - region.start)/page_size) - -static long pagemap_ioctl(void *start, int len, void *vec, int vec_len, int flag, - int max_pages, long required_mask, long anyof_mask, long excluded_mask, - long return_mask) -{ - struct pm_scan_arg arg; - - arg.start = (uintptr_t)start; - arg.end = (uintptr_t)(start + len); - arg.vec = (uintptr_t)vec; - arg.vec_len = vec_len; - arg.flags = flag; - arg.size = sizeof(struct pm_scan_arg); - arg.max_pages = max_pages; - arg.category_mask = required_mask; - arg.category_anyof_mask = anyof_mask; - arg.category_inverted = excluded_mask; - arg.return_mask = return_mask; - - return ioctl(pagemap_fd, PAGEMAP_SCAN, &arg); -} - -static long pagemap_ioc(void *start, int len, void *vec, int vec_len, int flag, - int max_pages, long required_mask, long anyof_mask, long excluded_mask, - long return_mask, long *walk_end) -{ - struct pm_scan_arg arg; - int ret; - - arg.start = (uintptr_t)start; - arg.end = (uintptr_t)(start + len); - arg.vec = (uintptr_t)vec; - arg.vec_len = vec_len; - arg.flags = flag; - arg.size = sizeof(struct pm_scan_arg); - arg.max_pages = max_pages; - arg.category_mask = required_mask; - arg.category_anyof_mask = anyof_mask; - arg.category_inverted = excluded_mask; - arg.return_mask = return_mask; - - ret = ioctl(pagemap_fd, PAGEMAP_SCAN, &arg); - - if (walk_end) - *walk_end = arg.walk_end; - - return ret; -} - - -int init_uffd(void) -{ - struct uffdio_api uffdio_api; - - uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); - if (uffd == -1) - return uffd; - - uffdio_api.api = UFFD_API; - uffdio_api.features = UFFD_FEATURE_WP_UNPOPULATED | UFFD_FEATURE_WP_ASYNC | - UFFD_FEATURE_WP_HUGETLBFS_SHMEM; - if (ioctl(uffd, UFFDIO_API, &uffdio_api)) - return -1; - - if (!(uffdio_api.api & UFFDIO_REGISTER_MODE_WP) || - !(uffdio_api.features & UFFD_FEATURE_WP_UNPOPULATED) || - !(uffdio_api.features & UFFD_FEATURE_WP_ASYNC) || - !(uffdio_api.features & UFFD_FEATURE_WP_HUGETLBFS_SHMEM)) - return -1; - - return 0; -} - -int wp_init(void *lpBaseAddress, int dwRegionSize) -{ - struct uffdio_register uffdio_register; - struct uffdio_writeprotect wp; - - uffdio_register.range.start = (unsigned long)lpBaseAddress; - uffdio_register.range.len = dwRegionSize; - uffdio_register.mode = UFFDIO_REGISTER_MODE_WP; - if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register)) - ksft_exit_fail_msg("ioctl(UFFDIO_REGISTER) %d %s\n", errno, strerror(errno)); - - if (!(uffdio_register.ioctls & UFFDIO_WRITEPROTECT)) - ksft_exit_fail_msg("ioctl set is incorrect\n"); - - wp.range.start = (unsigned long)lpBaseAddress; - wp.range.len = dwRegionSize; - wp.mode = UFFDIO_WRITEPROTECT_MODE_WP; - - if (ioctl(uffd, UFFDIO_WRITEPROTECT, &wp)) - ksft_exit_fail_msg("ioctl(UFFDIO_WRITEPROTECT)\n"); - - return 0; -} - -int wp_free(void *lpBaseAddress, int dwRegionSize) -{ - struct uffdio_register uffdio_register; - - uffdio_register.range.start = (unsigned long)lpBaseAddress; - uffdio_register.range.len = dwRegionSize; - uffdio_register.mode = UFFDIO_REGISTER_MODE_WP; - if (ioctl(uffd, UFFDIO_UNREGISTER, &uffdio_register.range)) - ksft_exit_fail_msg("ioctl unregister failure\n"); - return 0; -} - -int wp_addr_range(void *lpBaseAddress, int dwRegionSize) -{ - if (pagemap_ioctl(lpBaseAddress, dwRegionSize, NULL, 0, - PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, - 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) < 0) - ksft_exit_fail_msg("error %d %d %s\n", 1, errno, strerror(errno)); - - return 0; -} - -void *gethugetlb_mem(int size, int *shmid) -{ - char *mem; - - if (shmid) { - *shmid = shmget(2, size, SHM_HUGETLB | IPC_CREAT | SHM_R | SHM_W); - if (*shmid < 0) - return NULL; - - mem = shmat(*shmid, 0, 0); - if (mem == (char *)-1) { - shmctl(*shmid, IPC_RMID, NULL); - ksft_exit_fail_msg("Shared memory attach failure\n"); - } - } else { - mem = mmap(NULL, size, PROT_READ | PROT_WRITE, - MAP_ANONYMOUS | MAP_HUGETLB | MAP_PRIVATE, -1, 0); - if (mem == MAP_FAILED) - return NULL; - } - - return mem; -} - -int userfaultfd_tests(void) -{ - int mem_size, vec_size, written, num_pages = 16; - char *mem, *vec; - - mem_size = num_pages * page_size; - mem = mmap(NULL, mem_size, PROT_NONE, MAP_PRIVATE | MAP_ANON, -1, 0); - if (mem == MAP_FAILED) - ksft_exit_fail_msg("error nomem\n"); - - wp_init(mem, mem_size); - - /* Change protection of pages differently */ - mprotect(mem, mem_size/8, PROT_READ|PROT_WRITE); - mprotect(mem + 1 * mem_size/8, mem_size/8, PROT_READ); - mprotect(mem + 2 * mem_size/8, mem_size/8, PROT_READ|PROT_WRITE); - mprotect(mem + 3 * mem_size/8, mem_size/8, PROT_READ); - mprotect(mem + 4 * mem_size/8, mem_size/8, PROT_READ|PROT_WRITE); - mprotect(mem + 5 * mem_size/8, mem_size/8, PROT_NONE); - mprotect(mem + 6 * mem_size/8, mem_size/8, PROT_READ|PROT_WRITE); - mprotect(mem + 7 * mem_size/8, mem_size/8, PROT_READ); - - wp_addr_range(mem + (mem_size/16), mem_size - 2 * (mem_size/8)); - wp_addr_range(mem, mem_size); - - vec_size = mem_size/page_size; - vec = malloc(sizeof(struct page_region) * vec_size); - - written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, - vec_size - 2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); - if (written < 0) - ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); - - ksft_test_result(written == 0, "%s all new pages must not be written (dirty)\n", __func__); - - wp_free(mem, mem_size); - munmap(mem, mem_size); - free(vec); - return 0; -} - -int get_reads(struct page_region *vec, int vec_size) -{ - int i, sum = 0; - - for (i = 0; i < vec_size; i++) - sum += LEN(vec[i]); - - return sum; -} - -int sanity_tests_sd(void) -{ - int mem_size, vec_size, ret, ret2, ret3, i, num_pages = 1000, total_pages = 0; - int total_writes, total_reads, reads, count; - struct page_region *vec, *vec2; - char *mem, *m[2]; - long walk_end; - - vec_size = num_pages/2; - mem_size = num_pages * page_size; - - vec = malloc(sizeof(struct page_region) * vec_size); - if (!vec) - ksft_exit_fail_msg("error nomem\n"); - - vec2 = malloc(sizeof(struct page_region) * vec_size); - if (!vec2) - ksft_exit_fail_msg("error nomem\n"); - - mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); - if (mem == MAP_FAILED) - ksft_exit_fail_msg("error nomem\n"); - - wp_init(mem, mem_size); - wp_addr_range(mem, mem_size); - - /* 1. wrong operation */ - ksft_test_result(pagemap_ioctl(mem, 0, vec, vec_size, 0, - 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) == 0, - "%s Zero range size is valid\n", __func__); - - ksft_test_result(pagemap_ioctl(mem, mem_size, NULL, vec_size, 0, - 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) < 0, - "%s output buffer must be specified with size\n", __func__); - - ksft_test_result(pagemap_ioctl(mem, mem_size, vec, 0, 0, - 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) == 0, - "%s output buffer can be 0\n", __func__); - - ksft_test_result(pagemap_ioctl(mem, mem_size, 0, 0, 0, - 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) == 0, - "%s output buffer can be 0\n", __func__); - - ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, -1, - 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) < 0, - "%s wrong flag specified\n", __func__); - - ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, - PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC | 0xFF, - 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) < 0, - "%s flag has extra bits specified\n", __func__); - - ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, - 0, 0, 0, 0, PAGE_IS_WRITTEN) >= 0, - "%s no selection mask is specified\n", __func__); - - ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, - 0, PAGE_IS_WRITTEN, PAGE_IS_WRITTEN, 0, 0) == 0, - "%s no return mask is specified\n", __func__); - - ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, - 0, PAGE_IS_WRITTEN, 0, 0, 0x1000) < 0, - "%s wrong return mask specified\n", __func__); - - ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, - PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, - 0, 0xFFF, PAGE_IS_WRITTEN, 0, PAGE_IS_WRITTEN) < 0, - "%s mixture of correct and wrong flag\n", __func__); - - ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, - PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, - 0, 0, 0, PAGEMAP_BITS_ALL, PAGE_IS_WRITTEN) >= 0, - "%s PAGEMAP_BITS_ALL can be specified with PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC\n", - __func__); - - /* 2. Clear area with larger vec size */ - ret = pagemap_ioctl(mem, mem_size, vec, vec_size, - PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 0, - PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); - ksft_test_result(ret >= 0, "%s Clear area with larger vec size\n", __func__); - - /* 3. Repeated pattern of written and non-written pages */ - for (i = 0; i < mem_size; i += 2 * page_size) - mem[i]++; - - ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, PAGE_IS_WRITTEN, 0, - 0, PAGE_IS_WRITTEN); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - - ksft_test_result(ret == mem_size/(page_size * 2), - "%s Repeated pattern of written and non-written pages\n", __func__); - - /* 4. Repeated pattern of written and non-written pages in parts */ - ret = pagemap_ioctl(mem, mem_size, vec, vec_size, - PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, - num_pages/2 - 2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - - ret2 = pagemap_ioctl(mem, mem_size, vec, 2, 0, 0, PAGE_IS_WRITTEN, 0, 0, - PAGE_IS_WRITTEN); - if (ret2 < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret2, errno, strerror(errno)); - - ret3 = pagemap_ioctl(mem, mem_size, vec, vec_size, - PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, - 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); - if (ret3 < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret3, errno, strerror(errno)); - - ksft_test_result((ret + ret3) == num_pages/2 && ret2 == 2, - "%s Repeated pattern of written and non-written pages in parts %d %d %d\n", - __func__, ret, ret3, ret2); - - /* 5. Repeated pattern of written and non-written pages max_pages */ - for (i = 0; i < mem_size; i += 2 * page_size) - mem[i]++; - mem[(mem_size/page_size - 1) * page_size]++; - - ret = pagemap_ioctl(mem, mem_size, vec, vec_size, - PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, - num_pages/2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - - ret2 = pagemap_ioctl(mem, mem_size, vec, vec_size, - PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, - 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); - if (ret2 < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret2, errno, strerror(errno)); - - ksft_test_result(ret == num_pages/2 && ret2 == 1, - "%s Repeated pattern of written and non-written pages max_pages\n", - __func__); - - /* 6. only get 2 dirty pages and clear them as well */ - vec_size = mem_size/page_size; - memset(mem, -1, mem_size); - - /* get and clear second and third pages */ - ret = pagemap_ioctl(mem + page_size, 2 * page_size, vec, 1, - PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, - 2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - - ret2 = pagemap_ioctl(mem, mem_size, vec2, vec_size, 0, 0, - PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); - if (ret2 < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret2, errno, strerror(errno)); - - ksft_test_result(ret == 1 && LEN(vec[0]) == 2 && - vec[0].start == (uintptr_t)(mem + page_size) && - ret2 == 2 && LEN(vec2[0]) == 1 && vec2[0].start == (uintptr_t)mem && - LEN(vec2[1]) == vec_size - 3 && - vec2[1].start == (uintptr_t)(mem + 3 * page_size), - "%s only get 2 written pages and clear them as well\n", __func__); - - wp_free(mem, mem_size); - munmap(mem, mem_size); - - /* 7. Two regions */ - m[0] = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); - if (m[0] == MAP_FAILED) - ksft_exit_fail_msg("error nomem\n"); - m[1] = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); - if (m[1] == MAP_FAILED) - ksft_exit_fail_msg("error nomem\n"); - - wp_init(m[0], mem_size); - wp_init(m[1], mem_size); - wp_addr_range(m[0], mem_size); - wp_addr_range(m[1], mem_size); - - memset(m[0], 'a', mem_size); - memset(m[1], 'b', mem_size); - - wp_addr_range(m[0], mem_size); - - ret = pagemap_ioctl(m[1], mem_size, vec, 1, 0, 0, PAGE_IS_WRITTEN, 0, 0, - PAGE_IS_WRITTEN); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - - ksft_test_result(ret == 1 && LEN(vec[0]) == mem_size/page_size, - "%s Two regions\n", __func__); - - wp_free(m[0], mem_size); - wp_free(m[1], mem_size); - munmap(m[0], mem_size); - munmap(m[1], mem_size); - - free(vec); - free(vec2); - - /* 8. Smaller vec */ - mem_size = 1050 * page_size; - vec_size = mem_size/(page_size*2); - - vec = malloc(sizeof(struct page_region) * vec_size); - if (!vec) - ksft_exit_fail_msg("error nomem\n"); - - mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); - if (mem == MAP_FAILED) - ksft_exit_fail_msg("error nomem\n"); - - wp_init(mem, mem_size); - wp_addr_range(mem, mem_size); - - ret = pagemap_ioctl(mem, mem_size, vec, vec_size, - PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 0, - PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - - for (i = 0; i < mem_size/page_size; i += 2) - mem[i * page_size]++; - - ret = pagemap_ioctl(mem, mem_size, vec, vec_size, - PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, - mem_size/(page_size*5), PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - - total_pages += ret; - - ret = pagemap_ioctl(mem, mem_size, vec, vec_size, - PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, - mem_size/(page_size*5), PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - - total_pages += ret; - - ret = pagemap_ioctl(mem, mem_size, vec, vec_size, - PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, - mem_size/(page_size*5), PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - - total_pages += ret; - - ksft_test_result(total_pages == mem_size/(page_size*2), "%s Smaller max_pages\n", __func__); - - free(vec); - wp_free(mem, mem_size); - munmap(mem, mem_size); - total_pages = 0; - - /* 9. Smaller vec */ - mem_size = 10000 * page_size; - vec_size = 50; - - vec = malloc(sizeof(struct page_region) * vec_size); - if (!vec) - ksft_exit_fail_msg("error nomem\n"); - - mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); - if (mem == MAP_FAILED) - ksft_exit_fail_msg("error nomem\n"); - - wp_init(mem, mem_size); - wp_addr_range(mem, mem_size); - - for (count = 0; count < TEST_ITERATIONS; count++) { - total_writes = total_reads = 0; - walk_end = (long)mem; - - for (i = 0; i < mem_size; i += page_size) { - if (rand() % 2) { - mem[i]++; - total_writes++; - } - } - - while (total_reads < total_writes) { - ret = pagemap_ioc((void *)walk_end, mem_size-(walk_end - (long)mem), vec, - vec_size, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, - 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - - if (ret > vec_size) - break; - - reads = get_reads(vec, ret); - total_reads += reads; - } - - if (total_reads != total_writes) - break; - } - - ksft_test_result(count == TEST_ITERATIONS, "Smaller vec\n"); - - free(vec); - wp_free(mem, mem_size); - munmap(mem, mem_size); - - /* 10. Walk_end tester */ - vec_size = 1000; - mem_size = vec_size * page_size; - - vec = malloc(sizeof(struct page_region) * vec_size); - if (!vec) - ksft_exit_fail_msg("error nomem\n"); - - mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); - if (mem == MAP_FAILED) - ksft_exit_fail_msg("error nomem\n"); - - wp_init(mem, mem_size); - wp_addr_range(mem, mem_size); - - memset(mem, 0, mem_size); - - ret = pagemap_ioc(mem, 0, vec, vec_size, 0, - 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - ksft_test_result(ret == 0 && walk_end == (long)mem, - "Walk_end: Same start and end address\n"); - - ret = pagemap_ioc(mem, 0, vec, vec_size, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, - 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - ksft_test_result(ret == 0 && walk_end == (long)mem, - "Walk_end: Same start and end with WP\n"); - - ret = pagemap_ioc(mem, 0, vec, 0, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, - 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - ksft_test_result(ret == 0 && walk_end == (long)mem, - "Walk_end: Same start and end with 0 output buffer\n"); - - ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, - 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size), - "Walk_end: Big vec\n"); - - ret = pagemap_ioc(mem, mem_size, vec, 1, 0, - 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size), - "Walk_end: vec of minimum length\n"); - - ret = pagemap_ioc(mem, mem_size, vec, 1, 0, - vec_size, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size), - "Walk_end: Max pages specified\n"); - - ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, - vec_size/2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size/2), - "Walk_end: Half max pages\n"); - - ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, - 1, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size), - "Walk_end: 1 max page\n"); - - ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, - -1, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size), - "Walk_end: max pages\n"); - - wp_addr_range(mem, mem_size); - for (i = 0; i < mem_size; i += 2 * page_size) - mem[i]++; - - ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, - 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - ksft_test_result(ret == vec_size/2 && walk_end == (long)(mem + mem_size), - "Walk_end sparse: Big vec\n"); - - ret = pagemap_ioc(mem, mem_size, vec, 1, 0, - 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size * 2), - "Walk_end sparse: vec of minimum length\n"); - - ret = pagemap_ioc(mem, mem_size, vec, 1, 0, - vec_size, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size * 2), - "Walk_end sparse: Max pages specified\n"); - - ret = pagemap_ioc(mem, mem_size, vec, vec_size/2, 0, - vec_size, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - ksft_test_result(ret == vec_size/2 && walk_end == (long)(mem + mem_size), - "Walk_end sparse: Max pages specified\n"); - - ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, - vec_size, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - ksft_test_result(ret == vec_size/2 && walk_end == (long)(mem + mem_size), - "Walk_end sparse: Max pages specified\n"); - - ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, - vec_size/2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - ksft_test_result(ret == vec_size/2 && walk_end == (long)(mem + mem_size), - "Walk_endsparse : Half max pages\n"); - - ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0, - 1, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size * 2), - "Walk_end: 1 max page\n"); - - free(vec); - wp_free(mem, mem_size); - munmap(mem, mem_size); - - return 0; -} - -int base_tests(char *prefix, char *mem, int mem_size, int skip) -{ - int vec_size, written; - struct page_region *vec, *vec2; - - if (skip) { - ksft_test_result_skip("%s all new pages must not be written (dirty)\n", prefix); - ksft_test_result_skip("%s all pages must be written (dirty)\n", prefix); - ksft_test_result_skip("%s all pages dirty other than first and the last one\n", - prefix); - ksft_test_result_skip("%s PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC\n", prefix); - ksft_test_result_skip("%s only middle page dirty\n", prefix); - ksft_test_result_skip("%s only two middle pages dirty\n", prefix); - return 0; - } - - vec_size = mem_size/page_size; - vec = malloc(sizeof(struct page_region) * vec_size); - vec2 = malloc(sizeof(struct page_region) * vec_size); - - /* 1. all new pages must be not be written (dirty) */ - written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, - vec_size - 2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); - if (written < 0) - ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); - - ksft_test_result(written == 0, "%s all new pages must not be written (dirty)\n", prefix); - - /* 2. all pages must be written */ - memset(mem, -1, mem_size); - - written = pagemap_ioctl(mem, mem_size, vec, 1, 0, 0, PAGE_IS_WRITTEN, 0, 0, - PAGE_IS_WRITTEN); - if (written < 0) - ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); - - ksft_test_result(written == 1 && LEN(vec[0]) == mem_size/page_size, - "%s all pages must be written (dirty)\n", prefix); - - /* 3. all pages dirty other than first and the last one */ - written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, - 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); - if (written < 0) - ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); - - memset(mem + page_size, 0, mem_size - (2 * page_size)); - - written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, - 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); - if (written < 0) - ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); - - ksft_test_result(written == 1 && LEN(vec[0]) >= vec_size - 2 && LEN(vec[0]) <= vec_size, - "%s all pages dirty other than first and the last one\n", prefix); - - written = pagemap_ioctl(mem, mem_size, vec, 1, 0, 0, - PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); - if (written < 0) - ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); - - ksft_test_result(written == 0, - "%s PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC\n", prefix); - - /* 4. only middle page dirty */ - written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, - 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); - if (written < 0) - ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); - - mem[vec_size/2 * page_size]++; - - written = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, PAGE_IS_WRITTEN, - 0, 0, PAGE_IS_WRITTEN); - if (written < 0) - ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); - - ksft_test_result(written == 1 && LEN(vec[0]) >= 1, - "%s only middle page dirty\n", prefix); - - /* 5. only two middle pages dirty and walk over only middle pages */ - written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, - 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN | PAGE_IS_HUGE); - if (written < 0) - ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); - - mem[vec_size/2 * page_size]++; - mem[(vec_size/2 + 1) * page_size]++; - - written = pagemap_ioctl(&mem[vec_size/2 * page_size], 2 * page_size, vec, 1, 0, - 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN | PAGE_IS_HUGE); - if (written < 0) - ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); - - ksft_test_result(written == 1 && vec[0].start == (uintptr_t)(&mem[vec_size/2 * page_size]) - && LEN(vec[0]) == 2, - "%s only two middle pages dirty\n", prefix); - - free(vec); - free(vec2); - return 0; -} - -void *gethugepage(int map_size) -{ - int ret; - char *map; - - map = memalign(hpage_size, map_size); - if (!map) - ksft_exit_fail_msg("memalign failed %d %s\n", errno, strerror(errno)); - - ret = madvise(map, map_size, MADV_HUGEPAGE); - if (ret) - return NULL; - - memset(map, 0, map_size); - - return map; -} - -int hpage_unit_tests(void) -{ - char *map; - int ret, ret2; - size_t num_pages = 10; - int map_size = hpage_size * num_pages; - int vec_size = map_size/page_size; - struct page_region *vec, *vec2; - - vec = malloc(sizeof(struct page_region) * vec_size); - vec2 = malloc(sizeof(struct page_region) * vec_size); - if (!vec || !vec2) - ksft_exit_fail_msg("malloc failed\n"); - - map = gethugepage(map_size); - if (map) { - wp_init(map, map_size); - wp_addr_range(map, map_size); - - /* 1. all new huge page must not be written (dirty) */ - ret = pagemap_ioctl(map, map_size, vec, vec_size, - PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 0, - PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - - ksft_test_result(ret == 0, "%s all new huge page must not be written (dirty)\n", - __func__); - - /* 2. all the huge page must not be written */ - ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0, - PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - - ksft_test_result(ret == 0, "%s all the huge page must not be written\n", __func__); - - /* 3. all the huge page must be written and clear dirty as well */ - memset(map, -1, map_size); - ret = pagemap_ioctl(map, map_size, vec, vec_size, - PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, - 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - - ksft_test_result(ret == 1 && vec[0].start == (uintptr_t)map && - LEN(vec[0]) == vec_size && vec[0].categories == PAGE_IS_WRITTEN, - "%s all the huge page must be written and clear\n", __func__); - - /* 4. only middle page written */ - wp_free(map, map_size); - free(map); - map = gethugepage(map_size); - wp_init(map, map_size); - wp_addr_range(map, map_size); - map[vec_size/2 * page_size]++; - - ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0, - PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - - ksft_test_result(ret == 1 && LEN(vec[0]) > 0, - "%s only middle page written\n", __func__); - - wp_free(map, map_size); - free(map); - } else { - ksft_test_result_skip("%s all new huge page must be written\n", __func__); - ksft_test_result_skip("%s all the huge page must not be written\n", __func__); - ksft_test_result_skip("%s all the huge page must be written and clear\n", __func__); - ksft_test_result_skip("%s only middle page written\n", __func__); - } - - /* 5. clear first half of huge page */ - map = gethugepage(map_size); - if (map) { - wp_init(map, map_size); - wp_addr_range(map, map_size); - - memset(map, 0, map_size); - - wp_addr_range(map, map_size/2); - - ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0, - PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - - ksft_test_result(ret == 1 && LEN(vec[0]) == vec_size/2 && - vec[0].start == (uintptr_t)(map + map_size/2), - "%s clear first half of huge page\n", __func__); - wp_free(map, map_size); - free(map); - } else { - ksft_test_result_skip("%s clear first half of huge page\n", __func__); - } - - /* 6. clear first half of huge page with limited buffer */ - map = gethugepage(map_size); - if (map) { - wp_init(map, map_size); - wp_addr_range(map, map_size); - - memset(map, 0, map_size); - - ret = pagemap_ioctl(map, map_size, vec, vec_size, - PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, - vec_size/2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - - ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0, - PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - - ksft_test_result(ret == 1 && LEN(vec[0]) == vec_size/2 && - vec[0].start == (uintptr_t)(map + map_size/2), - "%s clear first half of huge page with limited buffer\n", - __func__); - wp_free(map, map_size); - free(map); - } else { - ksft_test_result_skip("%s clear first half of huge page with limited buffer\n", - __func__); - } - - /* 7. clear second half of huge page */ - map = gethugepage(map_size); - if (map) { - wp_init(map, map_size); - wp_addr_range(map, map_size); - - memset(map, -1, map_size); - - ret = pagemap_ioctl(map + map_size/2, map_size/2, vec, vec_size, - PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, vec_size/2, - PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - - ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0, - PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - - ksft_test_result(ret == 1 && LEN(vec[0]) == vec_size/2, - "%s clear second half huge page\n", __func__); - wp_free(map, map_size); - free(map); - } else { - ksft_test_result_skip("%s clear second half huge page\n", __func__); - } - - /* 8. get half huge page */ - map = gethugepage(map_size); - if (map) { - wp_init(map, map_size); - wp_addr_range(map, map_size); - - memset(map, -1, map_size); - usleep(100); - - ret = pagemap_ioctl(map, map_size, vec, 1, - PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, - hpage_size/(2*page_size), PAGE_IS_WRITTEN, 0, 0, - PAGE_IS_WRITTEN); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - - ksft_test_result(ret == 1 && LEN(vec[0]) == hpage_size/(2*page_size), - "%s get half huge page\n", __func__); - - ret2 = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0, - PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); - if (ret2 < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret2, errno, strerror(errno)); - - ksft_test_result(ret2 == 1 && LEN(vec[0]) == (map_size - hpage_size/2)/page_size, - "%s get half huge page\n", __func__); - - wp_free(map, map_size); - free(map); - } else { - ksft_test_result_skip("%s get half huge page\n", __func__); - ksft_test_result_skip("%s get half huge page\n", __func__); - } - - free(vec); - free(vec2); - return 0; -} - -int unmapped_region_tests(void) -{ - void *start = (void *)0x10000000; - int written, len = 0x00040000; - int vec_size = len / page_size; - struct page_region *vec = malloc(sizeof(struct page_region) * vec_size); - - /* 1. Get written pages */ - written = pagemap_ioctl(start, len, vec, vec_size, 0, 0, - PAGEMAP_NON_WRITTEN_BITS, 0, 0, PAGEMAP_NON_WRITTEN_BITS); - if (written < 0) - ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno)); - - ksft_test_result(written >= 0, "%s Get status of pages\n", __func__); - - free(vec); - return 0; -} - -static void test_simple(void) -{ - int i; - char *map; - struct page_region vec; - - map = aligned_alloc(page_size, page_size); - if (!map) - ksft_exit_fail_msg("aligned_alloc failed\n"); - - wp_init(map, page_size); - wp_addr_range(map, page_size); - - for (i = 0 ; i < TEST_ITERATIONS; i++) { - if (pagemap_ioctl(map, page_size, &vec, 1, 0, 0, - PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) == 1) { - ksft_print_msg("written bit was 1, but should be 0 (i=%d)\n", i); - break; - } - - wp_addr_range(map, page_size); - /* Write something to the page to get the written bit enabled on the page */ - map[0]++; - - if (pagemap_ioctl(map, page_size, &vec, 1, 0, 0, - PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) == 0) { - ksft_print_msg("written bit was 0, but should be 1 (i=%d)\n", i); - break; - } - - wp_addr_range(map, page_size); - } - wp_free(map, page_size); - free(map); - - ksft_test_result(i == TEST_ITERATIONS, "Test %s\n", __func__); -} - -int sanity_tests(void) -{ - int mem_size, vec_size, ret, fd, i, buf_size; - struct page_region *vec; - char *mem, *fmem; - struct stat sbuf; - char *tmp_buf; - - /* 1. wrong operation */ - mem_size = 10 * page_size; - vec_size = mem_size / page_size; - - vec = malloc(sizeof(struct page_region) * vec_size); - mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); - if (mem == MAP_FAILED || vec == MAP_FAILED) - ksft_exit_fail_msg("error nomem\n"); - - wp_init(mem, mem_size); - wp_addr_range(mem, mem_size); - - ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, - PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, - 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) >= 0, - "%s WP op can be specified with !PAGE_IS_WRITTEN\n", __func__); - ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, - PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) >= 0, - "%s required_mask specified\n", __func__); - ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, - 0, PAGEMAP_BITS_ALL, 0, PAGEMAP_BITS_ALL) >= 0, - "%s anyof_mask specified\n", __func__); - ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, - 0, 0, PAGEMAP_BITS_ALL, PAGEMAP_BITS_ALL) >= 0, - "%s excluded_mask specified\n", __func__); - ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, - PAGEMAP_BITS_ALL, PAGEMAP_BITS_ALL, 0, - PAGEMAP_BITS_ALL) >= 0, - "%s required_mask and anyof_mask specified\n", __func__); - wp_free(mem, mem_size); - munmap(mem, mem_size); - - /* 2. Get sd and present pages with anyof_mask */ - mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); - if (mem == MAP_FAILED) - ksft_exit_fail_msg("error nomem\n"); - wp_init(mem, mem_size); - wp_addr_range(mem, mem_size); - - memset(mem, 0, mem_size); - - ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, - 0, PAGEMAP_BITS_ALL, 0, PAGEMAP_BITS_ALL); - ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size && - (vec[0].categories & (PAGE_IS_WRITTEN | PAGE_IS_PRESENT)) == - (PAGE_IS_WRITTEN | PAGE_IS_PRESENT), - "%s Get sd and present pages with anyof_mask\n", __func__); - - /* 3. Get sd and present pages with required_mask */ - ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, - PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL); - ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size && - (vec[0].categories & (PAGE_IS_WRITTEN | PAGE_IS_PRESENT)) == - (PAGE_IS_WRITTEN | PAGE_IS_PRESENT), - "%s Get all the pages with required_mask\n", __func__); - - /* 4. Get sd and present pages with required_mask and anyof_mask */ - ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, - PAGE_IS_WRITTEN, PAGE_IS_PRESENT, 0, PAGEMAP_BITS_ALL); - ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size && - (vec[0].categories & (PAGE_IS_WRITTEN | PAGE_IS_PRESENT)) == - (PAGE_IS_WRITTEN | PAGE_IS_PRESENT), - "%s Get sd and present pages with required_mask and anyof_mask\n", - __func__); - - /* 5. Don't get sd pages */ - ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, - PAGE_IS_WRITTEN, 0, PAGE_IS_WRITTEN, PAGEMAP_BITS_ALL); - ksft_test_result(ret == 0, "%s Don't get sd pages\n", __func__); - - /* 6. Don't get present pages */ - ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, - PAGE_IS_PRESENT, 0, PAGE_IS_PRESENT, PAGEMAP_BITS_ALL); - ksft_test_result(ret == 0, "%s Don't get present pages\n", __func__); - - wp_free(mem, mem_size); - munmap(mem, mem_size); - - /* 8. Find written present pages with return mask */ - mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); - if (mem == MAP_FAILED) - ksft_exit_fail_msg("error nomem\n"); - wp_init(mem, mem_size); - wp_addr_range(mem, mem_size); - - memset(mem, 0, mem_size); - - ret = pagemap_ioctl(mem, mem_size, vec, vec_size, - PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 0, - 0, PAGEMAP_BITS_ALL, 0, PAGE_IS_WRITTEN); - ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size && - vec[0].categories == PAGE_IS_WRITTEN, - "%s Find written present pages with return mask\n", __func__); - wp_free(mem, mem_size); - munmap(mem, mem_size); - - /* 9. Memory mapped file */ - fd = open(__FILE__, O_RDONLY); - if (fd < 0) - ksft_exit_fail_msg("%s Memory mapped file\n", __func__); - - ret = stat(__FILE__, &sbuf); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - - fmem = mmap(NULL, sbuf.st_size, PROT_READ, MAP_PRIVATE, fd, 0); - if (fmem == MAP_FAILED) - ksft_exit_fail_msg("error nomem %d %s\n", errno, strerror(errno)); - - tmp_buf = malloc(sbuf.st_size); - memcpy(tmp_buf, fmem, sbuf.st_size); - - ret = pagemap_ioctl(fmem, sbuf.st_size, vec, vec_size, 0, 0, - 0, PAGEMAP_NON_WRITTEN_BITS, 0, PAGEMAP_NON_WRITTEN_BITS); - - ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)fmem && - LEN(vec[0]) == ceilf((float)sbuf.st_size/page_size) && - (vec[0].categories & PAGE_IS_FILE), - "%s Memory mapped file\n", __func__); - - munmap(fmem, sbuf.st_size); - close(fd); - - /* 10. Create and read/write to a memory mapped file */ - buf_size = page_size * 10; - - fd = open(__FILE__".tmp2", O_RDWR | O_CREAT, 0666); - if (fd < 0) - ksft_exit_fail_msg("Read/write to memory: %s\n", - strerror(errno)); - - for (i = 0; i < buf_size; i++) - if (write(fd, "c", 1) < 0) - ksft_exit_fail_msg("Create and read/write to a memory mapped file\n"); - - fmem = mmap(NULL, buf_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); - if (fmem == MAP_FAILED) - ksft_exit_fail_msg("error nomem %d %s\n", errno, strerror(errno)); - - wp_init(fmem, buf_size); - wp_addr_range(fmem, buf_size); - - for (i = 0; i < buf_size; i++) - fmem[i] = 'z'; - - msync(fmem, buf_size, MS_SYNC); - - ret = pagemap_ioctl(fmem, buf_size, vec, vec_size, 0, 0, - PAGE_IS_WRITTEN, PAGE_IS_PRESENT | PAGE_IS_SWAPPED | PAGE_IS_FILE, 0, - PAGEMAP_BITS_ALL); - - ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)fmem && - LEN(vec[0]) == (buf_size/page_size) && - (vec[0].categories & PAGE_IS_WRITTEN), - "%s Read/write to memory\n", __func__); - - wp_free(fmem, buf_size); - munmap(fmem, buf_size); - close(fd); - - free(vec); - return 0; -} - -int mprotect_tests(void) -{ - int ret; - char *mem, *mem2; - struct page_region vec; - int pagemap_fd = open("/proc/self/pagemap", O_RDONLY); - - if (pagemap_fd < 0) { - fprintf(stderr, "open() failed\n"); - exit(1); - } - - /* 1. Map two pages */ - mem = mmap(0, 2 * page_size, PROT_READ|PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); - if (mem == MAP_FAILED) - ksft_exit_fail_msg("error nomem\n"); - wp_init(mem, 2 * page_size); - wp_addr_range(mem, 2 * page_size); - - /* Populate both pages. */ - memset(mem, 1, 2 * page_size); - - ret = pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0, PAGE_IS_WRITTEN, - 0, 0, PAGE_IS_WRITTEN); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - - ksft_test_result(ret == 1 && LEN(vec) == 2, "%s Both pages written\n", __func__); - - /* 2. Start tracking */ - wp_addr_range(mem, 2 * page_size); - - ksft_test_result(pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0, - PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) == 0, - "%s Both pages are not written (dirty)\n", __func__); - - /* 3. Remap the second page */ - mem2 = mmap(mem + page_size, page_size, PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANON|MAP_FIXED, -1, 0); - if (mem2 == MAP_FAILED) - ksft_exit_fail_msg("error nomem\n"); - wp_init(mem2, page_size); - wp_addr_range(mem2, page_size); - - /* Protect + unprotect. */ - mprotect(mem, page_size, PROT_NONE); - mprotect(mem, 2 * page_size, PROT_READ); - mprotect(mem, 2 * page_size, PROT_READ|PROT_WRITE); - - /* Modify both pages. */ - memset(mem, 2, 2 * page_size); - - /* Protect + unprotect. */ - mprotect(mem, page_size, PROT_NONE); - mprotect(mem, page_size, PROT_READ); - mprotect(mem, page_size, PROT_READ|PROT_WRITE); - - ret = pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0, PAGE_IS_WRITTEN, - 0, 0, PAGE_IS_WRITTEN); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - - ksft_test_result(ret == 1 && LEN(vec) == 2, - "%s Both pages written after remap and mprotect\n", __func__); - - /* 4. Clear and make the pages written */ - wp_addr_range(mem, 2 * page_size); - - memset(mem, 'A', 2 * page_size); - - ret = pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0, PAGE_IS_WRITTEN, - 0, 0, PAGE_IS_WRITTEN); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - - ksft_test_result(ret == 1 && LEN(vec) == 2, - "%s Clear and make the pages written\n", __func__); - - wp_free(mem, 2 * page_size); - munmap(mem, 2 * page_size); - return 0; -} - -/* transact test */ -static const unsigned int nthreads = 6, pages_per_thread = 32, access_per_thread = 8; -static pthread_barrier_t start_barrier, end_barrier; -static unsigned int extra_thread_faults; -static unsigned int iter_count = 1000; -static volatile int finish; - -static ssize_t get_dirty_pages_reset(char *mem, unsigned int count, - int reset, int page_size) -{ - struct pm_scan_arg arg = {0}; - struct page_region rgns[256]; - int i, j, cnt, ret; - - arg.size = sizeof(struct pm_scan_arg); - arg.start = (uintptr_t)mem; - arg.max_pages = count; - arg.end = (uintptr_t)(mem + count * page_size); - arg.vec = (uintptr_t)rgns; - arg.vec_len = sizeof(rgns) / sizeof(*rgns); - if (reset) - arg.flags |= PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC; - arg.category_mask = PAGE_IS_WRITTEN; - arg.return_mask = PAGE_IS_WRITTEN; - - ret = ioctl(pagemap_fd, PAGEMAP_SCAN, &arg); - if (ret < 0) - ksft_exit_fail_msg("ioctl failed\n"); - - cnt = 0; - for (i = 0; i < ret; ++i) { - if (rgns[i].categories != PAGE_IS_WRITTEN) - ksft_exit_fail_msg("wrong flags\n"); - - for (j = 0; j < LEN(rgns[i]); ++j) - cnt++; - } - - return cnt; -} - -void *thread_proc(void *mem) -{ - int *m = mem; - long curr_faults, faults; - struct rusage r; - unsigned int i; - int ret; - - if (getrusage(RUSAGE_THREAD, &r)) - ksft_exit_fail_msg("getrusage\n"); - - curr_faults = r.ru_minflt; - - while (!finish) { - ret = pthread_barrier_wait(&start_barrier); - if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD) - ksft_exit_fail_msg("pthread_barrier_wait\n"); - - for (i = 0; i < access_per_thread; ++i) - __atomic_add_fetch(m + i * (0x1000 / sizeof(*m)), 1, __ATOMIC_SEQ_CST); - - ret = pthread_barrier_wait(&end_barrier); - if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD) - ksft_exit_fail_msg("pthread_barrier_wait\n"); - - if (getrusage(RUSAGE_THREAD, &r)) - ksft_exit_fail_msg("getrusage\n"); - - faults = r.ru_minflt - curr_faults; - if (faults < access_per_thread) - ksft_exit_fail_msg("faults < access_per_thread"); - - __atomic_add_fetch(&extra_thread_faults, faults - access_per_thread, - __ATOMIC_SEQ_CST); - curr_faults = r.ru_minflt; - } - - return NULL; -} - -static void transact_test(int page_size) -{ - unsigned int i, count, extra_pages; - pthread_t th; - char *mem; - int ret, c; - - if (pthread_barrier_init(&start_barrier, NULL, nthreads + 1)) - ksft_exit_fail_msg("pthread_barrier_init\n"); - - if (pthread_barrier_init(&end_barrier, NULL, nthreads + 1)) - ksft_exit_fail_msg("pthread_barrier_init\n"); - - mem = mmap(NULL, 0x1000 * nthreads * pages_per_thread, PROT_READ | PROT_WRITE, - MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - if (mem == MAP_FAILED) - ksft_exit_fail_msg("Error mmap %s.\n", strerror(errno)); - - wp_init(mem, 0x1000 * nthreads * pages_per_thread); - wp_addr_range(mem, 0x1000 * nthreads * pages_per_thread); - - memset(mem, 0, 0x1000 * nthreads * pages_per_thread); - - count = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, page_size); - ksft_test_result(count > 0, "%s count %d\n", __func__, count); - count = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, page_size); - ksft_test_result(count == 0, "%s count %d\n", __func__, count); - - finish = 0; - for (i = 0; i < nthreads; ++i) - pthread_create(&th, NULL, thread_proc, mem + 0x1000 * i * pages_per_thread); - - extra_pages = 0; - for (i = 0; i < iter_count; ++i) { - count = 0; - - ret = pthread_barrier_wait(&start_barrier); - if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD) - ksft_exit_fail_msg("pthread_barrier_wait\n"); - - count = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, - page_size); - - ret = pthread_barrier_wait(&end_barrier); - if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD) - ksft_exit_fail_msg("pthread_barrier_wait\n"); - - if (count > nthreads * access_per_thread) - ksft_exit_fail_msg("Too big count %d expected %d, iter %d\n", - count, nthreads * access_per_thread, i); - - c = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, page_size); - count += c; - - if (c > nthreads * access_per_thread) { - ksft_test_result_fail(" %s count > nthreads\n", __func__); - return; - } - - if (count != nthreads * access_per_thread) { - /* - * The purpose of the test is to make sure that no page updates are lost - * when the page updates and read-resetting soft dirty flags are performed - * in parallel. However, it is possible that the application will get the - * soft dirty flags twice on the two consecutive read-resets. This seems - * unavoidable as soft dirty flag is handled in software through page faults - * in kernel. While the updating the flags is supposed to be synchronized - * between page fault handling and read-reset, it is possible that - * read-reset happens after page fault PTE update but before the application - * re-executes write instruction. So read-reset gets the flag, clears write - * access and application gets page fault again for the same write. - */ - if (count < nthreads * access_per_thread) { - ksft_test_result_fail("Lost update, iter %d, %d vs %d.\n", i, count, - nthreads * access_per_thread); - return; - } - - extra_pages += count - nthreads * access_per_thread; - } - } - - pthread_barrier_wait(&start_barrier); - finish = 1; - pthread_barrier_wait(&end_barrier); - - ksft_test_result_pass("%s Extra pages %u (%.1lf%%), extra thread faults %d.\n", __func__, - extra_pages, - 100.0 * extra_pages / (iter_count * nthreads * access_per_thread), - extra_thread_faults); -} - -int main(void) -{ - int mem_size, shmid, buf_size, fd, i, ret; - char *mem, *map, *fmem; - struct stat sbuf; - - ksft_print_header(); - - if (init_uffd()) - return ksft_exit_pass(); - - ksft_set_plan(115); - - page_size = getpagesize(); - hpage_size = read_pmd_pagesize(); - - pagemap_fd = open(PAGEMAP, O_RDONLY); - if (pagemap_fd < 0) - return -EINVAL; - - /* 1. Sanity testing */ - sanity_tests_sd(); - - /* 2. Normal page testing */ - mem_size = 10 * page_size; - mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); - if (mem == MAP_FAILED) - ksft_exit_fail_msg("error nomem\n"); - wp_init(mem, mem_size); - wp_addr_range(mem, mem_size); - - base_tests("Page testing:", mem, mem_size, 0); - - wp_free(mem, mem_size); - munmap(mem, mem_size); - - /* 3. Large page testing */ - mem_size = 512 * 10 * page_size; - mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); - if (mem == MAP_FAILED) - ksft_exit_fail_msg("error nomem\n"); - wp_init(mem, mem_size); - wp_addr_range(mem, mem_size); - - base_tests("Large Page testing:", mem, mem_size, 0); - - wp_free(mem, mem_size); - munmap(mem, mem_size); - - /* 4. Huge page testing */ - map = gethugepage(hpage_size); - if (map) { - wp_init(map, hpage_size); - wp_addr_range(map, hpage_size); - base_tests("Huge page testing:", map, hpage_size, 0); - wp_free(map, hpage_size); - free(map); - } else { - base_tests("Huge page testing:", NULL, 0, 1); - } - - /* 5. SHM Hugetlb page testing */ - mem_size = 2*1024*1024; - mem = gethugetlb_mem(mem_size, &shmid); - if (mem) { - wp_init(mem, mem_size); - wp_addr_range(mem, mem_size); - - base_tests("Hugetlb shmem testing:", mem, mem_size, 0); - - wp_free(mem, mem_size); - shmctl(shmid, IPC_RMID, NULL); - } else { - base_tests("Hugetlb shmem testing:", NULL, 0, 1); - } - - /* 6. Hugetlb page testing */ - mem = gethugetlb_mem(mem_size, NULL); - if (mem) { - wp_init(mem, mem_size); - wp_addr_range(mem, mem_size); - - base_tests("Hugetlb mem testing:", mem, mem_size, 0); - - wp_free(mem, mem_size); - } else { - base_tests("Hugetlb mem testing:", NULL, 0, 1); - } - - /* 7. File Hugetlb testing */ - mem_size = 2*1024*1024; - fd = memfd_create("uffd-test", MFD_HUGETLB | MFD_NOEXEC_SEAL); - mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); - if (mem) { - wp_init(mem, mem_size); - wp_addr_range(mem, mem_size); - - base_tests("Hugetlb shmem testing:", mem, mem_size, 0); - - wp_free(mem, mem_size); - shmctl(shmid, IPC_RMID, NULL); - } else { - base_tests("Hugetlb shmem testing:", NULL, 0, 1); - } - close(fd); - - /* 8. File memory testing */ - buf_size = page_size * 10; - - fd = open(__FILE__".tmp0", O_RDWR | O_CREAT, 0777); - if (fd < 0) - ksft_exit_fail_msg("Create and read/write to a memory mapped file: %s\n", - strerror(errno)); - - for (i = 0; i < buf_size; i++) - if (write(fd, "c", 1) < 0) - ksft_exit_fail_msg("Create and read/write to a memory mapped file\n"); - - ret = stat(__FILE__".tmp0", &sbuf); - if (ret < 0) - ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno)); - - fmem = mmap(NULL, sbuf.st_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); - if (fmem == MAP_FAILED) - ksft_exit_fail_msg("error nomem %d %s\n", errno, strerror(errno)); - - wp_init(fmem, sbuf.st_size); - wp_addr_range(fmem, sbuf.st_size); - - base_tests("File memory testing:", fmem, sbuf.st_size, 0); - - wp_free(fmem, sbuf.st_size); - munmap(fmem, sbuf.st_size); - close(fd); - - /* 9. File memory testing */ - buf_size = page_size * 10; - - fd = memfd_create(__FILE__".tmp00", MFD_NOEXEC_SEAL); - if (fd < 0) - ksft_exit_fail_msg("Create and read/write to a memory mapped file: %s\n", - strerror(errno)); - - if (ftruncate(fd, buf_size)) - ksft_exit_fail_msg("Error ftruncate\n"); - - for (i = 0; i < buf_size; i++) - if (write(fd, "c", 1) < 0) - ksft_exit_fail_msg("Create and read/write to a memory mapped file\n"); - - fmem = mmap(NULL, buf_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); - if (fmem == MAP_FAILED) - ksft_exit_fail_msg("error nomem %d %s\n", errno, strerror(errno)); - - wp_init(fmem, buf_size); - wp_addr_range(fmem, buf_size); - - base_tests("File anonymous memory testing:", fmem, buf_size, 0); - - wp_free(fmem, buf_size); - munmap(fmem, buf_size); - close(fd); - - /* 10. Huge page tests */ - hpage_unit_tests(); - - /* 11. Iterative test */ - test_simple(); - - /* 12. Mprotect test */ - mprotect_tests(); - - /* 13. Transact test */ - transact_test(page_size); - - /* 14. Sanity testing */ - sanity_tests(); - - /*15. Unmapped address test */ - unmapped_region_tests(); - - /* 16. Userfaultfd tests */ - userfaultfd_tests(); - - close(pagemap_fd); - return ksft_exit_pass(); -} diff --git a/tools/testing/selftests/mm/run_vmtests.sh b/tools/testing/selftests/mm/run_vmtests.sh index 00757445278e..35db08cb8ba8 100755 --- a/tools/testing/selftests/mm/run_vmtests.sh +++ b/tools/testing/selftests/mm/run_vmtests.sh @@ -56,8 +56,6 @@ separated by spaces: memory protection key tests - soft_dirty test soft dirty page bit semantics -- pagemap - test pagemap_scan IOCTL - cow test copy-on-write semantics - thp @@ -352,8 +350,6 @@ then CATEGORY="soft_dirty" run_test ./soft-dirty fi -CATEGORY="pagemap" run_test ./pagemap_ioctl - # COW tests CATEGORY="cow" run_test ./cow