lkml.org 
[lkml]   [2022]   [Nov]   [29]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[PATCH 25/43] mm: Switch vma_merge(), split_vma(), and __split_vma to vma iterator
    Date
    From: "Liam R. Howlett" <Liam.Howlett@Oracle.com>

    Drop the vmi_* functions and transition all users to use the vma
    iterator directly.

    Signed-off-by: Liam R. Howlett <Liam.Howlett@oracle.com>
    ---
    fs/userfaultfd.c | 14 ++++----
    include/linux/mm.h | 16 +++-------
    mm/madvise.c | 6 ++--
    mm/mempolicy.c | 6 ++--
    mm/mlock.c | 6 ++--
    mm/mmap.c | 79 +++++++++++++---------------------------------
    mm/mprotect.c | 6 ++--
    mm/mremap.c | 2 +-
    8 files changed, 47 insertions(+), 88 deletions(-)

    diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c
    index b3249388696a..e60f86d6b91c 100644
    --- a/fs/userfaultfd.c
    +++ b/fs/userfaultfd.c
    @@ -883,7 +883,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file)
    continue;
    }
    new_flags = vma->vm_flags & ~__VM_UFFD_FLAGS;
    - prev = vmi_vma_merge(&vmi, mm, prev, vma->vm_start, vma->vm_end,
    + prev = vma_merge(&vmi, mm, prev, vma->vm_start, vma->vm_end,
    new_flags, vma->anon_vma,
    vma->vm_file, vma->vm_pgoff,
    vma_policy(vma),
    @@ -1426,7 +1426,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
    vma_end = min(end, vma->vm_end);

    new_flags = (vma->vm_flags & ~__VM_UFFD_FLAGS) | vm_flags;
    - prev = vmi_vma_merge(&vmi, mm, prev, start, vma_end, new_flags,
    + prev = vma_merge(&vmi, mm, prev, start, vma_end, new_flags,
    vma->anon_vma, vma->vm_file, vma->vm_pgoff,
    vma_policy(vma),
    ((struct vm_userfaultfd_ctx){ ctx }),
    @@ -1437,12 +1437,12 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
    goto next;
    }
    if (vma->vm_start < start) {
    - ret = vmi_split_vma(&vmi, mm, vma, start, 1);
    + ret = split_vma(&vmi, vma, start, 1);
    if (ret)
    break;
    }
    if (vma->vm_end > end) {
    - ret = vmi_split_vma(&vmi, mm, vma, end, 0);
    + ret = split_vma(&vmi, vma, end, 0);
    if (ret)
    break;
    }
    @@ -1606,7 +1606,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
    uffd_wp_range(mm, vma, start, vma_end - start, false);

    new_flags = vma->vm_flags & ~__VM_UFFD_FLAGS;
    - prev = vmi_vma_merge(&vmi, mm, prev, start, vma_end, new_flags,
    + prev = vma_merge(&vmi, mm, prev, start, vma_end, new_flags,
    vma->anon_vma, vma->vm_file, vma->vm_pgoff,
    vma_policy(vma),
    NULL_VM_UFFD_CTX, anon_vma_name(vma));
    @@ -1615,13 +1615,13 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
    goto next;
    }
    if (vma->vm_start < start) {
    - ret = vmi_split_vma(&vmi, mm, vma, start, 1);
    + ret = split_vma(&vmi, vma, start, 1);
    if (ret)
    break;
    }
    if (vma->vm_end > end) {
    vma_iter_set(&vmi, vma->vm_end);
    - ret = vmi_split_vma(&vmi, mm, vma, end, 0);
    + ret = split_vma(&vmi, vma, end, 0);
    if (ret)
    break;
    }
    diff --git a/include/linux/mm.h b/include/linux/mm.h
    index 17623a774e02..2fa8aae4c4f2 100644
    --- a/include/linux/mm.h
    +++ b/include/linux/mm.h
    @@ -2700,22 +2700,16 @@ static inline int vma_adjust(struct vm_area_struct *vma, unsigned long start,
    {
    return __vma_adjust(vma, start, end, pgoff, insert, NULL);
    }
    -extern struct vm_area_struct *vma_merge(struct mm_struct *,
    - struct vm_area_struct *prev, unsigned long addr, unsigned long end,
    - unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t,
    - struct mempolicy *, struct vm_userfaultfd_ctx, struct anon_vma_name *);
    -extern struct vm_area_struct *vmi_vma_merge(struct vma_iterator *vmi,
    +extern struct vm_area_struct *vma_merge(struct vma_iterator *vmi,
    struct mm_struct *, struct vm_area_struct *prev, unsigned long addr,
    unsigned long end, unsigned long vm_flags, struct anon_vma *,
    struct file *, pgoff_t, struct mempolicy *, struct vm_userfaultfd_ctx,
    struct anon_vma_name *);
    extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
    -extern int vmi__split_vma(struct vma_iterator *vmi, struct mm_struct *,
    - struct vm_area_struct *, unsigned long addr, int new_below);
    -extern int split_vma(struct mm_struct *, struct vm_area_struct *,
    - unsigned long addr, int new_below);
    -extern int vmi_split_vma(struct vma_iterator *vmi, struct mm_struct *,
    - struct vm_area_struct *, unsigned long addr, int new_below);
    +extern int __split_vma(struct vma_iterator *vmi, struct vm_area_struct *,
    + unsigned long addr, int new_below);
    +extern int split_vma(struct vma_iterator *vmi, struct vm_area_struct *,
    + unsigned long addr, int new_below);
    extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
    extern void unlink_file_vma(struct vm_area_struct *);
    extern struct vm_area_struct *copy_vma(struct vm_area_struct **,
    diff --git a/mm/madvise.c b/mm/madvise.c
    index 2beced2d5eee..bf23d64515e1 100644
    --- a/mm/madvise.c
    +++ b/mm/madvise.c
    @@ -153,7 +153,7 @@ static int madvise_update_vma(struct vm_area_struct *vma,
    }

    pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
    - *prev = vmi_vma_merge(&vmi, mm, *prev, start, end, new_flags,
    + *prev = vma_merge(&vmi, mm, *prev, start, end, new_flags,
    vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
    vma->vm_userfaultfd_ctx, anon_name);
    if (*prev) {
    @@ -166,7 +166,7 @@ static int madvise_update_vma(struct vm_area_struct *vma,
    if (start != vma->vm_start) {
    if (unlikely(mm->map_count >= sysctl_max_map_count))
    return -ENOMEM;
    - error = vmi__split_vma(&vmi, mm, vma, start, 1);
    + error = __split_vma(&vmi, vma, start, 1);
    if (error)
    return error;
    }
    @@ -174,7 +174,7 @@ static int madvise_update_vma(struct vm_area_struct *vma,
    if (end != vma->vm_end) {
    if (unlikely(mm->map_count >= sysctl_max_map_count))
    return -ENOMEM;
    - error = vmi__split_vma(&vmi, mm, vma, end, 0);
    + error = __split_vma(&vmi, vma, end, 0);
    if (error)
    return error;
    }
    diff --git a/mm/mempolicy.c b/mm/mempolicy.c
    index 5bf4ab8eee49..ee9751323c59 100644
    --- a/mm/mempolicy.c
    +++ b/mm/mempolicy.c
    @@ -810,7 +810,7 @@ static int mbind_range(struct mm_struct *mm, unsigned long start,

    pgoff = vma->vm_pgoff +
    ((vmstart - vma->vm_start) >> PAGE_SHIFT);
    - prev = vmi_vma_merge(&vmi, mm, prev, vmstart, vmend, vma->vm_flags,
    + prev = vma_merge(&vmi, mm, prev, vmstart, vmend, vma->vm_flags,
    vma->anon_vma, vma->vm_file, pgoff,
    new_pol, vma->vm_userfaultfd_ctx,
    anon_vma_name(vma));
    @@ -819,12 +819,12 @@ static int mbind_range(struct mm_struct *mm, unsigned long start,
    goto replace;
    }
    if (vma->vm_start != vmstart) {
    - err = vmi_split_vma(&vmi, vma->vm_mm, vma, vmstart, 1);
    + err = split_vma(&vmi, vma, vmstart, 1);
    if (err)
    goto out;
    }
    if (vma->vm_end != vmend) {
    - err = vmi_split_vma(&vmi, vma->vm_mm, vma, vmend, 0);
    + err = split_vma(&vmi, vma, vmend, 0);
    if (err)
    goto out;
    }
    diff --git a/mm/mlock.c b/mm/mlock.c
    index f06b02b631b5..393cddee2f06 100644
    --- a/mm/mlock.c
    +++ b/mm/mlock.c
    @@ -418,7 +418,7 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma,
    goto out;

    pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
    - *prev = vmi_vma_merge(vmi, mm, *prev, start, end, newflags,
    + *prev = vma_merge(vmi, mm, *prev, start, end, newflags,
    vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
    vma->vm_userfaultfd_ctx, anon_vma_name(vma));
    if (*prev) {
    @@ -427,13 +427,13 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma,
    }

    if (start != vma->vm_start) {
    - ret = vmi_split_vma(vmi, mm, vma, start, 1);
    + ret = split_vma(vmi, vma, start, 1);
    if (ret)
    goto out;
    }

    if (end != vma->vm_end) {
    - ret = vmi_split_vma(vmi, mm, vma, end, 0);
    + ret = split_vma(vmi, vma, end, 0);
    if (ret)
    goto out;
    }
    diff --git a/mm/mmap.c b/mm/mmap.c
    index c22365727ab4..0c194a2a568d 100644
    --- a/mm/mmap.c
    +++ b/mm/mmap.c
    @@ -1070,7 +1070,7 @@ can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
    * parameter) may establish ptes with the wrong permissions of NNNN
    * instead of the right permissions of XXXX.
    */
    -struct vm_area_struct *vma_merge(struct mm_struct *mm,
    +struct vm_area_struct *vma_merge(struct vma_iterator *vmi, struct mm_struct *mm,
    struct vm_area_struct *prev, unsigned long addr,
    unsigned long end, unsigned long vm_flags,
    struct anon_vma *anon_vma, struct file *file,
    @@ -1079,7 +1079,7 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm,
    struct anon_vma_name *anon_name)
    {
    pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
    - struct vm_area_struct *mid, *next, *res;
    + struct vm_area_struct *mid, *next, *res = NULL;
    int err = -1;
    bool merge_prev = false;
    bool merge_next = false;
    @@ -1145,26 +1145,11 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm,
    if (err)
    return NULL;
    khugepaged_enter_vma(res, vm_flags);
    - return res;
    -}

    -struct vm_area_struct *vmi_vma_merge(struct vma_iterator *vmi,
    - struct mm_struct *mm,
    - struct vm_area_struct *prev, unsigned long addr,
    - unsigned long end, unsigned long vm_flags,
    - struct anon_vma *anon_vma, struct file *file,
    - pgoff_t pgoff, struct mempolicy *policy,
    - struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
    - struct anon_vma_name *anon_name)
    -{
    - struct vm_area_struct *tmp;
    -
    - tmp = vma_merge(mm, prev, addr, end, vm_flags, anon_vma, file, pgoff,
    - policy, vm_userfaultfd_ctx, anon_name);
    - if (tmp)
    + if (res)
    vma_iter_set(vmi, end);

    - return tmp;
    + return res;
    }

    /*
    @@ -2287,12 +2272,14 @@ static void unmap_region(struct mm_struct *mm, struct maple_tree *mt,
    * __split_vma() bypasses sysctl_max_map_count checking. We use this where it
    * has already been checked or doesn't make sense to fail.
    */
    -int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
    +int __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
    unsigned long addr, int new_below)
    {
    struct vm_area_struct *new;
    int err;
    - validate_mm_mt(mm);
    + unsigned long end = vma->vm_end;
    +
    + validate_mm_mt(vma->vm_mm);

    if (vma->vm_ops && vma->vm_ops->may_split) {
    err = vma->vm_ops->may_split(vma, addr);
    @@ -2332,8 +2319,10 @@ int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
    err = vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new);

    /* Success. */
    - if (!err)
    + if (!err) {
    + vma_iter_set(vmi, end);
    return 0;
    + }

    /* Avoid vm accounting in close() operation */
    new->vm_start = new->vm_end;
    @@ -2348,46 +2337,21 @@ int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
    mpol_put(vma_policy(new));
    out_free_vma:
    vm_area_free(new);
    - validate_mm_mt(mm);
    + validate_mm_mt(vma->vm_mm);
    return err;
    }
    -int vmi__split_vma(struct vma_iterator *vmi, struct mm_struct *mm,
    - struct vm_area_struct *vma, unsigned long addr, int new_below)
    -{
    - int ret;
    - unsigned long end = vma->vm_end;
    -
    - ret = __split_vma(mm, vma, addr, new_below);
    - if (!ret)
    - vma_iter_set(vmi, end);
    -
    - return ret;
    -}

    /*
    * Split a vma into two pieces at address 'addr', a new vma is allocated
    * either for the first part or the tail.
    */
    -int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
    +int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
    unsigned long addr, int new_below)
    {
    - if (mm->map_count >= sysctl_max_map_count)
    + if (vma->vm_mm->map_count >= sysctl_max_map_count)
    return -ENOMEM;

    - return __split_vma(mm, vma, addr, new_below);
    -}
    -
    -int vmi_split_vma(struct vma_iterator *vmi, struct mm_struct *mm,
    - struct vm_area_struct *vma, unsigned long addr, int new_below)
    -{
    - int ret;
    - unsigned long end = vma->vm_end;
    -
    - ret = split_vma(mm, vma, addr, new_below);
    - if (!ret)
    - vma_iter_set(vmi, end);
    -
    - return ret;
    + return __split_vma(vmi, vma, addr, new_below);
    }

    static inline int munmap_sidetree(struct vm_area_struct *vma,
    @@ -2447,7 +2411,7 @@ do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
    if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count)
    goto map_count_exceeded;

    - error = vmi__split_vma(vmi, mm, vma, start, 0);
    + error = __split_vma(vmi, vma, start, 0);
    if (error)
    goto start_split_failed;

    @@ -2468,7 +2432,7 @@ do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
    if (next->vm_end > end) {
    struct vm_area_struct *split;

    - error = vmi__split_vma(vmi, mm, next, end, 1);
    + error = __split_vma(vmi, next, end, 1);
    if (error)
    goto end_split_failed;

    @@ -2749,9 +2713,10 @@ unsigned long mmap_region(struct file *file, unsigned long addr,
    * vma again as we may succeed this time.
    */
    if (unlikely(vm_flags != vma->vm_flags && prev)) {
    - merge = vmi_vma_merge(&vmi, mm, prev, vma->vm_start,
    - vma->vm_end, vma->vm_flags, NULL, vma->vm_file,
    - vma->vm_pgoff, NULL, NULL_VM_UFFD_CTX, NULL);
    + merge = vma_merge(&vmi, mm, prev, vma->vm_start,
    + vma->vm_end, vma->vm_flags, NULL,
    + vma->vm_file, vma->vm_pgoff, NULL,
    + NULL_VM_UFFD_CTX, NULL);
    if (merge) {
    /*
    * ->mmap() can change vma->vm_file and fput
    @@ -3303,7 +3268,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
    if (new_vma && new_vma->vm_start < addr + len)
    return NULL; /* should never get here */

    - new_vma = vmi_vma_merge(&vmi, mm, prev, addr, addr + len, vma->vm_flags,
    + new_vma = vma_merge(&vmi, mm, prev, addr, addr + len, vma->vm_flags,
    vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
    vma->vm_userfaultfd_ctx, anon_vma_name(vma));
    if (new_vma) {
    diff --git a/mm/mprotect.c b/mm/mprotect.c
    index 994e8d991e78..bd5c04974bbc 100644
    --- a/mm/mprotect.c
    +++ b/mm/mprotect.c
    @@ -602,7 +602,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb,
    * First try to merge with previous and/or next vma.
    */
    pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
    - *pprev = vmi_vma_merge(vmi, mm, *pprev, start, end, newflags,
    + *pprev = vma_merge(vmi, mm, *pprev, start, end, newflags,
    vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
    vma->vm_userfaultfd_ctx, anon_vma_name(vma));
    if (*pprev) {
    @@ -614,13 +614,13 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb,
    *pprev = vma;

    if (start != vma->vm_start) {
    - error = vmi_split_vma(vmi, mm, vma, start, 1);
    + error = split_vma(vmi, vma, start, 1);
    if (error)
    goto fail;
    }

    if (end != vma->vm_end) {
    - error = vmi_split_vma(vmi, mm, vma, end, 0);
    + error = split_vma(vmi, vma, end, 0);
    if (error)
    goto fail;
    }
    diff --git a/mm/mremap.c b/mm/mremap.c
    index 7302a71058ce..cbafea7fe895 100644
    --- a/mm/mremap.c
    +++ b/mm/mremap.c
    @@ -1033,7 +1033,7 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
    * with the next vma if it becomes adjacent to the expanded vma and
    * otherwise compatible.
    */
    - vma = vmi_vma_merge(&vmi, mm, vma, extension_start,
    + vma = vma_merge(&vmi, mm, vma, extension_start,
    extension_end, vma->vm_flags, vma->anon_vma,
    vma->vm_file, extension_pgoff, vma_policy(vma),
    vma->vm_userfaultfd_ctx, anon_vma_name(vma));
    --
    2.35.1
    \
     
     \ /
      Last update: 2022-11-29 17:47    [W:2.820 / U:0.012 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site