lkml.org 
[lkml]   [2019]   [Feb]   [15]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[RFC PATCH 09/31] mm: thp: 1GB anonymous page implementation.
    Date
    From: Zi Yan <ziy@nvidia.com>

    This adds 1GB THP support for anonymous pages. Applications can get 1GB
    pages during page faults when their VMAs are larger than 1GB. For
    read-only 1GB zero THP, a shared 1GB zero THP is created for all
    readers.

    Signed-off-by: Zi Yan <ziy@nvidia.com>
    ---
    arch/x86/include/asm/pgalloc.h | 58 +++++++
    arch/x86/include/asm/pgtable.h | 2 +
    arch/x86/mm/pgtable.c | 25 +++
    drivers/base/node.c | 4 +-
    fs/proc/meminfo.c | 3 +-
    include/asm-generic/pgtable.h | 3 +
    include/linux/huge_mm.h | 17 ++-
    include/linux/mm.h | 4 +
    include/linux/mm_types.h | 1 +
    include/linux/mmzone.h | 1 +
    include/linux/sched/coredump.h | 1 +
    include/linux/vm_event_item.h | 2 +
    kernel/fork.c | 5 +
    mm/huge_memory.c | 267 ++++++++++++++++++++++++++++++++-
    mm/memory.c | 28 +++-
    mm/page_alloc.c | 3 +-
    mm/pgtable-generic.c | 47 +++++-
    mm/rmap.c | 28 +++-
    mm/vmstat.c | 3 +
    19 files changed, 484 insertions(+), 18 deletions(-)

    diff --git a/arch/x86/include/asm/pgalloc.h b/arch/x86/include/asm/pgalloc.h
    index a281e61ec60c..6e29ad9b9d7f 100644
    --- a/arch/x86/include/asm/pgalloc.h
    +++ b/arch/x86/include/asm/pgalloc.h
    @@ -49,6 +49,7 @@ extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);

    extern pte_t *pte_alloc_one_kernel(struct mm_struct *);
    extern pgtable_t pte_alloc_one(struct mm_struct *);
    +extern pgtable_t pte_alloc_order(struct mm_struct *, unsigned long, int);

    /* Should really implement gc for free page table pages. This could be
    done with a reference count in struct page. */
    @@ -65,6 +66,17 @@ static inline void pte_free(struct mm_struct *mm, struct page *pte)
    __free_page(pte);
    }

    +static inline void pte_free_order(struct mm_struct *mm, struct page *pte,
    + int order)
    +{
    + int i;
    +
    + for (i = 0; i < (1<<order); i++) {
    + pgtable_page_dtor(&pte[i]);
    + __free_page(&pte[i]);
    + }
    +}
    +
    extern void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte);

    static inline void __pte_free_tlb(struct mmu_gather *tlb, struct page *pte,
    @@ -123,6 +135,52 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
    free_page((unsigned long)pmd);
    }

    +static inline pmd_t *pmd_alloc_one_page_with_ptes(struct mm_struct *mm, unsigned long addr)
    +{
    + pgtable_t pte_pgtables;
    + pmd_t *pmd;
    + spinlock_t *pmd_ptl;
    + int i;
    +
    + pte_pgtables = pte_alloc_order(mm, addr,
    + HPAGE_PUD_ORDER - HPAGE_PMD_ORDER);
    + if (!pte_pgtables)
    + return NULL;
    +
    + pmd = pmd_alloc_one(mm, addr);
    + if (unlikely(!pmd)) {
    + pte_free_order(mm, pte_pgtables,
    + HPAGE_PUD_ORDER - HPAGE_PMD_ORDER);
    + return NULL;
    + }
    + pmd_ptl = pmd_lock(mm, pmd);
    +
    + for (i = 0; i < (1<<(HPAGE_PUD_ORDER - HPAGE_PMD_ORDER)); i++)
    + pgtable_trans_huge_deposit(mm, pmd, pte_pgtables + i);
    +
    + spin_unlock(pmd_ptl);
    +
    + return pmd;
    +}
    +
    +static inline void pmd_free_page_with_ptes(struct mm_struct *mm, pmd_t *pmd)
    +{
    + spinlock_t *pmd_ptl;
    + int i;
    +
    + BUG_ON((unsigned long)pmd & (PAGE_SIZE-1));
    + pmd_ptl = pmd_lock(mm, pmd);
    +
    + for (i = 0; i < (1<<(HPAGE_PUD_ORDER - HPAGE_PMD_ORDER)); i++) {
    + pgtable_t pte_pgtable;
    +
    + pte_pgtable = pgtable_trans_huge_withdraw(mm, pmd);
    + pte_free(mm, pte_pgtable);
    + }
    +
    + spin_unlock(pmd_ptl);
    + pmd_free(mm, pmd);
    +}
    extern void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd);

    static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
    diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
    index 40616e805292..ae3ac49c32ad 100644
    --- a/arch/x86/include/asm/pgtable.h
    +++ b/arch/x86/include/asm/pgtable.h
    @@ -1165,6 +1165,8 @@ static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long
    return native_pmdp_get_and_clear(pmdp);
    }

    +#define mk_pud(page, pgprot) pfn_pud(page_to_pfn(page), (pgprot))
    +
    #define __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR
    static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm,
    unsigned long addr, pud_t *pudp)
    diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
    index 7bd01709a091..0a5008690d7c 100644
    --- a/arch/x86/mm/pgtable.c
    +++ b/arch/x86/mm/pgtable.c
    @@ -42,6 +42,31 @@ pgtable_t pte_alloc_one(struct mm_struct *mm)
    return pte;
    }

    +pgtable_t pte_alloc_order(struct mm_struct *mm, unsigned long address, int order)
    +{
    + struct page *pte;
    + int i;
    +
    + pte = alloc_pages(__userpte_alloc_gfp, order);
    + if (!pte)
    + return NULL;
    + split_page(pte, order);
    + for (i = 1; i < (1 << order); i++)
    + set_page_private(pte + i, 0);
    +
    + for (i = 0; i < (1<<order); i++) {
    + if (!pgtable_page_ctor(&pte[i])) {
    + __free_page(&pte[i]);
    + while (--i >= 0) {
    + pgtable_page_dtor(&pte[i]);
    + __free_page(&pte[i]);
    + }
    + return NULL;
    + }
    + }
    + return pte;
    +}
    +
    static int __init setup_userpte(char *arg)
    {
    if (!arg)
    diff --git a/drivers/base/node.c b/drivers/base/node.c
    index 86d6cd92ce3d..f21d2235bf97 100644
    --- a/drivers/base/node.c
    +++ b/drivers/base/node.c
    @@ -150,7 +150,9 @@ static ssize_t node_read_meminfo(struct device *dev,
    #ifdef CONFIG_TRANSPARENT_HUGEPAGE
    ,
    nid, K(node_page_state(pgdat, NR_ANON_THPS) *
    - HPAGE_PMD_NR),
    + HPAGE_PMD_NR) +
    + K(node_page_state(pgdat, NR_ANON_THPS_PUD) *
    + HPAGE_PUD_NR),
    nid, K(node_page_state(pgdat, NR_SHMEM_THPS) *
    HPAGE_PMD_NR),
    nid, K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) *
    diff --git a/fs/proc/meminfo.c b/fs/proc/meminfo.c
    index 568d90e17c17..9d127e440e4c 100644
    --- a/fs/proc/meminfo.c
    +++ b/fs/proc/meminfo.c
    @@ -131,7 +131,8 @@ static int meminfo_proc_show(struct seq_file *m, void *v)

    #ifdef CONFIG_TRANSPARENT_HUGEPAGE
    show_val_kb(m, "AnonHugePages: ",
    - global_node_page_state(NR_ANON_THPS) * HPAGE_PMD_NR);
    + global_node_page_state(NR_ANON_THPS) * HPAGE_PMD_NR +
    + global_node_page_state(NR_ANON_THPS_PUD) * HPAGE_PUD_NR);
    show_val_kb(m, "ShmemHugePages: ",
    global_node_page_state(NR_SHMEM_THPS) * HPAGE_PMD_NR);
    show_val_kb(m, "ShmemPmdMapped: ",
    diff --git a/include/asm-generic/pgtable.h b/include/asm-generic/pgtable.h
    index 05e61e6c843f..0f626d6177c3 100644
    --- a/include/asm-generic/pgtable.h
    +++ b/include/asm-generic/pgtable.h
    @@ -303,10 +303,13 @@ static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma,
    #ifndef __HAVE_ARCH_PGTABLE_DEPOSIT
    extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
    pgtable_t pgtable);
    +extern void pgtable_trans_huge_pud_deposit(struct mm_struct *mm, pud_t *pudp,
    + pgtable_t pgtable);
    #endif

    #ifndef __HAVE_ARCH_PGTABLE_WITHDRAW
    extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
    +extern pgtable_t pgtable_trans_huge_pud_withdraw(struct mm_struct *mm, pud_t *pudp);
    #endif

    #ifdef CONFIG_TRANSPARENT_HUGEPAGE
    diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
    index 381e872bfde0..c6272e6ffc35 100644
    --- a/include/linux/huge_mm.h
    +++ b/include/linux/huge_mm.h
    @@ -18,10 +18,15 @@ extern int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm,

    #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
    extern void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud);
    +extern int do_huge_pud_anonymous_page(struct vm_fault *vmf);
    #else
    static inline void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud)
    {
    }
    +extern int do_huge_pud_anonymous_page(struct vm_fault *vmf)
    +{
    + return VM_FAULT_FALLBACK;
    +}
    #endif

    extern vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf, pmd_t orig_pmd);
    @@ -80,6 +85,9 @@ extern struct kobj_attribute shmem_enabled_attr;
    #define HPAGE_PMD_ORDER (HPAGE_PMD_SHIFT-PAGE_SHIFT)
    #define HPAGE_PMD_NR (1<<HPAGE_PMD_ORDER)

    +#define HPAGE_PUD_ORDER (HPAGE_PUD_SHIFT-PAGE_SHIFT)
    +#define HPAGE_PUD_NR (1<<HPAGE_PUD_ORDER)
    +
    #ifdef CONFIG_TRANSPARENT_HUGEPAGE
    #define HPAGE_PMD_SHIFT PMD_SHIFT
    #define HPAGE_PMD_SIZE ((1UL) << HPAGE_PMD_SHIFT)
    @@ -214,7 +222,7 @@ static inline spinlock_t *pud_trans_huge_lock(pud_t *pud,
    static inline int hpage_nr_pages(struct page *page)
    {
    if (unlikely(PageTransHuge(page)))
    - return HPAGE_PMD_NR;
    + return (1<<page[1].compound_order);
    return 1;
    }

    @@ -226,10 +234,12 @@ struct page *follow_devmap_pud(struct vm_area_struct *vma, unsigned long addr,
    extern vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t orig_pmd);

    extern struct page *huge_zero_page;
    +extern struct page *huge_pud_zero_page;

    static inline bool is_huge_zero_page(struct page *page)
    {
    - return READ_ONCE(huge_zero_page) == page;
    + return (READ_ONCE(huge_zero_page) == page) ||
    + (READ_ONCE(huge_pud_zero_page) == page);
    }

    static inline bool is_huge_zero_pmd(pmd_t pmd)
    @@ -239,13 +249,14 @@ static inline bool is_huge_zero_pmd(pmd_t pmd)

    static inline bool is_huge_zero_pud(pud_t pud)
    {
    - return false;
    + return is_huge_zero_page(pud_page(pud));
    }

    struct page *mm_get_huge_zero_page(struct mm_struct *mm);
    void mm_put_huge_zero_page(struct mm_struct *mm);

    #define mk_huge_pmd(page, prot) pmd_mkhuge(mk_pmd(page, prot))
    +#define mk_huge_pud(page, prot) pud_mkhuge(mk_pud(page, prot))

    static inline bool thp_migration_supported(void)
    {
    diff --git a/include/linux/mm.h b/include/linux/mm.h
    index 5bcc1b03372a..d10dc9db2311 100644
    --- a/include/linux/mm.h
    +++ b/include/linux/mm.h
    @@ -26,6 +26,7 @@
    #include <linux/page_ref.h>
    #include <linux/memremap.h>
    #include <linux/overflow.h>
    +#include <linux/pagechain.h>

    struct mempolicy;
    struct anon_vma;
    @@ -1985,6 +1986,7 @@ static inline void pgtable_init(void)
    {
    ptlock_cache_init();
    pgtable_cache_init();
    + pagechain_cache_init();
    }

    static inline bool pgtable_page_ctor(struct page *page)
    @@ -2101,6 +2103,8 @@ static inline spinlock_t *pud_lock(struct mm_struct *mm, pud_t *pud)
    return ptl;
    }

    +#define pud_huge_pte(mm, pud) ((mm)->pud_huge_pte)
    +
    extern void __init pagecache_init(void);
    extern void free_area_init(unsigned long * zones_size);
    extern void __init free_area_init_node(int nid, unsigned long * zones_size,
    diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
    index 32549b255d25..a5ac5946a375 100644
    --- a/include/linux/mm_types.h
    +++ b/include/linux/mm_types.h
    @@ -466,6 +466,7 @@ struct mm_struct {
    #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
    pgtable_t pmd_huge_pte; /* protected by page_table_lock */
    #endif
    + struct list_head pud_huge_pte; /* protected by page_table_lock */
    #ifdef CONFIG_NUMA_BALANCING
    /*
    * numa_next_scan is the next time that the PTEs will be marked
    diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
    index 842f9189537b..ea84d6a1802d 100644
    --- a/include/linux/mmzone.h
    +++ b/include/linux/mmzone.h
    @@ -177,6 +177,7 @@ enum node_stat_item {
    NR_SHMEM_THPS,
    NR_SHMEM_PMDMAPPED,
    NR_ANON_THPS,
    + NR_ANON_THPS_PUD,
    NR_UNSTABLE_NFS, /* NFS unstable pages */
    NR_VMSCAN_WRITE,
    NR_VMSCAN_IMMEDIATE, /* Prioritise for reclaim when writeback ends */
    diff --git a/include/linux/sched/coredump.h b/include/linux/sched/coredump.h
    index 52ad71db6687..4893849d11eb 100644
    --- a/include/linux/sched/coredump.h
    +++ b/include/linux/sched/coredump.h
    @@ -73,6 +73,7 @@ static inline int get_dumpable(struct mm_struct *mm)
    #define MMF_OOM_VICTIM 25 /* mm is the oom victim */
    #define MMF_OOM_REAP_QUEUED 26 /* mm was queued for oom_reaper */
    #define MMF_DISABLE_THP_MASK (1 << MMF_DISABLE_THP)
    +#define MMF_HUGE_PUD_ZERO_PAGE 26 /* mm has ever used the global huge pud zero page */

    #define MMF_INIT_MASK (MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK |\
    MMF_DISABLE_THP_MASK)
    diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h
    index 6b32c8243616..4550667b2274 100644
    --- a/include/linux/vm_event_item.h
    +++ b/include/linux/vm_event_item.h
    @@ -82,6 +82,8 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
    THP_DEFERRED_SPLIT_PAGE,
    THP_SPLIT_PMD,
    #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
    + THP_FAULT_ALLOC_PUD,
    + THP_FAULT_FALLBACK_PUD,
    THP_SPLIT_PUD,
    #endif
    THP_ZERO_PAGE_ALLOC,
    diff --git a/kernel/fork.c b/kernel/fork.c
    index dcefa978c232..fc5a925e0496 100644
    --- a/kernel/fork.c
    +++ b/kernel/fork.c
    @@ -662,6 +662,10 @@ static void check_mm(struct mm_struct *mm)
    #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
    VM_BUG_ON_MM(mm->pmd_huge_pte, mm);
    #endif
    + VM_BUG_ON_MM(!list_empty(&mm->pud_huge_pte) &&
    + !pagechain_empty(list_first_entry(&mm->pud_huge_pte,
    + struct pagechain, list)),
    + mm);
    }

    #define allocate_mm() (kmem_cache_alloc(mm_cachep, GFP_KERNEL))
    @@ -1003,6 +1007,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
    #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
    mm->pmd_huge_pte = NULL;
    #endif
    + INIT_LIST_HEAD(&mm->pud_huge_pte);
    mm_init_uprobes_state(mm);

    if (current->mm) {
    diff --git a/mm/huge_memory.c b/mm/huge_memory.c
    index ffcae07a87d3..cad4ef01f607 100644
    --- a/mm/huge_memory.c
    +++ b/mm/huge_memory.c
    @@ -62,6 +62,8 @@ static struct shrinker deferred_split_shrinker;

    static atomic_t huge_zero_refcount;
    struct page *huge_zero_page __read_mostly;
    +static atomic_t huge_pud_zero_refcount;
    +struct page *huge_pud_zero_page __read_mostly;

    bool transparent_hugepage_enabled(struct vm_area_struct *vma)
    {
    @@ -109,6 +111,42 @@ static void put_huge_zero_page(void)
    BUG_ON(atomic_dec_and_test(&huge_zero_refcount));
    }

    +static struct page *get_huge_pud_zero_page(void)
    +{
    + struct page *zero_page;
    +retry:
    + if (likely(atomic_inc_not_zero(&huge_pud_zero_refcount)))
    + return READ_ONCE(huge_pud_zero_page);
    +
    + zero_page = alloc_pages((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE,
    + HPAGE_PUD_ORDER);
    + if (!zero_page) {
    + count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED);
    + return NULL;
    + }
    + count_vm_event(THP_ZERO_PAGE_ALLOC);
    + preempt_disable();
    + if (cmpxchg(&huge_pud_zero_page, NULL, zero_page)) {
    + preempt_enable();
    + __free_pages(zero_page, compound_order(zero_page));
    + goto retry;
    + }
    +
    + /* We take additional reference here. It will be put back by shrinker */
    + atomic_set(&huge_pud_zero_refcount, 2);
    + preempt_enable();
    + return READ_ONCE(huge_pud_zero_page);
    +}
    +
    +static void put_huge_pud_zero_page(void)
    +{
    + /*
    + * Counter should never go to zero here. Only shrinker can put
    + * last reference.
    + */
    + BUG_ON(atomic_dec_and_test(&huge_pud_zero_refcount));
    +}
    +
    struct page *mm_get_huge_zero_page(struct mm_struct *mm)
    {
    if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags))
    @@ -123,9 +161,23 @@ struct page *mm_get_huge_zero_page(struct mm_struct *mm)
    return READ_ONCE(huge_zero_page);
    }

    +struct page *mm_get_huge_pud_zero_page(struct mm_struct *mm)
    +{
    + if (test_bit(MMF_HUGE_PUD_ZERO_PAGE, &mm->flags))
    + return READ_ONCE(huge_pud_zero_page);
    +
    + if (!get_huge_pud_zero_page())
    + return NULL;
    +
    + if (test_and_set_bit(MMF_HUGE_PUD_ZERO_PAGE, &mm->flags))
    + put_huge_pud_zero_page();
    +
    + return READ_ONCE(huge_pud_zero_page);
    +}
    +
    void mm_put_huge_zero_page(struct mm_struct *mm)
    {
    - if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags))
    + if (test_bit(MMF_HUGE_PUD_ZERO_PAGE, &mm->flags))
    put_huge_zero_page();
    }

    @@ -859,6 +911,175 @@ vm_fault_t vmf_insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr,
    return VM_FAULT_NOPAGE;
    }
    EXPORT_SYMBOL_GPL(vmf_insert_pfn_pud);
    +
    +static int __do_huge_pud_anonymous_page(struct vm_fault *vmf, struct page *page,
    + gfp_t gfp)
    +{
    + struct vm_area_struct *vma = vmf->vma;
    + struct mem_cgroup *memcg;
    + pmd_t *pmd_pgtable;
    + unsigned long haddr = vmf->address & HPAGE_PUD_MASK;
    + int ret = 0;
    +
    + VM_BUG_ON_PAGE(!PageCompound(page), page);
    +
    + if (mem_cgroup_try_charge(page, vma->vm_mm, gfp, &memcg, true)) {
    + put_page(page);
    + count_vm_event(THP_FAULT_FALLBACK_PUD);
    + return VM_FAULT_FALLBACK;
    + }
    +
    + pmd_pgtable = pmd_alloc_one_page_with_ptes(vma->vm_mm, haddr);
    + if (unlikely(!pmd_pgtable)) {
    + ret = VM_FAULT_OOM;
    + goto release;
    + }
    +
    + clear_huge_page(page, vmf->address, HPAGE_PUD_NR);
    + /*
    + * The memory barrier inside __SetPageUptodate makes sure that
    + * clear_huge_page writes become visible before the set_pmd_at()
    + * write.
    + */
    + __SetPageUptodate(page);
    +
    + vmf->ptl = pud_lock(vma->vm_mm, vmf->pud);
    + if (unlikely(!pud_none(*vmf->pud))) {
    + goto unlock_release;
    + } else {
    + pud_t entry;
    + int i;
    +
    + ret = check_stable_address_space(vma->vm_mm);
    + if (ret)
    + goto unlock_release;
    +
    + /* Deliver the page fault to userland */
    + if (userfaultfd_missing(vma)) {
    + int ret;
    +
    + spin_unlock(vmf->ptl);
    + mem_cgroup_cancel_charge(page, memcg, true);
    + put_page(page);
    + pmd_free_page_with_ptes(vma->vm_mm, pmd_pgtable);
    + ret = handle_userfault(vmf, VM_UFFD_MISSING);
    + VM_BUG_ON(ret & VM_FAULT_FALLBACK);
    + return ret;
    + }
    +
    + entry = mk_huge_pud(page, vma->vm_page_prot);
    + entry = maybe_pud_mkwrite(pud_mkdirty(entry), vma);
    + page_add_new_anon_rmap(page, vma, haddr, true);
    + mem_cgroup_commit_charge(page, memcg, false, true);
    + lru_cache_add_active_or_unevictable(page, vma);
    + pgtable_trans_huge_pud_deposit(vma->vm_mm, vmf->pud,
    + virt_to_page(pmd_pgtable));
    + set_pud_at(vma->vm_mm, haddr, vmf->pud, entry);
    + add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PUD_NR);
    + mm_inc_nr_pmds(vma->vm_mm);
    + for (i = 0; i < (1<<(HPAGE_PUD_ORDER - HPAGE_PMD_ORDER)); i++)
    + mm_inc_nr_ptes(vma->vm_mm);
    + spin_unlock(vmf->ptl);
    + count_vm_event(THP_FAULT_ALLOC_PUD);
    + }
    +
    + return 0;
    +unlock_release:
    + spin_unlock(vmf->ptl);
    +release:
    + if (pmd_pgtable)
    + pmd_free_page_with_ptes(vma->vm_mm, pmd_pgtable);
    + mem_cgroup_cancel_charge(page, memcg, true);
    + put_page(page);
    + return ret;
    +
    +}
    +
    +/* Caller must hold page table lock. */
    +static bool set_huge_pud_zero_page(pgtable_t pmd_pgtable,
    + struct mm_struct *mm,
    + struct vm_area_struct *vma, unsigned long haddr, pud_t *pud,
    + struct page *zero_page)
    +{
    + pud_t entry;
    + int i;
    +
    + if (!pud_none(*pud))
    + return false;
    + entry = mk_pud(zero_page, vma->vm_page_prot);
    + entry = pud_mkhuge(entry);
    + if (pmd_pgtable)
    + pgtable_trans_huge_pud_deposit(mm, pud, pmd_pgtable);
    + set_pud_at(mm, haddr, pud, entry);
    + mm_inc_nr_pmds(mm);
    + for (i = 0; i < (1<<(HPAGE_PUD_ORDER - HPAGE_PMD_ORDER)); i++)
    + mm_inc_nr_ptes(mm);
    + return true;
    +}
    +
    +int do_huge_pud_anonymous_page(struct vm_fault *vmf)
    +{
    + struct vm_area_struct *vma = vmf->vma;
    + gfp_t gfp;
    + struct page *page;
    + unsigned long haddr = vmf->address & HPAGE_PUD_MASK;
    +
    + if (haddr < vma->vm_start || haddr + HPAGE_PUD_SIZE > vma->vm_end)
    + return VM_FAULT_FALLBACK;
    + if (unlikely(anon_vma_prepare(vma)))
    + return VM_FAULT_OOM;
    + if (unlikely(khugepaged_enter(vma, vma->vm_flags)))
    + return VM_FAULT_OOM;
    + if (!(vmf->flags & FAULT_FLAG_WRITE) &&
    + !mm_forbids_zeropage(vma->vm_mm) &&
    + transparent_hugepage_use_zero_page()) {
    + pmd_t *pmd_pgtable;
    + struct page *zero_page;
    + bool set;
    + int ret;
    +
    + pmd_pgtable = pmd_alloc_one_page_with_ptes(vma->vm_mm, haddr);
    + if (unlikely(!pmd_pgtable))
    + return VM_FAULT_OOM;
    + zero_page = mm_get_huge_pud_zero_page(vma->vm_mm);
    + if (unlikely(!zero_page)) {
    + pmd_free_page_with_ptes(vma->vm_mm, pmd_pgtable);
    + count_vm_event(THP_FAULT_FALLBACK_PUD);
    + return VM_FAULT_FALLBACK;
    + }
    + vmf->ptl = pud_lock(vma->vm_mm, vmf->pud);
    + ret = 0;
    + set = false;
    + if (pud_none(*vmf->pud)) {
    + ret = check_stable_address_space(vma->vm_mm);
    + if (ret) {
    + spin_unlock(vmf->ptl);
    + } else if (userfaultfd_missing(vma)) {
    + spin_unlock(vmf->ptl);
    + ret = handle_userfault(vmf, VM_UFFD_MISSING);
    + VM_BUG_ON(ret & VM_FAULT_FALLBACK);
    + } else {
    + set_huge_pud_zero_page(virt_to_page(pmd_pgtable),
    + vma->vm_mm, vma, haddr, vmf->pud, zero_page);
    + spin_unlock(vmf->ptl);
    + set = true;
    + }
    + } else
    + spin_unlock(vmf->ptl);
    + if (!set)
    + pmd_free_page_with_ptes(vma->vm_mm, pmd_pgtable);
    + return ret;
    + }
    + gfp = alloc_hugepage_direct_gfpmask(vma);
    + page = alloc_hugepage_vma(gfp, vma, haddr, HPAGE_PUD_ORDER);
    + if (unlikely(!page)) {
    + count_vm_event(THP_FAULT_FALLBACK_PUD);
    + return VM_FAULT_FALLBACK;
    + }
    + prep_transhuge_page(page);
    + return __do_huge_pud_anonymous_page(vmf, page, gfp);
    +}
    +
    #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */

    static void touch_pmd(struct vm_area_struct *vma, unsigned long addr,
    @@ -1980,12 +2201,27 @@ spinlock_t *__pud_trans_huge_lock(pud_t *pud, struct vm_area_struct *vma)
    }

    #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
    +static inline void zap_pud_deposited_table(struct mm_struct *mm, pud_t *pud)
    +{
    + pgtable_t pgtable;
    + int i;
    +
    + pgtable = pgtable_trans_huge_pud_withdraw(mm, pud);
    + pmd_free_page_with_ptes(mm, (pmd_t *)page_address(pgtable));
    +
    + mm_dec_nr_pmds(mm);
    + for (i = 0; i < (1<<(HPAGE_PUD_ORDER - HPAGE_PMD_ORDER)); i++)
    + mm_dec_nr_ptes(mm);
    +}
    +
    int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma,
    pud_t *pud, unsigned long addr)
    {
    pud_t orig_pud;
    spinlock_t *ptl;

    + tlb_remove_check_page_size_change(tlb, HPAGE_PUD_SIZE);
    +
    ptl = __pud_trans_huge_lock(pud, vma);
    if (!ptl)
    return 0;
    @@ -2001,9 +2237,34 @@ int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma,
    if (vma_is_dax(vma)) {
    spin_unlock(ptl);
    /* No zero page support yet */
    + } else if (is_huge_zero_pud(orig_pud)) {
    + zap_pud_deposited_table(tlb->mm, pud);
    + spin_unlock(ptl);
    + tlb_remove_page_size(tlb, pud_page(orig_pud), HPAGE_PUD_SIZE);
    } else {
    - /* No support for anonymous PUD pages yet */
    - BUG();
    + struct page *page = NULL;
    + int flush_needed = 1;
    +
    + if (pud_present(orig_pud)) {
    + page = pud_page(orig_pud);
    + page_remove_rmap(page, true);
    + VM_BUG_ON_PAGE(page_mapcount(page) < 0, page);
    + VM_BUG_ON_PAGE(!PageHead(page), page);
    + } else
    + WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!");
    +
    + if (PageAnon(page)) {
    + zap_pud_deposited_table(tlb->mm, pud);
    + add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PUD_NR);
    + } else {
    + if (arch_needs_pgtable_deposit())
    + zap_pud_deposited_table(tlb->mm, pud);
    + add_mm_counter(tlb->mm, MM_FILEPAGES, -HPAGE_PUD_NR);
    + }
    +
    + spin_unlock(ptl);
    + if (flush_needed)
    + tlb_remove_page_size(tlb, page, HPAGE_PUD_SIZE);
    }
    return 1;
    }
    diff --git a/mm/memory.c b/mm/memory.c
    index 019036e87088..177478d5ee47 100644
    --- a/mm/memory.c
    +++ b/mm/memory.c
    @@ -3710,7 +3710,7 @@ static vm_fault_t create_huge_pud(struct vm_fault *vmf)
    #ifdef CONFIG_TRANSPARENT_HUGEPAGE
    /* No support for anonymous transparent PUD pages yet */
    if (vma_is_anonymous(vmf->vma))
    - return VM_FAULT_FALLBACK;
    + return do_huge_pud_anonymous_page(vmf);
    if (vmf->vma->vm_ops->huge_fault)
    return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PUD);
    #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
    @@ -4593,3 +4593,29 @@ void ptlock_free(struct page *page)
    kmem_cache_free(page_ptl_cachep, page->ptl);
    }
    #endif
    +
    +static struct kmem_cache *pagechain_cachep;
    +
    +void __init pagechain_cache_init(void)
    +{
    + pagechain_cachep = kmem_cache_create("pagechain",
    + sizeof(struct pagechain), 0, SLAB_PANIC, NULL);
    +}
    +
    +struct pagechain *pagechain_alloc(void)
    +{
    + struct pagechain *chain;
    +
    + chain = kmem_cache_alloc(pagechain_cachep, GFP_ATOMIC);
    +
    + if (!chain)
    + return NULL;
    +
    + pagechain_init(chain);
    + return chain;
    +}
    +
    +void pagechain_free(struct pagechain *pchain)
    +{
    + kmem_cache_free(pagechain_cachep, pchain);
    +}
    diff --git a/mm/page_alloc.c b/mm/page_alloc.c
    index cfa99bb54bd6..a3b295ea7348 100644
    --- a/mm/page_alloc.c
    +++ b/mm/page_alloc.c
    @@ -5157,7 +5157,8 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)
    K(node_page_state(pgdat, NR_SHMEM_THPS) * HPAGE_PMD_NR),
    K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)
    * HPAGE_PMD_NR),
    - K(node_page_state(pgdat, NR_ANON_THPS) * HPAGE_PMD_NR),
    + K(node_page_state(pgdat, NR_ANON_THPS) * HPAGE_PMD_NR +
    + node_page_state(pgdat, NR_ANON_THPS_PUD) * HPAGE_PUD_NR),
    #endif
    K(node_page_state(pgdat, NR_WRITEBACK_TEMP)),
    K(node_page_state(pgdat, NR_UNSTABLE_NFS)),
    diff --git a/mm/pgtable-generic.c b/mm/pgtable-generic.c
    index 532c29276fce..0b79568fba1c 100644
    --- a/mm/pgtable-generic.c
    +++ b/mm/pgtable-generic.c
    @@ -9,6 +9,7 @@

    #include <linux/pagemap.h>
    #include <linux/hugetlb.h>
    +#include <linux/pagechain.h>
    #include <asm/tlb.h>
    #include <asm-generic/pgtable.h>

    @@ -44,7 +45,7 @@ void pmd_clear_bad(pmd_t *pmd)

    #ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
    /*
    - * Only sets the access flags (dirty, accessed), as well as write
    + * Only sets the access flags (dirty, accessed), as well as write
    * permission. Furthermore, we know it always gets set to a "more
    * permissive" setting, which allows most architectures to optimize
    * this. We return whether the PTE actually changed, which in turn
    @@ -161,6 +162,23 @@ void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
    list_add(&pgtable->lru, &pmd_huge_pte(mm, pmdp)->lru);
    pmd_huge_pte(mm, pmdp) = pgtable;
    }
    +
    +void pgtable_trans_huge_pud_deposit(struct mm_struct *mm, pud_t *pudp,
    + pgtable_t pgtable)
    +{
    + struct pagechain *chain = NULL;
    +
    + assert_spin_locked(pud_lockptr(mm, pudp));
    + /* FIFO */
    + chain = list_first_entry_or_null(&pud_huge_pte(mm, pudp),
    + struct pagechain, list);
    +
    + if (!chain || !pagechain_space(chain)) {
    + chain = pagechain_alloc();
    + list_add(&chain->list, &pud_huge_pte(mm, pudp));
    + }
    + pagechain_deposit(chain, pgtable);
    +}
    #endif

    #ifndef __HAVE_ARCH_PGTABLE_WITHDRAW
    @@ -179,6 +197,33 @@ pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
    list_del(&pgtable->lru);
    return pgtable;
    }
    +
    +pgtable_t pgtable_trans_huge_pud_withdraw(struct mm_struct *mm, pud_t *pudp)
    +{
    + pgtable_t pgtable;
    + struct pagechain *chain = NULL;
    +
    + assert_spin_locked(pud_lockptr(mm, pudp));
    +
    + /* FIFO */
    +retry:
    + chain = list_first_entry_or_null(&pud_huge_pte(mm, pudp),
    + struct pagechain, list);
    +
    + if (!chain)
    + return NULL;
    +
    + if (pagechain_empty(chain)) {
    + if (list_is_singular(&chain->list))
    + return NULL;
    + list_del(&chain->list);
    + pagechain_free(chain);
    + goto retry;
    + }
    +
    + pgtable = pagechain_withdraw(chain);
    + return pgtable;
    +}
    #endif

    #ifndef __HAVE_ARCH_PMDP_INVALIDATE
    diff --git a/mm/rmap.c b/mm/rmap.c
    index 0454ecc29537..dae66a4329ea 100644
    --- a/mm/rmap.c
    +++ b/mm/rmap.c
    @@ -712,6 +712,7 @@ pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address)
    pgd_t *pgd;
    p4d_t *p4d;
    pud_t *pud;
    + pud_t pude;
    pmd_t *pmd = NULL;
    pmd_t pmde;

    @@ -724,7 +725,10 @@ pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address)
    goto out;

    pud = pud_offset(p4d, address);
    - if (!pud_present(*pud))
    +
    + pude = *pud;
    + barrier();
    + if (!pud_present(pude) || pud_trans_huge(pude))
    goto out;

    pmd = pmd_offset(pud, address);
    @@ -1121,8 +1125,12 @@ void do_page_add_anon_rmap(struct page *page,
    * pte lock(a spinlock) is held, which implies preemption
    * disabled.
    */
    - if (compound)
    - __inc_node_page_state(page, NR_ANON_THPS);
    + if (compound) {
    + if (nr == HPAGE_PMD_NR)
    + __inc_node_page_state(page, NR_ANON_THPS);
    + else
    + __inc_node_page_state(page, NR_ANON_THPS_PUD);
    + }
    __mod_node_page_state(page_pgdat(page), NR_ANON_MAPPED, nr);
    }
    if (unlikely(PageKsm(page)))
    @@ -1160,7 +1168,10 @@ void page_add_new_anon_rmap(struct page *page,
    VM_BUG_ON_PAGE(!PageTransHuge(page), page);
    /* increment count (starts at -1) */
    atomic_set(compound_mapcount_ptr(page), 0);
    - __inc_node_page_state(page, NR_ANON_THPS);
    + if (nr == HPAGE_PMD_NR)
    + __inc_node_page_state(page, NR_ANON_THPS);
    + else
    + __inc_node_page_state(page, NR_ANON_THPS_PUD);
    } else {
    /* Anon THP always mapped first with PMD */
    VM_BUG_ON_PAGE(PageTransCompound(page), page);
    @@ -1265,19 +1276,22 @@ static void page_remove_anon_compound_rmap(struct page *page)
    if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
    return;

    - __dec_node_page_state(page, NR_ANON_THPS);
    + if (hpage_nr_pages(page) == HPAGE_PMD_NR)
    + __dec_node_page_state(page, NR_ANON_THPS);
    + else
    + __dec_node_page_state(page, NR_ANON_THPS_PUD);

    if (TestClearPageDoubleMap(page)) {
    /*
    * Subpages can be mapped with PTEs too. Check how many of
    * themi are still mapped.
    */
    - for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) {
    + for (i = 0, nr = 0; i < hpage_nr_pages(page); i++) {
    if (atomic_add_negative(-1, &page[i]._mapcount))
    nr++;
    }
    } else {
    - nr = HPAGE_PMD_NR;
    + nr = hpage_nr_pages(page);
    }

    if (unlikely(PageMlocked(page)))
    diff --git a/mm/vmstat.c b/mm/vmstat.c
    index c18a42250a5c..25a88693e417 100644
    --- a/mm/vmstat.c
    +++ b/mm/vmstat.c
    @@ -1158,6 +1158,7 @@ const char * const vmstat_text[] = {
    "nr_shmem_hugepages",
    "nr_shmem_pmdmapped",
    "nr_anon_transparent_hugepages",
    + "nr_anon_transparent_pud_hugepages",
    "nr_unstable",
    "nr_vmscan_write",
    "nr_vmscan_immediate_reclaim",
    @@ -1259,6 +1260,8 @@ const char * const vmstat_text[] = {
    "thp_deferred_split_page",
    "thp_split_pmd",
    #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
    + "thp_fault_alloc_pud",
    + "thp_fault_fallback_pud",
    "thp_split_pud",
    #endif
    "thp_zero_page_alloc",
    --
    2.20.1
    \
     
     \ /
      Last update: 2019-02-15 23:11    [W:5.452 / U:0.020 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site