lkml.org 
[lkml]   [2011]   [May]   [8]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[PATCH v5 10/19] ARM: LPAE: Page table maintenance for the 3-level format
    Date
    This patch modifies the pgd/pmd/pte manipulation functions to support
    the 3-level page table format. Since there is no need for an 'ext'
    argument to cpu_set_pte_ext(), this patch conditionally defines a
    different prototype for this function when CONFIG_ARM_LPAE.

    The patch also introduces the L_PGD_SWAPPER flag to mark pgd entries
    pointing to pmd tables pre-allocated in the swapper_pg_dir and avoid
    trying to free them at run-time. This flag is 0 with the classic page
    table format.

    Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
    ---
    arch/arm/include/asm/pgalloc.h | 24 +++++++++++++
    arch/arm/include/asm/pgtable-3level.h | 5 +++
    arch/arm/include/asm/pgtable.h | 62 ++++++++++++++++++++++++++++++++-
    arch/arm/include/asm/proc-fns.h | 25 +++++++++++++
    arch/arm/mm/ioremap.c | 8 +++--
    arch/arm/mm/pgd.c | 51 +++++++++++++++++++++++++--
    arch/arm/mm/proc-v7.S | 8 ++++
    7 files changed, 175 insertions(+), 8 deletions(-)

    diff --git a/arch/arm/include/asm/pgalloc.h b/arch/arm/include/asm/pgalloc.h
    index 7418894..943504f 100644
    --- a/arch/arm/include/asm/pgalloc.h
    +++ b/arch/arm/include/asm/pgalloc.h
    @@ -25,6 +25,26 @@
    #define _PAGE_USER_TABLE (PMD_TYPE_TABLE | PMD_BIT4 | PMD_DOMAIN(DOMAIN_USER))
    #define _PAGE_KERNEL_TABLE (PMD_TYPE_TABLE | PMD_BIT4 | PMD_DOMAIN(DOMAIN_KERNEL))

    +#ifdef CONFIG_ARM_LPAE
    +
    +static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
    +{
    + return (pmd_t *)get_zeroed_page(GFP_KERNEL | __GFP_REPEAT);
    +}
    +
    +static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
    +{
    + BUG_ON((unsigned long)pmd & (PAGE_SIZE-1));
    + free_page((unsigned long)pmd);
    +}
    +
    +static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
    +{
    + set_pud(pud, __pud(__pa(pmd) | PMD_TYPE_TABLE));
    +}
    +
    +#else /* !CONFIG_ARM_LPAE */
    +
    /*
    * Since we have only two-level page tables, these are trivial
    */
    @@ -32,6 +52,8 @@
    #define pmd_free(mm, pmd) do { } while (0)
    #define pud_populate(mm,pmd,pte) BUG()

    +#endif /* CONFIG_ARM_LPAE */
    +
    extern pgd_t *pgd_alloc(struct mm_struct *mm);
    extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);

    @@ -109,7 +131,9 @@ static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t pte,
    {
    pmdval_t pmdval = (pte + PTE_HWTABLE_OFF) | prot;
    pmdp[0] = __pmd(pmdval);
    +#ifndef CONFIG_ARM_LPAE
    pmdp[1] = __pmd(pmdval + 256 * sizeof(pte_t));
    +#endif
    flush_pmd_entry(pmdp);
    }

    diff --git a/arch/arm/include/asm/pgtable-3level.h b/arch/arm/include/asm/pgtable-3level.h
    index ac45358..14a3e28 100644
    --- a/arch/arm/include/asm/pgtable-3level.h
    +++ b/arch/arm/include/asm/pgtable-3level.h
    @@ -98,4 +98,9 @@
    #define L_PTE_MT_DEV_CACHED (_AT(pteval_t, 3) << 2) /* normal inner write-back */
    #define L_PTE_MT_MASK (_AT(pteval_t, 7) << 2)

    +/*
    + * Software PGD flags.
    + */
    +#define L_PGD_SWAPPER (_AT(pgdval_t, 1) << 55) /* swapper_pg_dir entry */
    +
    #endif /* _ASM_PGTABLE_3LEVEL_H */
    diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h
    index 95fefd9..1db9ad6 100644
    --- a/arch/arm/include/asm/pgtable.h
    +++ b/arch/arm/include/asm/pgtable.h
    @@ -165,6 +165,31 @@ extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
    /* to find an entry in a kernel page-table-directory */
    #define pgd_offset_k(addr) pgd_offset(&init_mm, addr)

    +#ifdef CONFIG_ARM_LPAE
    +
    +#define pud_none(pud) (!pud_val(pud))
    +#define pud_bad(pud) (!(pud_val(pud) & 2))
    +#define pud_present(pud) (pud_val(pud))
    +
    +#define pud_clear(pudp) \
    + do { \
    + *pudp = __pud(0); \
    + clean_pmd_entry(pudp); \
    + } while (0)
    +
    +#define set_pud(pudp, pud) \
    + do { \
    + *pudp = pud; \
    + flush_pmd_entry(pudp); \
    + } while (0)
    +
    +static inline pmd_t *pud_page_vaddr(pud_t pud)
    +{
    + return __va(pud_val(pud) & PHYS_MASK & (s32)PAGE_MASK);
    +}
    +
    +#else /* !CONFIG_ARM_LPAE */
    +
    /*
    * The "pud_xxx()" functions here are trivial when the pmd is folded into
    * the pud: the pud entry is never bad, always exists, and can't be set or
    @@ -176,15 +201,43 @@ extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
    #define pud_clear(pudp) do { } while (0)
    #define set_pud(pud,pudp) do { } while (0)

    +#endif /* CONFIG_ARM_LPAE */

    /* Find an entry in the second-level page table.. */
    +#ifdef CONFIG_ARM_LPAE
    +#define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))
    +static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr)
    +{
    + return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(addr);
    +}
    +#else
    static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr)
    {
    return (pmd_t *)pud;
    }
    +#endif

    #define pmd_none(pmd) (!pmd_val(pmd))
    #define pmd_present(pmd) (pmd_val(pmd))
    +
    +#ifdef CONFIG_ARM_LPAE
    +
    +#define pmd_bad(pmd) (!(pmd_val(pmd) & 2))
    +
    +#define copy_pmd(pmdpd,pmdps) \
    + do { \
    + *pmdpd = *pmdps; \
    + flush_pmd_entry(pmdpd); \
    + } while (0)
    +
    +#define pmd_clear(pmdp) \
    + do { \
    + *pmdp = __pmd(0); \
    + clean_pmd_entry(pmdp); \
    + } while (0)
    +
    +#else /* !CONFIG_ARM_LPAE */
    +
    #define pmd_bad(pmd) (pmd_val(pmd) & 2)

    #define copy_pmd(pmdpd,pmdps) \
    @@ -201,6 +254,8 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr)
    clean_pmd_entry(pmdp); \
    } while (0)

    +#endif /* CONFIG_ARM_LPAE */
    +
    static inline pte_t *pmd_page_vaddr(pmd_t pmd)
    {
    return __va(pmd_val(pmd) & PHYS_MASK & (s32)PAGE_MASK);
    @@ -233,9 +288,14 @@ static inline pte_t *pmd_page_vaddr(pmd_t pmd)
    #define pte_page(pte) pfn_to_page(pte_pfn(pte))
    #define mk_pte(page,prot) pfn_pte(page_to_pfn(page), prot)

    -#define set_pte_ext(ptep,pte,ext) cpu_set_pte_ext(ptep,pte,ext)
    #define pte_clear(mm,addr,ptep) set_pte_ext(ptep, __pte(0), 0)

    +#ifdef CONFIG_ARM_LPAE
    +#define set_pte_ext(ptep,pte,ext) cpu_set_pte_ext(ptep,__pte(pte_val(pte)|(ext)))
    +#else
    +#define set_pte_ext(ptep,pte,ext) cpu_set_pte_ext(ptep,pte,ext)
    +#endif
    +
    #if __LINUX_ARM_ARCH__ < 6
    static inline void __sync_icache_dcache(pte_t pteval)
    {
    diff --git a/arch/arm/include/asm/proc-fns.h b/arch/arm/include/asm/proc-fns.h
    index 8ec535e..b5db4f4 100644
    --- a/arch/arm/include/asm/proc-fns.h
    +++ b/arch/arm/include/asm/proc-fns.h
    @@ -65,7 +65,11 @@ extern struct processor {
    * Set a possibly extended PTE. Non-extended PTEs should
    * ignore 'ext'.
    */
    +#ifdef CONFIG_ARM_LPAE
    + void (*set_pte_ext)(pte_t *ptep, pte_t pte);
    +#else
    void (*set_pte_ext)(pte_t *ptep, pte_t pte, unsigned int ext);
    +#endif

    /* Suspend/resume */
    unsigned int suspend_size;
    @@ -79,7 +83,11 @@ extern void cpu_proc_fin(void);
    extern int cpu_do_idle(void);
    extern void cpu_dcache_clean_area(void *, int);
    extern void cpu_do_switch_mm(unsigned long pgd_phys, struct mm_struct *mm);
    +#ifdef CONFIG_ARM_LPAE
    +extern void cpu_set_pte_ext(pte_t *ptep, pte_t pte);
    +#else
    extern void cpu_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext);
    +#endif
    extern void cpu_reset(unsigned long addr) __attribute__((noreturn));
    #else
    #define cpu_proc_init() processor._proc_init()
    @@ -87,7 +95,11 @@ extern void cpu_reset(unsigned long addr) __attribute__((noreturn));
    #define cpu_reset(addr) processor.reset(addr)
    #define cpu_do_idle() processor._do_idle()
    #define cpu_dcache_clean_area(addr,sz) processor.dcache_clean_area(addr,sz)
    +#ifdef CONFIG_ARM_LPAE
    +#define cpu_set_pte_ext(ptep,pte) processor.set_pte_ext(ptep,pte)
    +#else
    #define cpu_set_pte_ext(ptep,pte,ext) processor.set_pte_ext(ptep,pte,ext)
    +#endif
    #define cpu_do_switch_mm(pgd,mm) processor.switch_mm(pgd,mm)
    #endif

    @@ -99,6 +111,18 @@ extern void cpu_resume(void);

    #define cpu_switch_mm(pgd,mm) cpu_do_switch_mm(virt_to_phys(pgd),mm)

    +#ifdef CONFIG_ARM_LPAE
    +#define cpu_get_pgd() \
    + ({ \
    + unsigned long pg, pg2; \
    + __asm__("mrrc p15, 0, %0, %1, c2" \
    + : "=r" (pg), "=r" (pg2) \
    + : \
    + : "cc"); \
    + pg &= ~(PTRS_PER_PGD*sizeof(pgd_t)-1); \
    + (pgd_t *)phys_to_virt(pg); \
    + })
    +#else
    #define cpu_get_pgd() \
    ({ \
    unsigned long pg; \
    @@ -107,6 +131,7 @@ extern void cpu_resume(void);
    pg &= ~0x3fff; \
    (pgd_t *)phys_to_virt(pg); \
    })
    +#endif

    #endif

    diff --git a/arch/arm/mm/ioremap.c b/arch/arm/mm/ioremap.c
    index ab50627..6bdf42c 100644
    --- a/arch/arm/mm/ioremap.c
    +++ b/arch/arm/mm/ioremap.c
    @@ -64,7 +64,7 @@ void __check_kvm_seq(struct mm_struct *mm)
    } while (seq != init_mm.context.kvm_seq);
    }

    -#ifndef CONFIG_SMP
    +#if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE)
    /*
    * Section support is unsafe on SMP - If you iounmap and ioremap a region,
    * the other CPUs will not see this change until their next context switch.
    @@ -195,11 +195,13 @@ void __iomem * __arm_ioremap_pfn_caller(unsigned long pfn,
    unsigned long addr;
    struct vm_struct * area;

    +#ifndef CONFIG_ARM_LPAE
    /*
    * High mappings must be supersection aligned
    */
    if (pfn >= 0x100000 && (__pfn_to_phys(pfn) & ~SUPERSECTION_MASK))
    return NULL;
    +#endif

    /*
    * Don't allow RAM to be mapped - this causes problems with ARMv6+
    @@ -221,7 +223,7 @@ void __iomem * __arm_ioremap_pfn_caller(unsigned long pfn,
    return NULL;
    addr = (unsigned long)area->addr;

    -#ifndef CONFIG_SMP
    +#if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE)
    if (DOMAIN_IO == 0 &&
    (((cpu_architecture() >= CPU_ARCH_ARMv6) && (get_cr() & CR_XP)) ||
    cpu_is_xsc3()) && pfn >= 0x100000 &&
    @@ -292,7 +294,7 @@ EXPORT_SYMBOL(__arm_ioremap);
    void __iounmap(volatile void __iomem *io_addr)
    {
    void *addr = (void *)(PAGE_MASK & (unsigned long)io_addr);
    -#ifndef CONFIG_SMP
    +#if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE)
    struct vm_struct **p, *tmp;

    /*
    diff --git a/arch/arm/mm/pgd.c b/arch/arm/mm/pgd.c
    index b2027c1..a3e78cc 100644
    --- a/arch/arm/mm/pgd.c
    +++ b/arch/arm/mm/pgd.c
    @@ -10,6 +10,7 @@
    #include <linux/mm.h>
    #include <linux/gfp.h>
    #include <linux/highmem.h>
    +#include <linux/slab.h>

    #include <asm/pgalloc.h>
    #include <asm/page.h>
    @@ -17,6 +18,14 @@

    #include "mm.h"

    +#ifdef CONFIG_ARM_LPAE
    +#define __pgd_alloc() kmalloc(PTRS_PER_PGD * sizeof(pgd_t), GFP_KERNEL)
    +#define __pgd_free(pgd) kfree(pgd)
    +#else
    +#define __pgd_alloc() (pgd_t *)__get_free_pages(GFP_KERNEL, 2)
    +#define __pgd_free(pgd) free_pages((unsigned long)pgd, 2)
    +#endif
    +
    /*
    * need to get a 16k page for level 1
    */
    @@ -27,7 +36,7 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
    pmd_t *new_pmd, *init_pmd;
    pte_t *new_pte, *init_pte;

    - new_pgd = (pgd_t *)__get_free_pages(GFP_KERNEL, 2);
    + new_pgd = __pgd_alloc();
    if (!new_pgd)
    goto no_pgd;

    @@ -42,10 +51,25 @@ pgd_t *pgd_alloc(struct mm_struct *mm)

    clean_dcache_area(new_pgd, PTRS_PER_PGD * sizeof(pgd_t));

    +#ifdef CONFIG_ARM_LPAE
    + /*
    + * Allocate PMD table for modules and pkmap mappings.
    + */
    + new_pud = pud_alloc(mm, new_pgd + pgd_index(MODULES_VADDR),
    + MODULES_VADDR);
    + if (!new_pud)
    + goto no_pud;
    +
    + new_pmd = pmd_alloc(mm, new_pud, 0);
    + if (!new_pmd)
    + goto no_pmd;
    +#endif
    +
    if (!vectors_high()) {
    /*
    * On ARM, first page must always be allocated since it
    - * contains the machine vectors.
    + * contains the machine vectors. The vectors are always high
    + * with LPAE.
    */
    new_pud = pud_alloc(mm, new_pgd, 0);
    if (!new_pud)
    @@ -74,7 +98,7 @@ no_pte:
    no_pmd:
    pud_free(mm, new_pud);
    no_pud:
    - free_pages((unsigned long)new_pgd, 2);
    + __pgd_free(new_pgd);
    no_pgd:
    return NULL;
    }
    @@ -111,5 +135,24 @@ no_pud:
    pgd_clear(pgd);
    pud_free(mm, pud);
    no_pgd:
    - free_pages((unsigned long) pgd_base, 2);
    +#ifdef CONFIG_ARM_LPAE
    + /*
    + * Free modules/pkmap or identity pmd tables.
    + */
    + for (pgd = pgd_base; pgd < pgd_base + PTRS_PER_PGD; pgd++) {
    + if (pgd_none_or_clear_bad(pgd))
    + continue;
    + if (pgd_val(*pgd) & L_PGD_SWAPPER)
    + continue;
    + pud = pud_offset(pgd, 0);
    + if (pud_none_or_clear_bad(pud))
    + continue;
    + pmd = pmd_offset(pud, 0);
    + pud_clear(pud);
    + pmd_free(mm, pmd);
    + pgd_clear(pgd);
    + pud_free(mm, pud);
    + }
    +#endif
    + __pgd_free(pgd_base);
    }
    diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S
    index 864a5c9..0459397 100644
    --- a/arch/arm/mm/proc-v7.S
    +++ b/arch/arm/mm/proc-v7.S
    @@ -136,6 +136,13 @@ ENDPROC(cpu_v7_switch_mm)
    */
    ENTRY(cpu_v7_set_pte_ext)
    #ifdef CONFIG_MMU
    +#ifdef CONFIG_ARM_LPAE
    + tst r2, #L_PTE_PRESENT
    + beq 1f
    + tst r3, #1 << (55 - 32) @ L_PTE_DIRTY
    + orreq r2, #L_PTE_RDONLY
    +1: strd r2, r3, [r0]
    +#else /* !CONFIG_ARM_LPAE */
    str r1, [r0] @ linux version

    bic r3, r1, #0x000003f0
    @@ -168,6 +175,7 @@ ENTRY(cpu_v7_set_pte_ext)
    ARM( str r3, [r0, #2048]! )
    THUMB( add r0, r0, #2048 )
    THUMB( str r3, [r0] )
    +#endif /* CONFIG_ARM_LPAE */
    mcr p15, 0, r0, c7, c10, 1 @ flush_pte
    #endif
    mov pc, lr


    \
     
     \ /
      Last update: 2011-05-08 14:57    [W:0.056 / U:31.840 seconds]
    ©2003-2016 Jasper Spaans. hosted at Digital OceanAdvertise on this site