lkml.org 
[lkml]   [2020]   [Apr]   [3]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[PATCH v1 4/6] tlb: mmu_gather: add tlb_set_*_range APIs
    Date
    From: "Peter Zijlstra (Intel)" <peterz@infradead.org>

    tlb_set_{pte|pmd|pud|p4d}_range() adjust the tlb->start and
    tlb->end, then set corresponding cleared_*.

    Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
    Signed-off-by: Zhenyu Ye <yezhenyu2@huawei.com>
    ---
    include/asm-generic/tlb.h | 55 ++++++++++++++++++++++++++++-----------
    1 file changed, 40 insertions(+), 15 deletions(-)

    diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
    index f391f6b500b4..ee91310a65c6 100644
    --- a/include/asm-generic/tlb.h
    +++ b/include/asm-generic/tlb.h
    @@ -511,6 +511,38 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm
    }
    #endif

    +/*
    + * tlb_set_{pte|pmd|pud|p4d}_range() adjust the tlb->start and tlb->end,
    + * and set corresponding cleared_*.
    + */
    +static inline void tlb_set_pte_range(struct mmu_gather *tlb,
    + unsigned long address, unsigned long size)
    +{
    + __tlb_adjust_range(tlb, address, size);
    + tlb->cleared_ptes = 1;
    +}
    +
    +static inline void tlb_set_pmd_range(struct mmu_gather *tlb,
    + unsigned long address, unsigned long size)
    +{
    + __tlb_adjust_range(tlb, address, size);
    + tlb->cleared_pmds = 1;
    +}
    +
    +static inline void tlb_set_pud_range(struct mmu_gather *tlb,
    + unsigned long address, unsigned long size)
    +{
    + __tlb_adjust_range(tlb, address, size);
    + tlb->cleared_puds = 1;
    +}
    +
    +static inline void tlb_set_p4d_range(struct mmu_gather *tlb,
    + unsigned long address, unsigned long size)
    +{
    + __tlb_adjust_range(tlb, address, size);
    + tlb->cleared_p4ds = 1;
    +}
    +
    #ifndef __tlb_remove_tlb_entry
    #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
    #endif
    @@ -524,19 +556,17 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm
    */
    #define tlb_remove_tlb_entry(tlb, ptep, address) \
    do { \
    - __tlb_adjust_range(tlb, address, PAGE_SIZE); \
    - tlb->cleared_ptes = 1; \
    + tlb_set_pte_range(tlb, address, PAGE_SIZE); \
    __tlb_remove_tlb_entry(tlb, ptep, address); \
    } while (0)

    #define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \
    do { \
    unsigned long _sz = huge_page_size(h); \
    - __tlb_adjust_range(tlb, address, _sz); \
    if (_sz == PMD_SIZE) \
    - tlb->cleared_pmds = 1; \
    + tlb_set_pmd_range(tlb, address, _sz); \
    else if (_sz == PUD_SIZE) \
    - tlb->cleared_puds = 1; \
    + tlb_set_pud_range(tlb, address, _sz); \
    __tlb_remove_tlb_entry(tlb, ptep, address); \
    } while (0)

    @@ -550,8 +580,7 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm

    #define tlb_remove_pmd_tlb_entry(tlb, pmdp, address) \
    do { \
    - __tlb_adjust_range(tlb, address, HPAGE_PMD_SIZE); \
    - tlb->cleared_pmds = 1; \
    + tlb_set_pmd_range(tlb, address, HPAGE_PMD_SIZE); \
    __tlb_remove_pmd_tlb_entry(tlb, pmdp, address); \
    } while (0)

    @@ -565,8 +594,7 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm

    #define tlb_remove_pud_tlb_entry(tlb, pudp, address) \
    do { \
    - __tlb_adjust_range(tlb, address, HPAGE_PUD_SIZE); \
    - tlb->cleared_puds = 1; \
    + tlb_set_pud_range(tlb, address, HPAGE_PUD_SIZE); \
    __tlb_remove_pud_tlb_entry(tlb, pudp, address); \
    } while (0)

    @@ -591,9 +619,8 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm
    #ifndef pte_free_tlb
    #define pte_free_tlb(tlb, ptep, address) \
    do { \
    - __tlb_adjust_range(tlb, address, PAGE_SIZE); \
    + tlb_set_pmd_range(tlb, address, PAGE_SIZE); \
    tlb->freed_tables = 1; \
    - tlb->cleared_pmds = 1; \
    __pte_free_tlb(tlb, ptep, address); \
    } while (0)
    #endif
    @@ -601,9 +628,8 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm
    #ifndef pmd_free_tlb
    #define pmd_free_tlb(tlb, pmdp, address) \
    do { \
    - __tlb_adjust_range(tlb, address, PAGE_SIZE); \
    + tlb_set_pud_range(tlb, address, PAGE_SIZE); \
    tlb->freed_tables = 1; \
    - tlb->cleared_puds = 1; \
    __pmd_free_tlb(tlb, pmdp, address); \
    } while (0)
    #endif
    @@ -611,9 +637,8 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm
    #ifndef pud_free_tlb
    #define pud_free_tlb(tlb, pudp, address) \
    do { \
    - __tlb_adjust_range(tlb, address, PAGE_SIZE); \
    + tlb_set_p4d_range(tlb, address, PAGE_SIZE); \
    tlb->freed_tables = 1; \
    - tlb->cleared_p4ds = 1; \
    __pud_free_tlb(tlb, pudp, address); \
    } while (0)
    #endif
    --
    2.19.1

    \
     
     \ /
      Last update: 2020-04-03 11:01    [W:2.443 / U:0.192 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site