lkml.org 
[lkml]   [2008]   [Jan]   [10]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    Date
    From
    Subject[patch 04/11] PAT x86: Basic PAT implementation
    Originally based on a patch from Eric Biederman, but heavily changed.

    PAT set as below
    PAT(0,WB) | PAT(1,WT) | PAT(2,WC) | PAT(3,UC)
    So, only change from boot setting is UC_MINUS -> WC.

    Also, PAT WC is enabled only on recent Intel CPUs. Other CPUs can be added as
    they are tested with these patches.

    Signed-off-by: Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>
    Signed-off-by: Suresh Siddha <suresh.b.siddha@intel.com>
    Index: linux-2.6.git/arch/x86/mm/Makefile_64
    ===================================================================
    --- linux-2.6.git.orig/arch/x86/mm/Makefile_64 2008-01-08 12:42:58.000000000 -0800
    +++ linux-2.6.git/arch/x86/mm/Makefile_64 2008-01-08 12:43:13.000000000 -0800
    @@ -2,7 +2,7 @@
    # Makefile for the linux x86_64-specific parts of the memory manager.
    #

    -obj-y := init_64.o fault_64.o ioremap_64.o extable.o pageattr_64.o mmap.o
    +obj-y := init_64.o fault_64.o ioremap_64.o extable.o pageattr_64.o mmap.o pat.o
    obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o
    obj-$(CONFIG_NUMA) += numa_64.o
    obj-$(CONFIG_K8_NUMA) += k8topology_64.o
    Index: linux-2.6.git/arch/x86/mm/pat.c
    ===================================================================
    --- /dev/null 1970-01-01 00:00:00.000000000 +0000
    +++ linux-2.6.git/arch/x86/mm/pat.c 2008-01-08 12:43:13.000000000 -0800
    @@ -0,0 +1,70 @@
    +/* Handle caching attributes in page tables (PAT) */
    +#include <linux/mm.h>
    +#include <linux/kernel.h>
    +#include <linux/gfp.h>
    +#include <asm/msr.h>
    +#include <asm/tlbflush.h>
    +#include <asm/processor.h>
    +
    +static u64 boot_pat_state;
    +int pat_wc_enabled = 0;
    +
    +enum {
    + PAT_UC = 0, /* uncached */
    + PAT_WC = 1, /* Write combining */
    + PAT_WT = 4, /* Write Through */
    + PAT_WP = 5, /* Write Protected */
    + PAT_WB = 6, /* Write Back (default) */
    + PAT_UC_MINUS = 7, /* UC, but can be overriden by MTRR */
    +};
    +
    +#define PAT(x,y) ((u64)PAT_ ## y << ((x)*8))
    +
    +static int pat_known_cpu(void)
    +{
    + if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
    + (boot_cpu_data.x86 == 0xF ||
    + (boot_cpu_data.x86 == 0x6 && boot_cpu_data.x86_model >= 0x15)))
    + return cpu_has_pat;
    +
    + return 0;
    +}
    +
    +void pat_init(void)
    +{
    + u64 pat;
    + if (!smp_processor_id() && !pat_known_cpu())
    + return;
    +
    + if (smp_processor_id() && !pat_wc_enabled)
    + return;
    +
    + /* Set PWT+PCD to Write-Combining. All other bits stay the same */
    + /* PTE encoding used in Linux:
    + PAT
    + |PCD
    + ||PWT
    + |||
    + 000 WB default
    + 010 WC _PAGE_WC
    + 011 UC _PAGE_PCD
    + PAT bit unused */
    + pat = PAT(0,WB) | PAT(1,WT) | PAT(2,WC) | PAT(3,UC) |
    + PAT(4,WB) | PAT(5,WT) | PAT(6,WC) | PAT(7,UC);
    +
    + if (!pat_wc_enabled) {
    + rdmsrl(MSR_IA32_CR_PAT, boot_pat_state);
    + pat_wc_enabled = 1;
    + }
    +
    + wrmsrl(MSR_IA32_CR_PAT, pat);
    + printk("cpu %d, old 0x%Lx, new 0x%Lx\n", smp_processor_id(),
    + boot_pat_state, pat);
    +}
    +
    +#undef PAT
    +
    +void pat_shutdown(void)
    +{
    +}
    +
    Index: linux-2.6.git/arch/x86/pci/i386.c
    ===================================================================
    --- linux-2.6.git.orig/arch/x86/pci/i386.c 2008-01-08 12:42:58.000000000 -0800
    +++ linux-2.6.git/arch/x86/pci/i386.c 2008-01-08 12:43:13.000000000 -0800
    @@ -301,7 +301,6 @@
    enum pci_mmap_state mmap_state, int write_combine)
    {
    unsigned long prot;
    -
    /* I/O space cannot be accessed via normal processor loads and
    * stores on this platform.
    */
    @@ -311,14 +310,25 @@
    /* Leave vm_pgoff as-is, the PCI space address is the physical
    * address on this platform.
    */
    - prot = pgprot_val(vma->vm_page_prot);
    - if (boot_cpu_data.x86 > 3)
    - prot |= _PAGE_PCD | _PAGE_PWT;
    - vma->vm_page_prot = __pgprot(prot);
    + if (pat_wc_enabled) {
    + if (write_combine) {
    + vma->vm_page_prot =
    + pgprot_writecombine(vma->vm_page_prot);
    + } else {
    + vma->vm_page_prot =
    + pgprot_noncached(vma->vm_page_prot);
    + }
    + } else {
    + /* Write-combine setting is ignored, it is changed via the mtrr
    + * interfaces on this platform.
    + */
    + prot = pgprot_val(vma->vm_page_prot);
    + if (boot_cpu_data.x86 > 3) {
    + prot |= _PAGE_PCD;
    + }
    + vma->vm_page_prot = __pgprot(prot);
    + }

    - /* Write-combine setting is ignored, it is changed via the mtrr
    - * interfaces on this platform.
    - */
    if (io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
    vma->vm_end - vma->vm_start,
    vma->vm_page_prot))
    Index: linux-2.6.git/include/asm-x86/cpufeature.h
    ===================================================================
    --- linux-2.6.git.orig/include/asm-x86/cpufeature.h 2008-01-08 12:42:58.000000000 -0800
    +++ linux-2.6.git/include/asm-x86/cpufeature.h 2008-01-08 12:43:13.000000000 -0800
    @@ -167,6 +167,7 @@
    #define cpu_has_pebs boot_cpu_has(X86_FEATURE_PEBS)
    #define cpu_has_clflush boot_cpu_has(X86_FEATURE_CLFLSH)
    #define cpu_has_bts boot_cpu_has(X86_FEATURE_BTS)
    +#define cpu_has_pat boot_cpu_has(X86_FEATURE_PAT)

    #if defined(CONFIG_X86_INVLPG) || defined(CONFIG_X86_64)
    # define cpu_has_invlpg 1
    Index: linux-2.6.git/include/asm-x86/msr-index.h
    ===================================================================
    --- linux-2.6.git.orig/include/asm-x86/msr-index.h 2008-01-08 12:42:58.000000000 -0800
    +++ linux-2.6.git/include/asm-x86/msr-index.h 2008-01-08 12:43:13.000000000 -0800
    @@ -70,6 +70,7 @@
    #define DEBUGCTLMSR_LBR (1UL << _DEBUGCTLMSR_LBR)
    #define DEBUGCTLMSR_BTF (1UL << _DEBUGCTLMSR_BTF)

    +#define MSR_IA32_CR_PAT 0x00000277
    #define MSR_IA32_MC0_CTL 0x00000400
    #define MSR_IA32_MC0_STATUS 0x00000401
    #define MSR_IA32_MC0_ADDR 0x00000402
    Index: linux-2.6.git/include/asm-x86/pgtable_64.h
    ===================================================================
    --- linux-2.6.git.orig/include/asm-x86/pgtable_64.h 2008-01-08 12:42:58.000000000 -0800
    +++ linux-2.6.git/include/asm-x86/pgtable_64.h 2008-01-08 12:43:13.000000000 -0800
    @@ -158,7 +158,7 @@
    #define _PAGE_RW (_AC(1, UL)<<_PAGE_BIT_RW)
    #define _PAGE_USER (_AC(1, UL)<<_PAGE_BIT_USER)
    #define _PAGE_PWT (_AC(1, UL)<<_PAGE_BIT_PWT)
    -#define _PAGE_PCD (_AC(1, UL)<<_PAGE_BIT_PCD)
    +#define _PAGE_PCD ((_AC(1, UL)<<_PAGE_BIT_PCD) | _PAGE_PWT)
    #define _PAGE_ACCESSED (_AC(1, UL)<<_PAGE_BIT_ACCESSED)
    #define _PAGE_DIRTY (_AC(1, UL)<<_PAGE_BIT_DIRTY)
    /* 2MB page */
    @@ -167,6 +167,10 @@
    #define _PAGE_FILE (_AC(1, UL)<<_PAGE_BIT_FILE)
    /* Global TLB entry */
    #define _PAGE_GLOBAL (_AC(1, UL)<<_PAGE_BIT_GLOBAL)
    +/* We redefine PCD to be write combining. PAT bit is not used */
    +#define _PAGE_WC (_AC(1, UL)<<_PAGE_BIT_PCD)
    +
    +#define _PAGE_CACHE_MASK (_PAGE_PCD)

    #define _PAGE_PROTNONE 0x080 /* If not present */
    #define _PAGE_NX (_AC(1, UL)<<_PAGE_BIT_NX)
    @@ -189,13 +193,15 @@
    #define __PAGE_KERNEL_EXEC \
    (_PAGE_PRESENT | _PAGE_RW | _PAGE_DIRTY | _PAGE_ACCESSED)
    #define __PAGE_KERNEL_NOCACHE \
    - (_PAGE_PRESENT | _PAGE_RW | _PAGE_DIRTY | _PAGE_PCD | _PAGE_PWT | _PAGE_ACCESSED | _PAGE_NX)
    + (_PAGE_PRESENT | _PAGE_RW | _PAGE_DIRTY | _PAGE_PCD | _PAGE_ACCESSED | _PAGE_NX)
    +#define __PAGE_KERNEL_WC \
    + (_PAGE_PRESENT | _PAGE_RW | _PAGE_DIRTY | _PAGE_WC | _PAGE_ACCESSED | _PAGE_NX)
    #define __PAGE_KERNEL_RO \
    (_PAGE_PRESENT | _PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_NX)
    #define __PAGE_KERNEL_VSYSCALL \
    (_PAGE_PRESENT | _PAGE_USER | _PAGE_ACCESSED)
    #define __PAGE_KERNEL_VSYSCALL_NOCACHE \
    - (_PAGE_PRESENT | _PAGE_USER | _PAGE_ACCESSED | _PAGE_PCD | _PAGE_PWT)
    + (_PAGE_PRESENT | _PAGE_USER | _PAGE_ACCESSED | _PAGE_PCD)
    #define __PAGE_KERNEL_LARGE \
    (__PAGE_KERNEL | _PAGE_PSE)
    #define __PAGE_KERNEL_LARGE_EXEC \
    @@ -207,6 +213,7 @@
    #define PAGE_KERNEL_EXEC MAKE_GLOBAL(__PAGE_KERNEL_EXEC)
    #define PAGE_KERNEL_RO MAKE_GLOBAL(__PAGE_KERNEL_RO)
    #define PAGE_KERNEL_NOCACHE MAKE_GLOBAL(__PAGE_KERNEL_NOCACHE)
    +#define PAGE_KERNEL_WC MAKE_GLOBAL(__PAGE_KERNEL_WC)
    #define PAGE_KERNEL_VSYSCALL32 __pgprot(__PAGE_KERNEL_VSYSCALL)
    #define PAGE_KERNEL_VSYSCALL MAKE_GLOBAL(__PAGE_KERNEL_VSYSCALL)
    #define PAGE_KERNEL_LARGE MAKE_GLOBAL(__PAGE_KERNEL_LARGE)
    @@ -302,8 +309,24 @@

    /*
    * Macro to mark a page protection value as "uncacheable".
    + * Accesses through a uncached translation bypasses the cache
    + * and do not allow for consecutive writes to be combined.
    */
    -#define pgprot_noncached(prot) (__pgprot(pgprot_val(prot) | _PAGE_PCD | _PAGE_PWT))
    +#define pgprot_noncached(prot) \
    + __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_MASK) | _PAGE_PCD)
    +
    +/*
    + * Macro to make mark a page protection value as "write-combining".
    + * Accesses through a write-combining translation works bypasses the
    + * caches, but does allow for consecutive writes to be combined into
    + * single (but larger) write transactions.
    + * This is mostly useful for IO accesses, for memory it is often slower.
    + * It also implies uncached.
    + */
    +#define pgprot_writecombine(prot) \
    + __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_MASK) | _PAGE_WC)
    +
    +#define pgprot_nonstd(prot) (pgprot_val(prot) & _PAGE_CACHE_MASK)

    static inline int pmd_large(pmd_t pte) {
    return (pmd_val(pte) & __LARGE_PTE) == __LARGE_PTE;
    @@ -417,6 +440,7 @@
    #define pgtable_cache_init() do { } while (0)
    #define check_pgt_cache() do { } while (0)

    +/* AGP users use MTRRs for now. Need to add an ioctl to agpgart for WC */
    #define PAGE_AGP PAGE_KERNEL_NOCACHE
    #define HAVE_PAGE_AGP 1

    Index: linux-2.6.git/arch/x86/kernel/cpu/mtrr/generic.c
    ===================================================================
    --- linux-2.6.git.orig/arch/x86/kernel/cpu/mtrr/generic.c 2008-01-08 12:42:58.000000000 -0800
    +++ linux-2.6.git/arch/x86/kernel/cpu/mtrr/generic.c 2008-01-08 12:43:13.000000000 -0800
    @@ -79,19 +79,23 @@
    base, base + step - 1, mtrr_attrib_to_str(*types));
    }

    +static void prepare_set(void);
    +static void post_set(void);
    +
    /* Grab all of the MTRR state for this CPU into *state */
    void __init get_mtrr_state(void)
    {
    unsigned int i;
    struct mtrr_var_range *vrs;
    unsigned lo, dummy;
    + unsigned long flags;

    if (!mtrr_state.var_ranges) {
    - mtrr_state.var_ranges = kmalloc(num_var_ranges * sizeof (struct mtrr_var_range),
    + mtrr_state.var_ranges = kmalloc(num_var_ranges * sizeof (struct mtrr_var_range),
    GFP_KERNEL);
    if (!mtrr_state.var_ranges)
    return;
    - }
    + }
    vrs = mtrr_state.var_ranges;

    rdmsr(MTRRcap_MSR, lo, dummy);
    @@ -137,6 +141,16 @@
    printk(KERN_INFO "MTRR %u disabled\n", i);
    }
    }
    +
    + /* PAT setup for BP. we need to go through sync steps here */
    + local_irq_save(flags);
    + prepare_set();
    +
    + pat_init();
    +
    + post_set();
    + local_irq_restore(flags);
    +
    }

    /* Some BIOS's are fucked and don't set all MTRRs the same! */
    @@ -399,6 +413,9 @@
    /* Actually set the state */
    mask = set_mtrr_state();

    + /* also set PAT */
    + pat_init();
    +
    post_set();
    local_irq_restore(flags);

    Index: linux-2.6.git/arch/x86/mm/Makefile_32
    ===================================================================
    --- linux-2.6.git.orig/arch/x86/mm/Makefile_32 2008-01-08 12:42:58.000000000 -0800
    +++ linux-2.6.git/arch/x86/mm/Makefile_32 2008-01-08 12:43:13.000000000 -0800
    @@ -2,7 +2,7 @@
    # Makefile for the linux i386-specific parts of the memory manager.
    #

    -obj-y := init_32.o pgtable_32.o fault_32.o ioremap_32.o extable.o pageattr_32.o mmap.o
    +obj-y := init_32.o pgtable_32.o fault_32.o ioremap_32.o extable.o pageattr_32.o mmap.o pat.o

    obj-$(CONFIG_NUMA) += discontig_32.o
    obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o
    Index: linux-2.6.git/include/asm-x86/pgtable_32.h
    ===================================================================
    --- linux-2.6.git.orig/include/asm-x86/pgtable_32.h 2008-01-08 12:42:58.000000000 -0800
    +++ linux-2.6.git/include/asm-x86/pgtable_32.h 2008-01-08 12:43:36.000000000 -0800
    @@ -107,7 +107,7 @@
    #define _PAGE_RW 0x002
    #define _PAGE_USER 0x004
    #define _PAGE_PWT 0x008
    -#define _PAGE_PCD 0x010
    +#define _PAGE_PCD 0x018
    #define _PAGE_ACCESSED 0x020
    #define _PAGE_DIRTY 0x040
    #define _PAGE_PSE 0x080 /* 4 MB (or 2MB) page, Pentium+, if present.. */
    @@ -116,6 +116,12 @@
    #define _PAGE_UNUSED2 0x400
    #define _PAGE_UNUSED3 0x800

    +/* We redefine PCD to be write combining. PAT bit is not used */
    +
    +#define _PAGE_WC 0x10
    +
    +#define _PAGE_CACHE_MASK 0x18
    +
    /* If _PAGE_PRESENT is clear, we use these: */
    #define _PAGE_FILE 0x040 /* nonlinear file mapping, saved PTE; unset:swap */
    #define _PAGE_PROTNONE 0x080 /* if the user mapped it with PROT_NONE;
    @@ -156,7 +162,7 @@
    extern unsigned long long __PAGE_KERNEL, __PAGE_KERNEL_EXEC;
    #define __PAGE_KERNEL_RO (__PAGE_KERNEL & ~_PAGE_RW)
    #define __PAGE_KERNEL_RX (__PAGE_KERNEL_EXEC & ~_PAGE_RW)
    -#define __PAGE_KERNEL_NOCACHE (__PAGE_KERNEL | _PAGE_PCD | _PAGE_PWT)
    +#define __PAGE_KERNEL_NOCACHE (__PAGE_KERNEL | _PAGE_PCD)
    #define __PAGE_KERNEL_LARGE (__PAGE_KERNEL | _PAGE_PSE)
    #define __PAGE_KERNEL_LARGE_EXEC (__PAGE_KERNEL_EXEC | _PAGE_PSE)

    @@ -358,7 +364,18 @@
    * it, this is a no-op.
    */
    #define pgprot_noncached(prot) ((boot_cpu_data.x86 > 3) \
    - ? (__pgprot(pgprot_val(prot) | _PAGE_PCD | _PAGE_PWT)) : (prot))
    + ? (__pgprot(pgprot_val(prot) | _PAGE_PCD)) : (prot))
    +
    +/*
    + * Macro to make mark a page protection value as "write-combining".
    + * Accesses through a write-combining translation works bypasses the
    + * caches, but does allow for consecutive writes to be combined into
    + * single (but larger) write transactions.
    + * This is mostly useful for IO accesses, for memory it is often slower.
    + * It also implies uncached.
    + */
    +#define pgprot_writecombine(prot) \
    + __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_MASK) | _PAGE_WC)

    /*
    * Conversion functions: convert a page and protection to a page entry,
    Index: linux-2.6.git/include/asm-x86/processor.h
    ===================================================================
    --- linux-2.6.git.orig/include/asm-x86/processor.h 2008-01-08 12:42:58.000000000 -0800
    +++ linux-2.6.git/include/asm-x86/processor.h 2008-01-08 12:43:13.000000000 -0800
    @@ -647,6 +647,10 @@

    extern int force_mwait;

    +extern int pat_wc_enabled;
    +extern void pat_init(void);
    +extern void pat_shutdown(void);
    +
    extern void select_idle_routine(const struct cpuinfo_x86 *c);

    extern unsigned long boot_option_idle_override;
    Index: linux-2.6.git/include/asm-generic/pgtable.h
    ===================================================================
    --- linux-2.6.git.orig/include/asm-generic/pgtable.h 2008-01-08 12:42:58.000000000 -0800
    +++ linux-2.6.git/include/asm-generic/pgtable.h 2008-01-08 12:43:13.000000000 -0800
    @@ -154,6 +154,10 @@
    })
    #endif

    +#ifndef pgprot_writecombine
    +#define pgprot_writecombine pgprot_noncached
    +#endif
    +
    /*
    * When walking page tables, we usually want to skip any p?d_none entries;
    * and any p?d_bad entries - reporting the error before resetting to none.
    Index: linux-2.6.git/arch/x86/mm/ioremap_32.c
    ===================================================================
    --- linux-2.6.git.orig/arch/x86/mm/ioremap_32.c 2008-01-08 12:42:58.000000000 -0800
    +++ linux-2.6.git/arch/x86/mm/ioremap_32.c 2008-01-08 12:43:13.000000000 -0800
    @@ -119,7 +119,7 @@
    void __iomem *ioremap_nocache (unsigned long phys_addr, unsigned long size)
    {
    unsigned long last_addr;
    - void __iomem *p = __ioremap(phys_addr, size, _PAGE_PCD | _PAGE_PWT);
    + void __iomem *p = __ioremap(phys_addr, size, _PAGE_PCD);
    if (!p)
    return p;

    Index: linux-2.6.git/arch/x86/mm/ioremap_64.c
    ===================================================================
    --- linux-2.6.git.orig/arch/x86/mm/ioremap_64.c 2008-01-08 12:42:58.000000000 -0800
    +++ linux-2.6.git/arch/x86/mm/ioremap_64.c 2008-01-08 12:43:13.000000000 -0800
    @@ -138,7 +138,7 @@

    void __iomem *ioremap_nocache (unsigned long phys_addr, unsigned long size)
    {
    - return __ioremap(phys_addr, size, _PAGE_PCD | _PAGE_PWT);
    + return __ioremap(phys_addr, size, _PAGE_PCD);
    }
    EXPORT_SYMBOL(ioremap_nocache);

    --


    \
     
     \ /
      Last update: 2008-01-10 19:51    [W:0.043 / U:124.864 seconds]
    ©2003-2016 Jasper Spaans. hosted at Digital OceanAdvertise on this site