lkml.org 
[lkml]   [2016]   [Aug]   [22]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[RFC PATCH v2 20/20] x86: Add support to make use of Secure Memory Encryption
    Date
    This patch adds the support to check if SME has been enabled and if the
    mem_encrypt=on command line option is set. If both of these conditions
    are true, then the encryption mask is set and the kernel is encrypted
    "in place."

    Signed-off-by: Tom Lendacky <thomas.lendacky@amd.com>
    ---
    Documentation/kernel-parameters.txt | 3
    arch/x86/kernel/asm-offsets.c | 2
    arch/x86/kernel/mem_encrypt.S | 302 +++++++++++++++++++++++++++++++++++
    arch/x86/mm/mem_encrypt.c | 2
    4 files changed, 309 insertions(+)

    diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt
    index 46c030a..a1986c8 100644
    --- a/Documentation/kernel-parameters.txt
    +++ b/Documentation/kernel-parameters.txt
    @@ -2268,6 +2268,9 @@ bytes respectively. Such letter suffixes can also be entirely omitted.
    memory contents and reserves bad memory
    regions that are detected.

    + mem_encrypt=on [X86_64] Enable memory encryption on processors
    + that support this feature.
    +
    meye.*= [HW] Set MotionEye Camera parameters
    See Documentation/video4linux/meye.txt.

    diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c
    index 2bd5c6f..e485ada 100644
    --- a/arch/x86/kernel/asm-offsets.c
    +++ b/arch/x86/kernel/asm-offsets.c
    @@ -85,6 +85,8 @@ void common(void) {
    OFFSET(BP_init_size, boot_params, hdr.init_size);
    OFFSET(BP_pref_address, boot_params, hdr.pref_address);
    OFFSET(BP_code32_start, boot_params, hdr.code32_start);
    + OFFSET(BP_cmd_line_ptr, boot_params, hdr.cmd_line_ptr);
    + OFFSET(BP_ext_cmd_line_ptr, boot_params, ext_cmd_line_ptr);

    BLANK();
    DEFINE(PTREGS_SIZE, sizeof(struct pt_regs));
    diff --git a/arch/x86/kernel/mem_encrypt.S b/arch/x86/kernel/mem_encrypt.S
    index f2e0536..bf9f6a9 100644
    --- a/arch/x86/kernel/mem_encrypt.S
    +++ b/arch/x86/kernel/mem_encrypt.S
    @@ -12,13 +12,230 @@

    #include <linux/linkage.h>

    +#include <asm/processor-flags.h>
    +#include <asm/pgtable.h>
    +#include <asm/page.h>
    +#include <asm/msr.h>
    +#include <asm/asm-offsets.h>
    +
    .text
    .code64
    ENTRY(sme_enable)
    +#ifdef CONFIG_AMD_MEM_ENCRYPT
    + /* Check for AMD processor */
    + xorl %eax, %eax
    + cpuid
    + cmpl $0x68747541, %ebx # AuthenticAMD
    + jne .Lmem_encrypt_exit
    + cmpl $0x69746e65, %edx
    + jne .Lmem_encrypt_exit
    + cmpl $0x444d4163, %ecx
    + jne .Lmem_encrypt_exit
    +
    + /* Check for memory encryption leaf */
    + movl $0x80000000, %eax
    + cpuid
    + cmpl $0x8000001f, %eax
    + jb .Lmem_encrypt_exit
    +
    + /*
    + * Check for memory encryption feature:
    + * CPUID Fn8000_001F[EAX] - Bit 0
    + * Secure Memory Encryption support
    + * CPUID Fn8000_001F[EBX] - Bits 5:0
    + * Pagetable bit position used to indicate encryption
    + * CPUID Fn8000_001F[EBX] - Bits 11:6
    + * Reduction in physical address space (in bits) when enabled
    + */
    + movl $0x8000001f, %eax
    + cpuid
    + bt $0, %eax
    + jnc .Lmem_encrypt_exit
    +
    + /* Check if BIOS/UEFI has allowed memory encryption */
    + movl $MSR_K8_SYSCFG, %ecx
    + rdmsr
    + bt $MSR_K8_SYSCFG_MEM_ENCRYPT_BIT, %eax
    + jnc .Lmem_encrypt_exit
    +
    + /* Check for the mem_encrypt=on command line option */
    + push %rsi /* Save RSI (real_mode_data) */
    + push %rbx /* Save CPUID information */
    + movl BP_ext_cmd_line_ptr(%rsi), %ecx
    + shlq $32, %rcx
    + movl BP_cmd_line_ptr(%rsi), %edi
    + addq %rcx, %rdi
    + leaq mem_encrypt_enable_option(%rip), %rsi
    + call cmdline_find_option_bool
    + pop %rbx /* Restore CPUID information */
    + pop %rsi /* Restore RSI (real_mode_data) */
    + testl %eax, %eax
    + jz .Lno_mem_encrypt
    +
    + /* Set memory encryption mask */
    + movl %ebx, %ecx
    + andl $0x3f, %ecx
    + bts %ecx, sme_me_mask(%rip)
    +
    +.Lno_mem_encrypt:
    + /*
    + * BIOS/UEFI has allowed memory encryption so we need to set
    + * the amount of physical address space reduction even if
    + * the user decides not to use memory encryption.
    + */
    + movl %ebx, %ecx
    + shrl $6, %ecx
    + andl $0x3f, %ecx
    + movb %cl, sme_me_loss(%rip)
    +
    +.Lmem_encrypt_exit:
    +#endif /* CONFIG_AMD_MEM_ENCRYPT */
    +
    ret
    ENDPROC(sme_enable)

    ENTRY(sme_encrypt_kernel)
    +#ifdef CONFIG_AMD_MEM_ENCRYPT
    + /* If SME is not active then no need to encrypt the kernel */
    + cmpq $0, sme_me_mask(%rip)
    + jz .Lencrypt_exit
    +
    + /*
    + * Encrypt the kernel.
    + * Pagetables for performing kernel encryption:
    + * 0x0000000000 - 0x00FFFFFFFF will map just the memory occupied by
    + * the kernel as encrypted memory
    + * 0x8000000000 - 0x80FFFFFFFF will map all memory as write-protected,
    + * non-encrypted
    + *
    + * The use of write-protected memory will prevent any of the
    + * non-encrypted memory from being cached.
    + *
    + * 0x00... and 0x80... represent the first and second PGD entries.
    + *
    + * This collection of entries will be created in an area outside
    + * of the area that is being encrypted (outside the kernel) and
    + * requires 11 4K pages:
    + * 1 - PGD
    + * 2 - PUDs (1 for each mapping)
    + * 8 - PMDs (4 for each mapping)
    + */
    + leaq _end(%rip), %rdi
    + addq $~PMD_PAGE_MASK, %rdi
    + andq $PMD_PAGE_MASK, %rdi /* RDI points to the new PGD */
    +
    + /* Clear the pagetable memory */
    + movq %rdi, %rbx /* Save pointer to PGD */
    + movl $(4096 * 11), %ecx
    + xorl %eax, %eax
    + rep stosb
    + movq %rbx, %rdi /* Restore pointer to PGD */
    +
    + /* Set up PGD entries for the two mappings */
    + leaq (0x1000 + 0x03)(%rdi), %rbx /* PUD for encrypted kernel */
    + movq %rbx, (%rdi)
    + leaq (0x2000 + 0x03)(%rdi), %rbx /* PUD for unencrypted kernel */
    + movq %rbx, 8(%rdi)
    +
    + /* Set up PUD entries (4 per mapping) for the two mappings */
    + leaq (0x3000 + 0x03)(%rdi), %rbx /* PMD for encrypted kernel */
    + leaq (0x7000 + 0x03)(%rdi), %rdx /* PMD for unencrypted kernel */
    + xorq %rcx, %rcx
    +1:
    + /* Populate the PUD entries in each mapping */
    + movq %rbx, 0x1000(%rdi, %rcx, 8)
    + movq %rdx, 0x2000(%rdi, %rcx, 8)
    + addq $0x1000, %rbx
    + addq $0x1000, %rdx
    + incq %rcx
    + cmpq $4, %rcx
    + jb 1b
    +
    + /*
    + * Set up PMD entries (4GB worth) for the two mappings.
    + * For the encrypted kernel mapping, when R11 is above RDX
    + * and below RDI then we know we are in the kernel and we
    + * set the encryption mask for that PMD entry.
    + *
    + * The use of _PAGE_PAT and _PAGE_PWT will provide for the
    + * write-protected mapping.
    + */
    + movq sme_me_mask(%rip), %r10
    + movq $__PAGE_KERNEL_LARGE_EXEC, %r11
    + andq $~_PAGE_GLOBAL, %r11
    + movq %r11, %r12
    + andq $~_PAGE_CACHE_MASK, %r12
    + orq $(_PAGE_PAT | _PAGE_PWT), %r12 /* PA5 index */
    + xorq %rcx, %rcx
    + leaq _text(%rip), %rdx /* RDX points to start of kernel */
    +1:
    + /* Populate the PMD entries in each mapping */
    + movq %r11, 0x3000(%rdi, %rcx, 8)
    + movq %r12, 0x7000(%rdi, %rcx, 8)
    +
    + /*
    + * Check if we are in the kernel range, and if so, set the
    + * memory encryption mask.
    + */
    + cmpq %r11, %rdx
    + jae 2f
    + cmpq %r11, %rdi
    + jbe 2f
    + orq %r10, 0x3000(%rdi, %rcx, 8)
    +2:
    + addq $PMD_SIZE, %r11
    + addq $PMD_SIZE, %r12
    + incq %rcx
    + cmpq $2048, %rcx
    + jb 1b
    +
    + /*
    + * Set up a one page stack in the non-encrypted memory area.
    + * Set RAX to point to the next page in memory after all the
    + * page tables. The stack grows from the bottom so point to
    + * the end of the page.
    + */
    + leaq (4096 * 11)(%rdi), %rax
    + addq $PAGE_SIZE, %rax
    + movq %rsp, %rbp
    + movq %rax, %rsp
    + push %rbp /* Save original stack pointer */
    +
    + push %rsi /* Save RSI (real mode data) */
    +
    + /*
    + * Copy encryption routine into safe memory
    + * - RAX points to the page after all the page tables and stack
    + * where the routine will copied
    + * - RDI points to the PGD table
    + * - Setup registers for call
    + * and then call it
    + */
    + movq %rdi, %rbx
    +
    + leaq .Lencrypt_start(%rip), %rsi
    + movq %rax, %rdi
    + movq $(.Lencrypt_stop - .Lencrypt_start), %rcx
    + rep movsb
    +
    + leaq _text(%rip), %rsi /* Kernel start */
    + movq %rbx, %rcx /* New PGD start */
    + subq %rsi, %rcx /* Size of area to encrypt */
    +
    + movq %rsi, %rdi /* Encrypted kernel space start */
    + movq $1, %rsi
    + shlq $PGDIR_SHIFT, %rsi
    + addq %rdi, %rsi /* Non-encrypted kernel start */
    +
    + /* Call the encryption routine */
    + call *%rax
    +
    + pop %rsi /* Restore RSI (real mode data ) */
    +
    + pop %rsp /* Restore original stack pointer */
    +.Lencrypt_exit:
    +#endif /* CONFIG_AMD_MEM_ENCRYPT */
    +
    ret
    ENDPROC(sme_encrypt_kernel)

    @@ -28,6 +245,87 @@ ENTRY(sme_get_me_loss)
    ret
    ENDPROC(sme_get_me_loss)

    +#ifdef CONFIG_AMD_MEM_ENCRYPT
    +/*
    + * Routine used to encrypt kernel.
    + * This routine must be run outside of the kernel proper since
    + * the kernel will be encrypted during the process. So this
    + * routine is defined here and then copied to an area outside
    + * of the kernel where it will remain and run un-encrypted
    + * during execution.
    + *
    + * On entry the registers must be:
    + * - RAX points to this routine
    + * - RBX points to new PGD to use
    + * - RCX contains the kernel length
    + * - RSI points to the non-encrypted kernel space
    + * - RDI points to the encrypted kernel space
    + *
    + * The kernel will be encrypted by copying from the non-encrypted
    + * kernel space to a temporary buffer and then copying from the
    + * temporary buffer back to the encrypted kernel space. The physical
    + * addresses of the two kernel space mappings are the same which
    + * results in the kernel being encrypted "in place".
    + */
    +.Lencrypt_start:
    + /* Enable the new page tables */
    + mov %rbx, %cr3
    +
    + /* Flush any global TLBs */
    + mov %cr4, %rbx
    + andq $~X86_CR4_PGE, %rbx
    + mov %rbx, %cr4
    + orq $X86_CR4_PGE, %rbx
    + mov %rbx, %cr4
    +
    + /* Set the PAT register PA5 entry to write-protect */
    + push %rax
    + push %rcx
    + movl $MSR_IA32_CR_PAT, %ecx
    + rdmsr
    + push %rdx /* Save original PAT value */
    + andl $0xffff00ff, %edx /* Clear PA5 */
    + orl $0x00000500, %edx /* Set PA5 to WP */
    + wrmsr
    + pop %rdx /* RDX contains original PAT value */
    + pop %rcx
    + pop %rax
    +
    + movq %rsi, %r10 /* Save source address */
    + movq %rdi, %r11 /* Save destination address */
    + movq %rcx, %r12 /* Save length */
    + addq $PAGE_SIZE, %rax /* RAX now points to temp copy page */
    +
    + wbinvd /* Invalidate any cache entries */
    +
    + /* Copy/encrypt 2MB at a time */
    +1:
    + movq %r10, %rsi
    + movq %rax, %rdi
    + movq $PMD_PAGE_SIZE, %rcx
    + rep movsb
    +
    + movq %rax, %rsi
    + movq %r11, %rdi
    + movq $PMD_PAGE_SIZE, %rcx
    + rep movsb
    +
    + addq $PMD_PAGE_SIZE, %r10
    + addq $PMD_PAGE_SIZE, %r11
    + subq $PMD_PAGE_SIZE, %r12
    + jnz 1b
    +
    + /* Restore PAT register */
    + push %rdx
    + movl $MSR_IA32_CR_PAT, %ecx
    + rdmsr
    + pop %rdx
    + wrmsr
    +
    + ret
    +.Lencrypt_stop:
    +#endif /* CONFIG_AMD_MEM_ENCRYPT */
    +
    .data
    .align 16
    ENTRY(sme_me_mask)
    @@ -35,3 +333,7 @@ ENTRY(sme_me_mask)
    sme_me_loss:
    .byte 0x00
    .align 8
    +
    +mem_encrypt_enable_option:
    + .asciz "mem_encrypt=on"
    + .align 8
    diff --git a/arch/x86/mm/mem_encrypt.c b/arch/x86/mm/mem_encrypt.c
    index 2f28d87..1154353 100644
    --- a/arch/x86/mm/mem_encrypt.c
    +++ b/arch/x86/mm/mem_encrypt.c
    @@ -183,6 +183,8 @@ void __init mem_encrypt_init(void)

    /* Make SWIOTLB use an unencrypted DMA area */
    swiotlb_clear_encryption();
    +
    + pr_info("memory encryption active\n");
    }

    unsigned long amd_iommu_get_me_mask(void)
    \
     
     \ /
      Last update: 2016-09-17 09:57    [W:2.735 / U:0.000 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site