lkml.org 
[lkml]   [2020]   [Mar]   [19]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[PATCH 70/70] x86/sev-es: Add NMI state tracking
    Date
    From: Joerg Roedel <jroedel@suse.de>

    Keep NMI state in SEV-ES code so the kernel can re-enable NMIs for the
    vCPU when it reaches IRET.

    Signed-off-by: Joerg Roedel <jroedel@suse.de>
    ---
    arch/x86/entry/entry_64.S | 48 +++++++++++++++++++++++++++++++++
    arch/x86/include/asm/sev-es.h | 27 +++++++++++++++++++
    arch/x86/include/uapi/asm/svm.h | 1 +
    arch/x86/kernel/nmi.c | 8 ++++++
    arch/x86/kernel/sev-es.c | 31 ++++++++++++++++++++-
    5 files changed, 114 insertions(+), 1 deletion(-)

    diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
    index 729876d368c5..355470b36896 100644
    --- a/arch/x86/entry/entry_64.S
    +++ b/arch/x86/entry/entry_64.S
    @@ -38,6 +38,7 @@
    #include <asm/export.h>
    #include <asm/frame.h>
    #include <asm/nospec-branch.h>
    +#include <asm/sev-es.h>
    #include <linux/err.h>

    #include "calling.h"
    @@ -629,6 +630,13 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)
    ud2
    1:
    #endif
    +
    + /*
    + * This code path is used by the NMI handler, so check if NMIs
    + * need to be re-enabled when running as an SEV-ES guest.
    + */
    + SEV_ES_IRET_CHECK
    +
    POP_REGS pop_rdi=0

    /*
    @@ -1474,6 +1482,8 @@ SYM_CODE_START(nmi)
    movq $-1, %rsi
    call do_nmi

    + SEV_ES_NMI_COMPLETE
    +
    /*
    * Return back to user mode. We must *not* do the normal exit
    * work, because we don't want to enable interrupts.
    @@ -1599,6 +1609,7 @@ nested_nmi_out:
    popq %rdx

    /* We are returning to kernel mode, so this cannot result in a fault. */
    + SEV_ES_NMI_COMPLETE
    iretq

    first_nmi:
    @@ -1687,6 +1698,12 @@ end_repeat_nmi:
    movq $-1, %rsi
    call do_nmi

    + /*
    + * When running as an SEV-ES guest, jump to the SEV-ES NMI IRET
    + * path.
    + */
    + SEV_ES_NMI_COMPLETE
    +
    /* Always restore stashed CR3 value (see paranoid_entry) */
    RESTORE_CR3 scratch_reg=%r15 save_reg=%r14

    @@ -1715,6 +1732,9 @@ nmi_restore:
    std
    movq $0, 5*8(%rsp) /* clear "NMI executing" */

    +nmi_return:
    + UNWIND_HINT_IRET_REGS
    +
    /*
    * iretq reads the "iret" frame and exits the NMI stack in a
    * single instruction. We are returning to kernel mode, so this
    @@ -1724,6 +1744,34 @@ nmi_restore:
    iretq
    SYM_CODE_END(nmi)

    +#ifdef CONFIG_AMD_MEM_ENCRYPT
    +SYM_CODE_START(sev_es_iret_user)
    + UNWIND_HINT_IRET_REGS offset=8
    + /*
    + * The kernel jumps here directly from
    + * swapgs_restore_regs_and_return_to_usermode. %rsp points already to
    + * trampoline stack, but %cr3 is still from kernel. User-regs are live
    + * except %rdi. Switch to user CR3, restore user %rdi and user gs_base
    + * and single-step over IRET
    + */
    + SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
    + popq %rdi
    + SWAPGS
    + /*
    + * Enable single-stepping and execute IRET. When IRET is
    + * finished the resulting #DB exception will cause a #VC
    + * exception to be raised. The #VC exception handler will send a
    + * NMI-complete message to the hypervisor to re-open the NMI
    + * window.
    + */
    +sev_es_iret_kernel:
    + pushf
    + btsq $X86_EFLAGS_TF_BIT, (%rsp)
    + popf
    + iretq
    +SYM_CODE_END(sev_es_iret_user)
    +#endif
    +
    #ifndef CONFIG_IA32_EMULATION
    /*
    * This handles SYSCALL from 32-bit code. There is no way to program
    diff --git a/arch/x86/include/asm/sev-es.h b/arch/x86/include/asm/sev-es.h
    index 63acf50e6280..d866adb3e6d4 100644
    --- a/arch/x86/include/asm/sev-es.h
    +++ b/arch/x86/include/asm/sev-es.h
    @@ -8,6 +8,8 @@
    #ifndef __ASM_ENCRYPTED_STATE_H
    #define __ASM_ENCRYPTED_STATE_H

    +#ifndef __ASSEMBLY__
    +
    #include <linux/types.h>
    #include <asm/insn.h>

    @@ -82,11 +84,36 @@ struct real_mode_header;

    #ifdef CONFIG_AMD_MEM_ENCRYPT
    int sev_es_setup_ap_jump_table(struct real_mode_header *rmh);
    +void sev_es_nmi_enter(void);
    #else /* CONFIG_AMD_MEM_ENCRYPT */
    static inline int sev_es_setup_ap_jump_table(struct real_mode_header *rmh)
    {
    return 0;
    }
    +static inline void sev_es_nmi_enter(void) { }
    +#endif /* CONFIG_AMD_MEM_ENCRYPT*/
    +
    +#else /* !__ASSEMBLY__ */
    +
    +#ifdef CONFIG_AMD_MEM_ENCRYPT
    +#define SEV_ES_NMI_COMPLETE \
    + ALTERNATIVE "", "callq sev_es_nmi_complete", X86_FEATURE_SEV_ES_GUEST
    +
    +.macro SEV_ES_IRET_CHECK
    + ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_SEV_ES_GUEST
    + movq PER_CPU_VAR(sev_es_in_nmi), %rdi
    + testq %rdi, %rdi
    + jz .Lend_\@
    + callq sev_es_nmi_complete
    +.Lend_\@:
    +.endm
    +
    +#else /* CONFIG_AMD_MEM_ENCRYPT */
    +#define SEV_ES_NMI_COMPLETE
    +.macro SEV_ES_IRET_CHECK
    +.endm
    #endif /* CONFIG_AMD_MEM_ENCRYPT*/

    +#endif /* __ASSEMBLY__ */
    +
    #endif
    diff --git a/arch/x86/include/uapi/asm/svm.h b/arch/x86/include/uapi/asm/svm.h
    index 20a05839dd9a..0f837339db66 100644
    --- a/arch/x86/include/uapi/asm/svm.h
    +++ b/arch/x86/include/uapi/asm/svm.h
    @@ -84,6 +84,7 @@
    /* SEV-ES software-defined VMGEXIT events */
    #define SVM_VMGEXIT_MMIO_READ 0x80000001
    #define SVM_VMGEXIT_MMIO_WRITE 0x80000002
    +#define SVM_VMGEXIT_NMI_COMPLETE 0x80000003
    #define SVM_VMGEXIT_AP_HLT_LOOP 0x80000004
    #define SVM_VMGEXIT_AP_JUMP_TABLE 0x80000005
    #define SVM_VMGEXIT_SET_AP_JUMP_TABLE 0
    diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c
    index 54c21d6abd5a..7312a6d4d50f 100644
    --- a/arch/x86/kernel/nmi.c
    +++ b/arch/x86/kernel/nmi.c
    @@ -37,6 +37,7 @@
    #include <asm/reboot.h>
    #include <asm/cache.h>
    #include <asm/nospec-branch.h>
    +#include <asm/sev-es.h>

    #define CREATE_TRACE_POINTS
    #include <trace/events/nmi.h>
    @@ -510,6 +511,13 @@ NOKPROBE_SYMBOL(is_debug_stack);
    dotraplinkage notrace void
    do_nmi(struct pt_regs *regs, long error_code)
    {
    + /*
    + * For SEV-ES the kernel needs to track whether NMIs are blocked until
    + * IRET is reached, even when the CPU is offline.
    + */
    + if (sev_es_active())
    + sev_es_nmi_enter();
    +
    if (IS_ENABLED(CONFIG_SMP) && cpu_is_offline(smp_processor_id()))
    return;

    diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c
    index 3c22f256645e..409a7a2aa630 100644
    --- a/arch/x86/kernel/sev-es.c
    +++ b/arch/x86/kernel/sev-es.c
    @@ -37,6 +37,7 @@ struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE);
    */
    struct ghcb __initdata *boot_ghcb;
    static DEFINE_PER_CPU(unsigned long, cached_dr7) = DR7_RESET_VALUE;
    +DEFINE_PER_CPU(bool, sev_es_in_nmi) = false;

    struct ghcb_state {
    struct ghcb *ghcb;
    @@ -270,6 +271,31 @@ static phys_addr_t vc_slow_virt_to_phys(struct ghcb *ghcb, long vaddr)
    /* Include code shared with pre-decompression boot stage */
    #include "sev-es-shared.c"

    +void sev_es_nmi_enter(void)
    +{
    + this_cpu_write(sev_es_in_nmi, true);
    +}
    +
    +void sev_es_nmi_complete(void)
    +{
    + struct ghcb_state state;
    + struct ghcb *ghcb;
    +
    + ghcb = sev_es_get_ghcb(&state);
    +
    + vc_ghcb_invalidate(ghcb);
    + ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_NMI_COMPLETE);
    + ghcb_set_sw_exit_info_1(ghcb, 0);
    + ghcb_set_sw_exit_info_2(ghcb, 0);
    +
    + sev_es_wr_ghcb_msr(__pa(ghcb));
    + VMGEXIT();
    +
    + sev_es_put_ghcb(&state);
    +
    + this_cpu_write(sev_es_in_nmi, false);
    +}
    +
    static u64 sev_es_get_jump_table_addr(void)
    {
    struct ghcb_state state;
    @@ -891,7 +917,10 @@ static enum es_result vc_handle_vmmcall(struct ghcb *ghcb,
    static enum es_result vc_handle_db_exception(struct ghcb *ghcb,
    struct es_em_ctxt *ctxt)
    {
    - do_debug(ctxt->regs, 0);
    + if (this_cpu_read(sev_es_in_nmi))
    + sev_es_nmi_complete();
    + else
    + do_debug(ctxt->regs, 0);

    /* Exception event, do not advance RIP */
    return ES_RETRY;
    --
    2.17.1
    \
     
     \ /
      Last update: 2020-03-19 10:16    [W:4.060 / U:0.040 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site