lkml.org 
[lkml]   [2023]   [Feb]   [26]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[PATCH v12 06/11] x86/smpboot: Remove initial_stack on 64-bit
    Date
    From: Brian Gerst <brgerst@gmail.com>

    Eliminating global variables from the CPU startup path in order to simplify
    it and facilitate parallel startup.

    Remove initial_stack, and load RSP from current_task->thread.sp instead.

    Signed-off-by: Brian Gerst <brgerst@gmail.com>
    Reviewed-by: David Woodhouse <dwmw@amazon.co.uk>
    Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
    Tested-by: Usama Arif <usama.arif@bytedance.com>
    Signed-off-by: Usama Arif <usama.arif@bytedance.com>
    ---
    arch/x86/include/asm/processor.h | 6 ++++-
    arch/x86/include/asm/smp.h | 5 +++-
    arch/x86/kernel/acpi/sleep.c | 5 ++--
    arch/x86/kernel/asm-offsets.c | 1 +
    arch/x86/kernel/head_64.S | 43 +++++++++++++++++++++-----------
    arch/x86/kernel/smpboot.c | 7 +++++-
    arch/x86/xen/xen-head.S | 2 +-
    7 files changed, 48 insertions(+), 21 deletions(-)

    diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
    index 4e35c66edeb7..bdde7316e75b 100644
    --- a/arch/x86/include/asm/processor.h
    +++ b/arch/x86/include/asm/processor.h
    @@ -648,7 +648,11 @@ static inline void spin_lock_prefetch(const void *x)
    #define KSTK_ESP(task) (task_pt_regs(task)->sp)

    #else
    -#define INIT_THREAD { }
    +extern unsigned long __end_init_task[];
    +
    +#define INIT_THREAD { \
    + .sp = (unsigned long)&__end_init_task - sizeof(struct pt_regs), \
    +}

    extern unsigned long KSTK_ESP(struct task_struct *task);

    diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h
    index b4dbb20dab1a..bf2c51df9e0b 100644
    --- a/arch/x86/include/asm/smp.h
    +++ b/arch/x86/include/asm/smp.h
    @@ -199,5 +199,8 @@ extern void nmi_selftest(void);
    #define nmi_selftest() do { } while (0)
    #endif

    -#endif /* __ASSEMBLY__ */
    +extern unsigned int smpboot_control;
    +
    +#endif /* !__ASSEMBLY__ */
    +
    #endif /* _ASM_X86_SMP_H */
    diff --git a/arch/x86/kernel/acpi/sleep.c b/arch/x86/kernel/acpi/sleep.c
    index 3b7f4cdbf2e0..ab6e29b32c04 100644
    --- a/arch/x86/kernel/acpi/sleep.c
    +++ b/arch/x86/kernel/acpi/sleep.c
    @@ -111,13 +111,14 @@ int x86_acpi_suspend_lowlevel(void)
    saved_magic = 0x12345678;
    #else /* CONFIG_64BIT */
    #ifdef CONFIG_SMP
    - initial_stack = (unsigned long)temp_stack + sizeof(temp_stack);
    + current->thread.sp = (unsigned long)temp_stack + sizeof(temp_stack);
    early_gdt_descr.address =
    (unsigned long)get_cpu_gdt_rw(smp_processor_id());
    initial_gs = per_cpu_offset(smp_processor_id());
    + smpboot_control = smp_processor_id();
    #endif
    initial_code = (unsigned long)wakeup_long64;
    - saved_magic = 0x123456789abcdef0L;
    + saved_magic = 0x123456789abcdef0L;
    #endif /* CONFIG_64BIT */

    /*
    diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c
    index 82c783da16a8..797ae1a15c91 100644
    --- a/arch/x86/kernel/asm-offsets.c
    +++ b/arch/x86/kernel/asm-offsets.c
    @@ -108,6 +108,7 @@ static void __used common(void)
    OFFSET(TSS_sp1, tss_struct, x86_tss.sp1);
    OFFSET(TSS_sp2, tss_struct, x86_tss.sp2);
    OFFSET(X86_top_of_stack, pcpu_hot, top_of_stack);
    + OFFSET(X86_current_task, pcpu_hot, current_task);
    #ifdef CONFIG_CALL_DEPTH_TRACKING
    OFFSET(X86_call_depth, pcpu_hot, call_depth);
    #endif
    diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
    index 222efd4a09bc..5a2417d788d1 100644
    --- a/arch/x86/kernel/head_64.S
    +++ b/arch/x86/kernel/head_64.S
    @@ -61,8 +61,8 @@ SYM_CODE_START_NOALIGN(startup_64)
    * tables and then reload them.
    */

    - /* Set up the stack for verify_cpu(), similar to initial_stack below */
    - leaq (__end_init_task - FRAME_SIZE)(%rip), %rsp
    + /* Set up the stack for verify_cpu() */
    + leaq (__end_init_task - PTREGS_SIZE)(%rip), %rsp

    leaq _text(%rip), %rdi

    @@ -241,6 +241,24 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
    UNWIND_HINT_EMPTY
    ANNOTATE_NOENDBR // above

    +#ifdef CONFIG_SMP
    + movl smpboot_control(%rip), %ecx
    +
    + /* Get the per cpu offset for the given CPU# which is in ECX */
    + movq __per_cpu_offset(,%rcx,8), %rdx
    +#else
    + xorl %edx, %edx
    +#endif /* CONFIG_SMP */
    +
    + /*
    + * Setup a boot time stack - Any secondary CPU will have lost its stack
    + * by now because the cr3-switch above unmaps the real-mode stack.
    + *
    + * RDX contains the per-cpu offset
    + */
    + movq pcpu_hot + X86_current_task(%rdx), %rax
    + movq TASK_threadsp(%rax), %rsp
    +
    /*
    * We must switch to a new descriptor in kernel space for the GDT
    * because soon the kernel won't have access anymore to the userspace
    @@ -275,12 +293,6 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
    movl initial_gs+4(%rip),%edx
    wrmsr

    - /*
    - * Setup a boot time stack - Any secondary CPU will have lost its stack
    - * by now because the cr3-switch above unmaps the real-mode stack
    - */
    - movq initial_stack(%rip), %rsp
    -
    /* Setup and Load IDT */
    pushq %rsi
    call early_setup_idt
    @@ -372,7 +384,11 @@ SYM_CODE_END(secondary_startup_64)
    SYM_CODE_START(start_cpu0)
    ANNOTATE_NOENDBR
    UNWIND_HINT_EMPTY
    - movq initial_stack(%rip), %rsp
    +
    + /* Find the idle task stack */
    + movq PER_CPU_VAR(pcpu_hot) + X86_current_task, %rcx
    + movq TASK_threadsp(%rcx), %rsp
    +
    jmp .Ljump_to_C_code
    SYM_CODE_END(start_cpu0)
    #endif
    @@ -420,12 +436,6 @@ SYM_DATA(initial_gs, .quad INIT_PER_CPU_VAR(fixed_percpu_data))
    #ifdef CONFIG_AMD_MEM_ENCRYPT
    SYM_DATA(initial_vc_handler, .quad handle_vc_boot_ghcb)
    #endif
    -
    -/*
    - * The FRAME_SIZE gap is a convention which helps the in-kernel unwinder
    - * reliably detect the end of the stack.
    - */
    -SYM_DATA(initial_stack, .quad init_thread_union + THREAD_SIZE - FRAME_SIZE)
    __FINITDATA

    __INIT
    @@ -660,6 +670,9 @@ SYM_DATA_END(level1_fixmap_pgt)
    SYM_DATA(early_gdt_descr, .word GDT_ENTRIES*8-1)
    SYM_DATA_LOCAL(early_gdt_descr_base, .quad INIT_PER_CPU_VAR(gdt_page))

    + .align 16
    +SYM_DATA(smpboot_control, .long 0)
    +
    .align 16
    /* This must match the first entry in level2_kernel_pgt */
    SYM_DATA(phys_base, .quad 0x0)
    diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
    index b18c1385e181..62e3bf37f0b8 100644
    --- a/arch/x86/kernel/smpboot.c
    +++ b/arch/x86/kernel/smpboot.c
    @@ -1112,7 +1112,12 @@ static int do_boot_cpu(int apicid, int cpu, struct task_struct *idle,
    idle->thread.sp = (unsigned long)task_pt_regs(idle);
    early_gdt_descr.address = (unsigned long)get_cpu_gdt_rw(cpu);
    initial_code = (unsigned long)start_secondary;
    - initial_stack = idle->thread.sp;
    +
    + if (IS_ENABLED(CONFIG_X86_32)) {
    + initial_stack = idle->thread.sp;
    + } else {
    + smpboot_control = cpu;
    + }

    /* Enable the espfix hack for this CPU */
    init_espfix_ap(cpu);
    diff --git a/arch/x86/xen/xen-head.S b/arch/x86/xen/xen-head.S
    index ffaa62167f6e..6bd391476656 100644
    --- a/arch/x86/xen/xen-head.S
    +++ b/arch/x86/xen/xen-head.S
    @@ -49,7 +49,7 @@ SYM_CODE_START(startup_xen)
    ANNOTATE_NOENDBR
    cld

    - mov initial_stack(%rip), %rsp
    + leaq (__end_init_task - PTREGS_SIZE)(%rip), %rsp

    /* Set up %gs.
    *
    --
    2.25.1
    \
     
     \ /
      Last update: 2023-03-27 00:35    [W:7.125 / U:0.020 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site