lkml.org 
[lkml]   [2011]   [Dec]   [20]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
Patch in this message
/
Date
SubjectRe: [PATCH -v8] x86: Disable x2apic if nox2apic is specified or intr-remap can not be enabled
From
On Tue, Dec 20, 2011 at 5:49 PM, Yinghai Lu <yinghai@kernel.org> wrote:
> On Mon, Dec 19, 2011 at 10:51 PM, Suresh Siddha
> <suresh.b.siddha@intel.com> wrote:
>> While reviewing the code, I ended up doing some of this cleanup of your
>> patch. I have appended the modified version. Can you please incorporate
>> the smpboot checks before the AP bringup (apic-id should be less than
>> 255 in !x2apic_mode etc) and split the patch into couple of patches
>> (apic_probe etc) along with other needed cleanups.
>
> We could handle AP's apic id > 255 panic problem later if needed in
> another patch.
>
> Actually on system with apic id > 255, intr-remapping aka DMAR must be
> working, otherwise those big box can not be sold.
>
> And in this case, nox2apic must be passed to disable x2apic, because
> dmar is correct in both case kexec or first boot.
>
> when nox2apic is passed,  those entries will be skipped early. and BSP
> x2apic get disabled early.
> and code path is more like bios enable xapic only.
>
> Can we let this patch get into tip and have test cycle at first?
> at least we should not let it break any working set up.

put your updated version as v9.

like to let ingo or hpa or thomas to decide v8 or v9 to be used.

Changes:
1. clear x2apic cpu features
2. disable x2apic later instead of check_x2apic()

Thanks

Yinghai
From: Yinghai Lu <yinghai@kernel.org>
Subject: [PATCH -v9] x86: Disable x2apic if nox2apic is specified or intr-remap can not be enabled

For
1. x2apic preenabled system
2. first kernel have x2apic enabled, and try to kexec second kernel with "nox2apic"

Will put back cpu with apic id < 255 into xapic mode, instead of panic.

-v2: use variable x2apic_disabled instead of variable nox2apic, Suggested by Thomas
update x2apic_supported with x2apic_disabled, Suggested by Thomas

-v3: add checking for boot cpu apic id > 255. in that case will just panic
--- pointed out by Suresh.

-v4: according to Ingo, for x2apic pre-enabled system, if intr-remap can not
be enabled, try to disable x2apic instead of panic and request to
specify nox2apic for next boot.

-v5: fix the lapic mapping for falling path.

-v6: add warning when check_timer fails in strange case when BIOS enable x2apic
and intr_remapping but does not provide dmar table. requested by Suresh.

-v7: fix compiling with x2apic_preenabled

-v8: fix compiling with x2apic_disabled assigned with !X86_X2APIC

-v9: updated version from Suresh
Changes: 1. clear x2apic cpu features
2. disable x2apic later instead of check_x2apic()

Signed-off-by: Yinghai Lu <yinghai@kernel.org>
Tested-by: "Berck E. Nash" <flyboy@gmail.com>

---
arch/x86/include/asm/apic.h | 6 ++
arch/x86/include/asm/apicdef.h | 1
arch/x86/kernel/acpi/boot.c | 10 +++
arch/x86/kernel/apic/apic.c | 92 ++++++++++++++++++++++++++----------
arch/x86/kernel/apic/apic_flat_64.c | 7 ++
arch/x86/kernel/apic/io_apic.c | 4 +
arch/x86/mm/srat.c | 7 ++
7 files changed, 99 insertions(+), 28 deletions(-)

Index: linux-2.6/arch/x86/include/asm/apic.h
===================================================================
--- linux-2.6.orig/arch/x86/include/asm/apic.h
+++ linux-2.6/arch/x86/include/asm/apic.h
@@ -176,6 +176,8 @@ static inline u64 native_x2apic_icr_read
}

extern int x2apic_phys;
+extern int x2apic_preenabled;
+extern int x2apic_disabled;
extern void check_x2apic(void);
extern void enable_x2apic(void);
extern void x2apic_icr_write(u32 low, u32 id);
@@ -198,6 +200,9 @@ static inline void x2apic_force_phys(voi
x2apic_phys = 1;
}
#else
+static inline void disable_x2apic(void)
+{
+}
static inline void check_x2apic(void)
{
}
@@ -214,6 +219,7 @@ static inline void x2apic_force_phys(voi

#define x2apic_preenabled 0
#define x2apic_supported() 0
+#define x2apic_disabled 1
#endif

extern void enable_IR_x2apic(void);
Index: linux-2.6/arch/x86/include/asm/apicdef.h
===================================================================
--- linux-2.6.orig/arch/x86/include/asm/apicdef.h
+++ linux-2.6/arch/x86/include/asm/apicdef.h
@@ -144,6 +144,7 @@

#define APIC_BASE (fix_to_virt(FIX_APIC_BASE))
#define APIC_BASE_MSR 0x800
+#define XAPIC_ENABLE (1UL << 11)
#define X2APIC_ENABLE (1UL << 10)

#ifdef CONFIG_X86_32
Index: linux-2.6/arch/x86/kernel/acpi/boot.c
===================================================================
--- linux-2.6.orig/arch/x86/kernel/acpi/boot.c
+++ linux-2.6/arch/x86/kernel/acpi/boot.c
@@ -219,6 +219,8 @@ static int __init
acpi_parse_x2apic(struct acpi_subtable_header *header, const unsigned long end)
{
struct acpi_madt_local_x2apic *processor = NULL;
+ int apic_id;
+ u8 enabled;

processor = (struct acpi_madt_local_x2apic *)header;

@@ -227,6 +229,8 @@ acpi_parse_x2apic(struct acpi_subtable_h

acpi_table_print_madt_entry(header);

+ apic_id = processor->local_apic_id;
+ enabled = processor->lapic_flags & ACPI_MADT_ENABLED;
#ifdef CONFIG_X86_X2APIC
/*
* We need to register disabled CPU as well to permit
@@ -235,8 +239,10 @@ acpi_parse_x2apic(struct acpi_subtable_h
* to not preallocating memory for all NR_CPUS
* when we use CPU hotplug.
*/
- acpi_register_lapic(processor->local_apic_id, /* APIC ID */
- processor->lapic_flags & ACPI_MADT_ENABLED);
+ if (x2apic_disabled && (apic_id >= 0xff) && enabled)
+ printk(KERN_WARNING PREFIX "x2apic entry ignored\n");
+ else
+ acpi_register_lapic(apic_id, enabled);
#else
printk(KERN_WARNING PREFIX "x2apic entry ignored\n");
#endif
Index: linux-2.6/arch/x86/kernel/apic/apic.c
===================================================================
--- linux-2.6.orig/arch/x86/kernel/apic/apic.c
+++ linux-2.6/arch/x86/kernel/apic/apic.c
@@ -146,16 +146,16 @@ __setup("apicpmtimer", setup_apicpmtimer
int x2apic_mode;
#ifdef CONFIG_X86_X2APIC
/* x2apic enabled before OS handover */
-static int x2apic_preenabled;
+int x2apic_preenabled;
+int x2apic_disabled;
static __init int setup_nox2apic(char *str)
{
- if (x2apic_enabled()) {
- pr_warning("Bios already enabled x2apic, "
- "can't enforce nox2apic");
- return 0;
- }
+ if (x2apic_enabled())
+ pr_warning("Bios already enabled x2apic, will disable it");
+ else
+ setup_clear_cpu_cap(X86_FEATURE_X2APIC);
+ x2apic_disabled = 1;

- setup_clear_cpu_cap(X86_FEATURE_X2APIC);
return 0;
}
early_param("nox2apic", setup_nox2apic);
@@ -1432,6 +1432,45 @@ void __init bsp_end_local_APIC_setup(voi
}

#ifdef CONFIG_X86_X2APIC
+
+static inline void __disable_x2apic(u64 msr)
+{
+ wrmsrl(MSR_IA32_APICBASE,
+ msr & ~(X2APIC_ENABLE | XAPIC_ENABLE));
+ wrmsrl(MSR_IA32_APICBASE, msr & ~X2APIC_ENABLE);
+}
+
+static __init void disable_x2apic(void)
+{
+ u64 msr;
+
+ if (!cpu_has_x2apic)
+ return;
+
+ rdmsrl(MSR_IA32_APICBASE, msr);
+ if (msr & X2APIC_ENABLE) {
+ u32 x2apic_id = read_apic_id();
+
+ if (x2apic_id > 255)
+ panic("Can not disable x2apic, id: %08x\n", x2apic_id);
+
+ pr_info("Disabling x2apic\n");
+ /*
+ * Need to disable xapic and x2apic at the same time at first
+ * then enable xapic
+ */
+ __disable_x2apic(msr);
+
+ if (x2apic_disabled)
+ setup_clear_cpu_cap(X86_FEATURE_X2APIC);
+
+ x2apic_disabled = 1;
+ x2apic_mode = 0;
+
+ register_lapic_address(mp_lapic_addr);
+ }
+}
+
void check_x2apic(void)
{
if (x2apic_enabled()) {
@@ -1442,15 +1481,20 @@ void check_x2apic(void)

void enable_x2apic(void)
{
- int msr, msr2;
+ u64 msr;
+
+ rdmsrl(MSR_IA32_APICBASE, msr);
+ if (x2apic_disabled) {
+ __disable_x2apic(msr);
+ return;
+ }

if (!x2apic_mode)
return;

- rdmsr(MSR_IA32_APICBASE, msr, msr2);
if (!(msr & X2APIC_ENABLE)) {
printk_once(KERN_INFO "Enabling x2apic\n");
- wrmsr(MSR_IA32_APICBASE, msr | X2APIC_ENABLE, msr2);
+ wrmsrl(MSR_IA32_APICBASE, msr | X2APIC_ENABLE);
}
}
#endif /* CONFIG_X86_X2APIC */
@@ -1494,25 +1538,34 @@ void __init enable_IR_x2apic(void)
ret = save_ioapic_entries();
if (ret) {
pr_info("Saving IO-APIC state failed: %d\n", ret);
- goto out;
+ return;
}

local_irq_save(flags);
legacy_pic->mask_all();
mask_ioapic_entries();

+ if (x2apic_disabled)
+ disable_x2apic();
+
if (dmar_table_init_ret)
ret = -1;
else
ret = enable_IR();

+ if (!x2apic_supported())
+ goto nox2apic;
+
if (ret < 0) {
/* IR is required if there is APIC ID > 255 even when running
* under KVM
*/
if (max_physical_apicid > 255 ||
- !hypervisor_x2apic_available())
+ !hypervisor_x2apic_available()) {
+ if (x2apic_preenabled && !x2apic_disabled)
+ disable_x2apic();
goto nox2apic;
+ }
/*
* without IR all CPUs can be addressed by IOAPIC/MSI
* only in physical mode
@@ -1520,8 +1573,10 @@ void __init enable_IR_x2apic(void)
x2apic_force_phys();
}

- if (ret == IRQ_REMAP_XAPIC_MODE)
+ if (ret == IRQ_REMAP_XAPIC_MODE) {
+ pr_info("x2apic not enabled, IRQ remapping is in xapic mode\n");
goto nox2apic;
+ }

x2apic_enabled = 1;

@@ -1536,17 +1591,6 @@ nox2apic:
restore_ioapic_entries();
legacy_pic->restore_mask();
local_irq_restore(flags);
-
-out:
- if (x2apic_enabled || !x2apic_supported())
- return;
-
- if (x2apic_preenabled)
- panic("x2apic: enabled by BIOS but kernel init failed.");
- else if (ret == IRQ_REMAP_XAPIC_MODE)
- pr_info("x2apic not enabled, IRQ remapping is in xapic mode\n");
- else if (ret < 0)
- pr_info("x2apic not enabled, IRQ remapping init failed\n");
}

#ifdef CONFIG_X86_64
Index: linux-2.6/arch/x86/kernel/apic/apic_flat_64.c
===================================================================
--- linux-2.6.orig/arch/x86/kernel/apic/apic_flat_64.c
+++ linux-2.6/arch/x86/kernel/apic/apic_flat_64.c
@@ -171,9 +171,14 @@ static int flat_phys_pkg_id(int initial_
return initial_apic_id >> index_msb;
}

+static int flat_probe(void)
+{
+ return 1;
+}
+
static struct apic apic_flat = {
.name = "flat",
- .probe = NULL,
+ .probe = flat_probe,
.acpi_madt_oem_check = flat_acpi_madt_oem_check,
.apic_id_registered = flat_apic_id_registered,

Index: linux-2.6/arch/x86/kernel/apic/io_apic.c
===================================================================
--- linux-2.6.orig/arch/x86/kernel/apic/io_apic.c
+++ linux-2.6/arch/x86/kernel/apic/io_apic.c
@@ -2947,6 +2947,10 @@ static inline void __init check_timer(vo
}
local_irq_disable();
apic_printk(APIC_QUIET, KERN_INFO "..... failed :(.\n");
+ if (x2apic_preenabled && !x2apic_mode)
+ pr_info("Your system has x2apic pre-enabled, but kernel can not enable intr-remapping.\n"
+ "So kernel try to disable x2apic, but interrupt still have problem.\n"
+ "You could disable x2apic from BIOS setup\n");
panic("IO-APIC + timer doesn't work! Boot with apic=debug and send a "
"report. Then try booting with the 'noapic' option.\n");
out:
Index: linux-2.6/arch/x86/mm/srat.c
===================================================================
--- linux-2.6.orig/arch/x86/mm/srat.c
+++ linux-2.6/arch/x86/mm/srat.c
@@ -69,6 +69,12 @@ acpi_numa_x2apic_affinity_init(struct ac
if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
return;
pxm = pa->proximity_domain;
+ apic_id = pa->apic_id;
+ if (x2apic_disabled && (apic_id >= 0xff)) {
+ printk(KERN_INFO "SRAT: PXM %u -> X2APIC 0x%04x ignored\n",
+ pxm, apic_id);
+ return;
+ }
node = setup_node(pxm);
if (node < 0) {
printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
@@ -76,7 +82,6 @@ acpi_numa_x2apic_affinity_init(struct ac
return;
}

- apic_id = pa->apic_id;
if (apic_id >= MAX_LOCAL_APIC) {
printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%04x -> Node %u skipped apicid that is too big\n", pxm, apic_id, node);
return;
\
 
 \ /
  Last update: 2011-12-21 03:27    [W:0.107 / U:0.032 seconds]
©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site