samsung_sm8750/android_kernel_samsung_sm8750
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

x86: fold pda into percpu area on SMP

Browse Source 
[ Based on original patch from Christoph Lameter and Mike Travis. ]

Currently pdas and percpu areas are allocated separately.  %gs points
to local pda and percpu area can be reached using pda->data_offset.
This patch folds pda into percpu area.

Due to strange gcc requirement, pda needs to be at the beginning of
the percpu area so that pda->stack_canary is at %gs:40.  To achieve
this, a new percpu output section macro - PERCPU_VADDR_PREALLOC() - is
added and used to reserve pda sized chunk at the start of the percpu
area.

After this change, for boot cpu, %gs first points to pda in the
data.init area and later during setup_per_cpu_areas() gets updated to
point to the actual pda.  This means that setup_per_cpu_areas() need
to reload %gs for CPU0 while clearing pda area for other cpus as cpu0
already has modified it when control reaches setup_per_cpu_areas().

This patch also removes now unnecessary get_local_pda() and its call
sites.

A lot of this patch is taken from Mike Travis' "x86_64: Fold pda into
per cpu area" patch.

Signed-off-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
This commit is contained in:
Tejun Heo
2009-01-13 20:41:35 +09:00
committed by Ingo Molnar
parent c8f3329a0d
commit 1a51e3a0ae
11 changed files with 104 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

107
arch/x86/kernel/setup_percpu.c
View File

@@ -13,6 +13,7 @@
#include <asm/mpspec.h>
#include <asm/apicdef.h>
#include <asm/highmem.h>
#include <asm/proto.h>
#include <asm/cpumask.h>
#ifdef CONFIG_DEBUG_PER_CPU_MAPS
@@ -65,6 +66,36 @@ static void __init setup_node_to_cpumask_map(void);
static inline void setup_node_to_cpumask_map(void) { }
#endif
#ifdef CONFIG_X86_64
void __cpuinit load_pda_offset(int cpu)
{
/* Memory clobbers used to order pda/percpu accesses */
mb();
wrmsrl(MSR_GS_BASE, cpu_pda(cpu));
mb();
}
#endif /* CONFIG_SMP && CONFIG_X86_64 */
#ifdef CONFIG_X86_64
/* correctly size the local cpu masks */
static void setup_cpu_local_masks(void)
{
alloc_bootmem_cpumask_var(&cpu_initialized_mask);
alloc_bootmem_cpumask_var(&cpu_callin_mask);
alloc_bootmem_cpumask_var(&cpu_callout_mask);
alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask);
}
#else /* CONFIG_X86_32 */
static inline void setup_cpu_local_masks(void)
{
}
#endif /* CONFIG_X86_32 */
#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
/*
* Copy data used in early init routines from the initial arrays to the
@@ -101,63 +132,7 @@ static void __init setup_per_cpu_maps(void)
*/
unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
EXPORT_SYMBOL(__per_cpu_offset);
static inline void setup_cpu_pda_map(void) { }
#elif !defined(CONFIG_SMP)
static inline void setup_cpu_pda_map(void) { }
#else /* CONFIG_SMP && CONFIG_X86_64 */
/*
* Allocate cpu_pda pointer table and array via alloc_bootmem.
*/
static void __init setup_cpu_pda_map(void)
{
char *pda;
unsigned long size;
int cpu;
size = roundup(sizeof(struct x8664_pda), cache_line_size());
/* allocate cpu_pda array and pointer table */
{
unsigned long asize = size * (nr_cpu_ids - 1);
pda = alloc_bootmem(asize);
}
/* initialize pointer table to static pda's */
for_each_possible_cpu(cpu) {
if (cpu == 0) {
/* leave boot cpu pda in place */
continue;
}
cpu_pda(cpu) = (struct x8664_pda *)pda;
cpu_pda(cpu)->in_bootmem = 1;
pda += size;
}
}
#endif /* CONFIG_SMP && CONFIG_X86_64 */
#ifdef CONFIG_X86_64
/* correctly size the local cpu masks */
static void setup_cpu_local_masks(void)
{
alloc_bootmem_cpumask_var(&cpu_initialized_mask);
alloc_bootmem_cpumask_var(&cpu_callin_mask);
alloc_bootmem_cpumask_var(&cpu_callout_mask);
alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask);
}
#else /* CONFIG_X86_32 */
static inline void setup_cpu_local_masks(void)
{
}
#endif /* CONFIG_X86_32 */
#endif
/*
* Great future plan:
@@ -171,9 +146,6 @@ void __init setup_per_cpu_areas(void)
int cpu;
unsigned long align = 1;
/* Setup cpu_pda map */
setup_cpu_pda_map();
/* Copy section for each CPU (we discard the original) */
old_size = PERCPU_ENOUGH_ROOM;
align = max_t(unsigned long, PAGE_SIZE, align);
@@ -204,8 +176,21 @@ void __init setup_per_cpu_areas(void)
cpu, node, __pa(ptr));
}
#endif
per_cpu_offset(cpu) = ptr - __per_cpu_start;
memcpy(ptr, __per_cpu_load, __per_cpu_end - __per_cpu_start);
#ifdef CONFIG_X86_64
cpu_pda(cpu) = (void *)ptr;
/*
* CPU0 modified pda in the init data area, reload pda
* offset for CPU0 and clear the area for others.
*/
if (cpu == 0)
load_pda_offset(0);
else
memset(cpu_pda(cpu), 0, sizeof(*cpu_pda(cpu)));
#endif
per_cpu_offset(cpu) = ptr - __per_cpu_start;
DBG("PERCPU: cpu %4d %p\n", cpu, ptr);
}