@@ -109,6 +109,34 @@
DECLARE_PER_CPU(int, next_tlbcam_idx);
#endif
+#ifdef CONFIG_JUMP_LABEL
+#include <linux/jump_label.h>
+
+#define MAX_MMU_FEATURES (8 * sizeof(((struct cpu_spec *)0)->mmu_features))
+
+extern struct static_key_true mmu_feat_keys[MAX_MMU_FEATURES];
+
+extern void mmu_feat_keys_init(void);
+
+static inline int mmu_has_feature(unsigned long feature)
+{
+ int i;
+
+ i = __builtin_ctzl(feature);
+ return static_branch_likely(&mmu_feat_keys[i]);
+}
+
+static inline void mmu_clear_feature(unsigned long feature)
+{
+ int i;
+
+ i = __builtin_ctzl(feature);
+ cur_cpu_spec->mmu_features &= ~feature;
+ static_branch_disable(&mmu_feat_keys[i]);
+}
+#else
+static inline void mmu_feat_keys_init(void) { }
+
static inline int mmu_has_feature(unsigned long feature)
{
return (cur_cpu_spec->mmu_features & feature);
@@ -118,6 +146,7 @@ static inline void mmu_clear_feature(unsigned long feature)
{
cur_cpu_spec->mmu_features &= ~feature;
}
+#endif
extern unsigned int __start___mmu_ftr_fixup, __stop___mmu_ftr_fixup;
@@ -2214,4 +2214,21 @@ void __init cpu_feat_keys_init(void)
static_branch_disable(&cpu_feat_keys[i]);
}
}
+
+struct static_key_true mmu_feat_keys[MAX_MMU_FEATURES] = {
+ [0 ... MAX_MMU_FEATURES - 1] = STATIC_KEY_TRUE_INIT
+};
+EXPORT_SYMBOL_GPL(mmu_feat_keys);
+
+void __init mmu_feat_keys_init(void)
+{
+ int i;
+
+ for (i = 0; i < MAX_MMU_FEATURES; i++) {
+ unsigned long f = 1ul << i;
+
+ if (!(cur_cpu_spec->mmu_features & f))
+ static_branch_disable(&mmu_feat_keys[i]);
+ }
+}
#endif
@@ -116,6 +116,7 @@ notrace void __init machine_init(u64 dt_ptr)
jump_label_init();
cpu_feat_keys_init();
+ mmu_feat_keys_init();
/* Enable early debugging if any specified (see udbg.h) */
udbg_early_init();
@@ -252,6 +252,7 @@ void __init early_setup(unsigned long dt_ptr)
jump_label_init();
cpu_feat_keys_init();
+ mmu_feat_keys_init();
/* -------- printk is now safe to use ------- */
The mmu features are fixed once the probe of mmu features are done. And the function mmu_has_feature() does be used in some hot path. The checking of the mmu features for each time of invoking of mmu_has_feature() seems suboptimal. This tries to reduce this overhead of this check by using jump label. The generated assemble code of the following c program: if (mmu_has_feature(MMU_FTR_XXX)) xxx() Before: lis r9,-16230 lwz r9,12324(r9) lwz r9,24(r9) andi. r10,r9,16 beqlr+ After: nop if MMU_FTR_XXX is enabled b xxx if MMU_FTR_XXX is not enabled Signed-off-by: Kevin Hao <haokexin@gmail.com> --- v2: Use the open-coded definition and initialization for mmu_feat_keys[]. arch/powerpc/include/asm/mmu.h | 29 +++++++++++++++++++++++++++++ arch/powerpc/kernel/cputable.c | 17 +++++++++++++++++ arch/powerpc/kernel/setup_32.c | 1 + arch/powerpc/kernel/setup_64.c | 1 + 4 files changed, 48 insertions(+)