| Message ID | 20241018173632.277333-14-hbathini@linux.ibm.com (mailing list archive) |
|---|---|
| State | New |
| Headers | show |
| Series | powerpc: Core ftrace rework, support for ftrace direct and bpf trampolines | expand |
On Sat, Oct 19, 2024 at 2:38 AM Hari Bathini <hbathini@linux.ibm.com> wrote: > > From: Naveen N Rao <naveen@kernel.org> > > We are restricted to a .text size of ~32MB when using out-of-line > function profile sequence. Allow this to be extended up to the previous > limit of ~64MB by reserving space in the middle of .text. > > A new config option CONFIG_PPC_FTRACE_OUT_OF_LINE_NUM_RESERVE is > introduced to specify the number of function stubs that are reserved in > .text. On boot, ftrace utilizes stubs from this area first before using > the stub area at the end of .text. > > A ppc64le defconfig has ~44k functions that can be traced. A more > conservative value of 32k functions is chosen as the default value of > PPC_FTRACE_OUT_OF_LINE_NUM_RESERVE so that we do not allot more space > than necessary by default. If building a kernel that only has 32k > trace-able functions, we won't allot any more space at the end of .text > during the pass on vmlinux.o. Otherwise, only the remaining functions > get space for stubs at the end of .text. This default value should help > cover a .text size of ~48MB in total (including space reserved at the > end of .text which can cover up to 32MB), which should be sufficient for > most common builds. For a very small kernel build, this can be set to 0. > Or, this can be bumped up to a larger value to support vmlinux .text > size up to ~64MB. > > Signed-off-by: Naveen N Rao <naveen@kernel.org> > Signed-off-by: Hari Bathini <hbathini@linux.ibm.com> > --- > > Changes in v6: > * Updated with Masahiro's suggestions at > https://lore.kernel.org/all/CAK7LNAREkj5OQ_HA2H=iV_32qdOcaguCOBKV1j+dJW0YaQh3UA@mail.gmail.com/ > > > arch/powerpc/Kconfig | 12 ++++++++++++ > arch/powerpc/include/asm/ftrace.h | 6 ++++-- > arch/powerpc/kernel/trace/ftrace.c | 21 +++++++++++++++++---- > arch/powerpc/kernel/trace/ftrace_entry.S | 8 ++++++++ > arch/powerpc/tools/Makefile | 3 ++- > arch/powerpc/tools/ftrace-gen-ool-stubs.sh | 19 +++++++++++++------ > 6 files changed, 56 insertions(+), 13 deletions(-) > > diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig > index 26e3060e44f4..2e347f682c15 100644 > --- a/arch/powerpc/Kconfig > +++ b/arch/powerpc/Kconfig > @@ -573,6 +573,18 @@ config PPC_FTRACE_OUT_OF_LINE > def_bool PPC64 && ARCH_USING_PATCHABLE_FUNCTION_ENTRY > select ARCH_WANTS_PRE_LINK_VMLINUX > > +config PPC_FTRACE_OUT_OF_LINE_NUM_RESERVE > + int "Number of ftrace out-of-line stubs to reserve within .text" > + depends on PPC_FTRACE_OUT_OF_LINE > + default 32768 > + help > + Number of stubs to reserve for use by ftrace. This space is > + reserved within .text, and is distinct from any additional space > + added at the end of .text before the final vmlinux link. Set to > + zero to have stubs only be generated at the end of vmlinux (only > + if the size of vmlinux is less than 32MB). Set to a higher value > + if building vmlinux larger than 48MB. > + > config HOTPLUG_CPU > bool "Support for enabling/disabling CPUs" > depends on SMP && (PPC_PSERIES || \ > diff --git a/arch/powerpc/include/asm/ftrace.h b/arch/powerpc/include/asm/ftrace.h > index bdbafc668b20..28f3590ca780 100644 > --- a/arch/powerpc/include/asm/ftrace.h > +++ b/arch/powerpc/include/asm/ftrace.h > @@ -138,8 +138,10 @@ extern unsigned int ftrace_tramp_text[], ftrace_tramp_init[]; > struct ftrace_ool_stub { > u32 insn[4]; > }; > -extern struct ftrace_ool_stub ftrace_ool_stub_text_end[], ftrace_ool_stub_inittext[]; > -extern unsigned int ftrace_ool_stub_text_end_count, ftrace_ool_stub_inittext_count; > +extern struct ftrace_ool_stub ftrace_ool_stub_text_end[], ftrace_ool_stub_text[], > + ftrace_ool_stub_inittext[]; > +extern unsigned int ftrace_ool_stub_text_end_count, ftrace_ool_stub_text_count, > + ftrace_ool_stub_inittext_count; > #endif > void ftrace_free_init_tramp(void); > unsigned long ftrace_call_adjust(unsigned long addr); > diff --git a/arch/powerpc/kernel/trace/ftrace.c b/arch/powerpc/kernel/trace/ftrace.c > index 1fee074388cc..bee2c54a8c04 100644 > --- a/arch/powerpc/kernel/trace/ftrace.c > +++ b/arch/powerpc/kernel/trace/ftrace.c > @@ -168,7 +168,7 @@ static int ftrace_get_call_inst(struct dyn_ftrace *rec, unsigned long addr, ppc_ > static int ftrace_init_ool_stub(struct module *mod, struct dyn_ftrace *rec) > { > #ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE > - static int ool_stub_text_end_index, ool_stub_inittext_index; > + static int ool_stub_text_index, ool_stub_text_end_index, ool_stub_inittext_index; > int ret = 0, ool_stub_count, *ool_stub_index; > ppc_inst_t inst; > /* > @@ -191,9 +191,22 @@ static int ftrace_init_ool_stub(struct module *mod, struct dyn_ftrace *rec) > ool_stub_index = &ool_stub_inittext_index; > ool_stub_count = ftrace_ool_stub_inittext_count; > } else if (is_kernel_text(rec->ip)) { > - ool_stub = ftrace_ool_stub_text_end; > - ool_stub_index = &ool_stub_text_end_index; > - ool_stub_count = ftrace_ool_stub_text_end_count; > + /* > + * ftrace records are sorted, so we first use up the stub area within .text > + * (ftrace_ool_stub_text) before using the area at the end of .text > + * (ftrace_ool_stub_text_end), unless the stub is out of range of the record. > + */ > + if (ool_stub_text_index >= ftrace_ool_stub_text_count || > + !is_offset_in_branch_range((long)rec->ip - > + (long)&ftrace_ool_stub_text[ool_stub_text_index])) { > + ool_stub = ftrace_ool_stub_text_end; > + ool_stub_index = &ool_stub_text_end_index; > + ool_stub_count = ftrace_ool_stub_text_end_count; > + } else { > + ool_stub = ftrace_ool_stub_text; > + ool_stub_index = &ool_stub_text_index; > + ool_stub_count = ftrace_ool_stub_text_count; > + } > #ifdef CONFIG_MODULES > } else if (mod) { > ool_stub = mod->arch.ool_stubs; > diff --git a/arch/powerpc/kernel/trace/ftrace_entry.S b/arch/powerpc/kernel/trace/ftrace_entry.S > index 5b2fc6483dce..a6bf7f841040 100644 > --- a/arch/powerpc/kernel/trace/ftrace_entry.S > +++ b/arch/powerpc/kernel/trace/ftrace_entry.S > @@ -374,6 +374,14 @@ _GLOBAL(return_to_handler) > blr > #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ > > +#ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE > +SYM_DATA(ftrace_ool_stub_text_count, .long CONFIG_PPC_FTRACE_OUT_OF_LINE_NUM_RESERVE) > + > +SYM_CODE_START(ftrace_ool_stub_text) > + .space CONFIG_PPC_FTRACE_OUT_OF_LINE_NUM_RESERVE * FTRACE_OOL_STUB_SIZE > +SYM_CODE_END(ftrace_ool_stub_text) > +#endif > + > .pushsection ".tramp.ftrace.text","aw",@progbits; > .globl ftrace_tramp_text > ftrace_tramp_text: > diff --git a/arch/powerpc/tools/Makefile b/arch/powerpc/tools/Makefile > index d2e7ecd5f46f..e1f7afcd9fdf 100644 > --- a/arch/powerpc/tools/Makefile > +++ b/arch/powerpc/tools/Makefile > @@ -1,7 +1,8 @@ > # SPDX-License-Identifier: GPL-2.0-or-later > > quiet_cmd_gen_ftrace_ool_stubs = GEN $@ > - cmd_gen_ftrace_ool_stubs = $< "$(CONFIG_64BIT)" "$(OBJDUMP)" vmlinux.o $@ > + cmd_gen_ftrace_ool_stubs = $< "$(CONFIG_PPC_FTRACE_OUT_OF_LINE_NUM_RESERVE)" "$(CONFIG_64BIT)" \ > + "$(OBJDUMP)" vmlinux.o $@ > > $(obj)/vmlinux.arch.S: $(src)/ftrace-gen-ool-stubs.sh vmlinux.o FORCE > $(call if_changed,gen_ftrace_ool_stubs) > diff --git a/arch/powerpc/tools/ftrace-gen-ool-stubs.sh b/arch/powerpc/tools/ftrace-gen-ool-stubs.sh > index 96e1ca5803e4..3ea0f23f2501 100755 > --- a/arch/powerpc/tools/ftrace-gen-ool-stubs.sh > +++ b/arch/powerpc/tools/ftrace-gen-ool-stubs.sh > @@ -4,10 +4,11 @@ > # Error out on error > set -e > > -is_64bit="$1" > -objdump="$2" > -vmlinux_o="$3" > -arch_vmlinux_S="$4" > +num_ool_stubs_text_builtin="$1" > +is_64bit="$2" > +objdump="$3" > +vmlinux_o="$4" > +arch_vmlinux_S="$5" > > RELOCATION=R_PPC64_ADDR64 > if [ -z "$is_64bit" ]; then > @@ -19,15 +20,21 @@ num_ool_stubs_text=$($objdump -r -j __patchable_function_entries "$vmlinux_o" | > num_ool_stubs_inittext=$($objdump -r -j __patchable_function_entries "$vmlinux_o" | > grep ".init.text" | grep -c "$RELOCATION") > > +if [ "$num_ool_stubs_text" -gt "$num_ool_stubs_text_builtin" ]; then > + num_ool_stubs_text_end=$((num_ool_stubs_text - num_ool_stubs_text_builtin)) > +else > + num_ool_stubs_text_end=0 > +fi > + > cat > "$arch_vmlinux_S" <<EOF > #include <asm/asm-offsets.h> > #include <linux/linkage.h> > > .pushsection .tramp.ftrace.text,"aw" > -SYM_DATA(ftrace_ool_stub_text_end_count, .long $num_ool_stubs_text) > +SYM_DATA(ftrace_ool_stub_text_end_count, .long $num_ool_stubs_text_end) > > SYM_CODE_START(ftrace_ool_stub_text_end) > - .space $num_ool_stubs_text * FTRACE_OOL_STUB_SIZE > + .space $num_ool_stubs_text_end * FTRACE_OOL_STUB_SIZE > SYM_CODE_END(ftrace_ool_stub_text_end) > .popsection I got this warning: GEN arch/powerpc/tools/vmlinux.arch.S AS arch/powerpc/tools/vmlinux.arch.o arch/powerpc/tools/vmlinux.arch.S: Assembler messages: arch/powerpc/tools/vmlinux.arch.S:9: Warning: .space repeat count is zero, ignored
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 26e3060e44f4..2e347f682c15 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -573,6 +573,18 @@ config PPC_FTRACE_OUT_OF_LINE def_bool PPC64 && ARCH_USING_PATCHABLE_FUNCTION_ENTRY select ARCH_WANTS_PRE_LINK_VMLINUX +config PPC_FTRACE_OUT_OF_LINE_NUM_RESERVE + int "Number of ftrace out-of-line stubs to reserve within .text" + depends on PPC_FTRACE_OUT_OF_LINE + default 32768 + help + Number of stubs to reserve for use by ftrace. This space is + reserved within .text, and is distinct from any additional space + added at the end of .text before the final vmlinux link. Set to + zero to have stubs only be generated at the end of vmlinux (only + if the size of vmlinux is less than 32MB). Set to a higher value + if building vmlinux larger than 48MB. + config HOTPLUG_CPU bool "Support for enabling/disabling CPUs" depends on SMP && (PPC_PSERIES || \ diff --git a/arch/powerpc/include/asm/ftrace.h b/arch/powerpc/include/asm/ftrace.h index bdbafc668b20..28f3590ca780 100644 --- a/arch/powerpc/include/asm/ftrace.h +++ b/arch/powerpc/include/asm/ftrace.h @@ -138,8 +138,10 @@ extern unsigned int ftrace_tramp_text[], ftrace_tramp_init[]; struct ftrace_ool_stub { u32 insn[4]; }; -extern struct ftrace_ool_stub ftrace_ool_stub_text_end[], ftrace_ool_stub_inittext[]; -extern unsigned int ftrace_ool_stub_text_end_count, ftrace_ool_stub_inittext_count; +extern struct ftrace_ool_stub ftrace_ool_stub_text_end[], ftrace_ool_stub_text[], + ftrace_ool_stub_inittext[]; +extern unsigned int ftrace_ool_stub_text_end_count, ftrace_ool_stub_text_count, + ftrace_ool_stub_inittext_count; #endif void ftrace_free_init_tramp(void); unsigned long ftrace_call_adjust(unsigned long addr); diff --git a/arch/powerpc/kernel/trace/ftrace.c b/arch/powerpc/kernel/trace/ftrace.c index 1fee074388cc..bee2c54a8c04 100644 --- a/arch/powerpc/kernel/trace/ftrace.c +++ b/arch/powerpc/kernel/trace/ftrace.c @@ -168,7 +168,7 @@ static int ftrace_get_call_inst(struct dyn_ftrace *rec, unsigned long addr, ppc_ static int ftrace_init_ool_stub(struct module *mod, struct dyn_ftrace *rec) { #ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE - static int ool_stub_text_end_index, ool_stub_inittext_index; + static int ool_stub_text_index, ool_stub_text_end_index, ool_stub_inittext_index; int ret = 0, ool_stub_count, *ool_stub_index; ppc_inst_t inst; /* @@ -191,9 +191,22 @@ static int ftrace_init_ool_stub(struct module *mod, struct dyn_ftrace *rec) ool_stub_index = &ool_stub_inittext_index; ool_stub_count = ftrace_ool_stub_inittext_count; } else if (is_kernel_text(rec->ip)) { - ool_stub = ftrace_ool_stub_text_end; - ool_stub_index = &ool_stub_text_end_index; - ool_stub_count = ftrace_ool_stub_text_end_count; + /* + * ftrace records are sorted, so we first use up the stub area within .text + * (ftrace_ool_stub_text) before using the area at the end of .text + * (ftrace_ool_stub_text_end), unless the stub is out of range of the record. + */ + if (ool_stub_text_index >= ftrace_ool_stub_text_count || + !is_offset_in_branch_range((long)rec->ip - + (long)&ftrace_ool_stub_text[ool_stub_text_index])) { + ool_stub = ftrace_ool_stub_text_end; + ool_stub_index = &ool_stub_text_end_index; + ool_stub_count = ftrace_ool_stub_text_end_count; + } else { + ool_stub = ftrace_ool_stub_text; + ool_stub_index = &ool_stub_text_index; + ool_stub_count = ftrace_ool_stub_text_count; + } #ifdef CONFIG_MODULES } else if (mod) { ool_stub = mod->arch.ool_stubs; diff --git a/arch/powerpc/kernel/trace/ftrace_entry.S b/arch/powerpc/kernel/trace/ftrace_entry.S index 5b2fc6483dce..a6bf7f841040 100644 --- a/arch/powerpc/kernel/trace/ftrace_entry.S +++ b/arch/powerpc/kernel/trace/ftrace_entry.S @@ -374,6 +374,14 @@ _GLOBAL(return_to_handler) blr #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ +#ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE +SYM_DATA(ftrace_ool_stub_text_count, .long CONFIG_PPC_FTRACE_OUT_OF_LINE_NUM_RESERVE) + +SYM_CODE_START(ftrace_ool_stub_text) + .space CONFIG_PPC_FTRACE_OUT_OF_LINE_NUM_RESERVE * FTRACE_OOL_STUB_SIZE +SYM_CODE_END(ftrace_ool_stub_text) +#endif + .pushsection ".tramp.ftrace.text","aw",@progbits; .globl ftrace_tramp_text ftrace_tramp_text: diff --git a/arch/powerpc/tools/Makefile b/arch/powerpc/tools/Makefile index d2e7ecd5f46f..e1f7afcd9fdf 100644 --- a/arch/powerpc/tools/Makefile +++ b/arch/powerpc/tools/Makefile @@ -1,7 +1,8 @@ # SPDX-License-Identifier: GPL-2.0-or-later quiet_cmd_gen_ftrace_ool_stubs = GEN $@ - cmd_gen_ftrace_ool_stubs = $< "$(CONFIG_64BIT)" "$(OBJDUMP)" vmlinux.o $@ + cmd_gen_ftrace_ool_stubs = $< "$(CONFIG_PPC_FTRACE_OUT_OF_LINE_NUM_RESERVE)" "$(CONFIG_64BIT)" \ + "$(OBJDUMP)" vmlinux.o $@ $(obj)/vmlinux.arch.S: $(src)/ftrace-gen-ool-stubs.sh vmlinux.o FORCE $(call if_changed,gen_ftrace_ool_stubs) diff --git a/arch/powerpc/tools/ftrace-gen-ool-stubs.sh b/arch/powerpc/tools/ftrace-gen-ool-stubs.sh index 96e1ca5803e4..3ea0f23f2501 100755 --- a/arch/powerpc/tools/ftrace-gen-ool-stubs.sh +++ b/arch/powerpc/tools/ftrace-gen-ool-stubs.sh @@ -4,10 +4,11 @@ # Error out on error set -e -is_64bit="$1" -objdump="$2" -vmlinux_o="$3" -arch_vmlinux_S="$4" +num_ool_stubs_text_builtin="$1" +is_64bit="$2" +objdump="$3" +vmlinux_o="$4" +arch_vmlinux_S="$5" RELOCATION=R_PPC64_ADDR64 if [ -z "$is_64bit" ]; then @@ -19,15 +20,21 @@ num_ool_stubs_text=$($objdump -r -j __patchable_function_entries "$vmlinux_o" | num_ool_stubs_inittext=$($objdump -r -j __patchable_function_entries "$vmlinux_o" | grep ".init.text" | grep -c "$RELOCATION") +if [ "$num_ool_stubs_text" -gt "$num_ool_stubs_text_builtin" ]; then + num_ool_stubs_text_end=$((num_ool_stubs_text - num_ool_stubs_text_builtin)) +else + num_ool_stubs_text_end=0 +fi + cat > "$arch_vmlinux_S" <<EOF #include <asm/asm-offsets.h> #include <linux/linkage.h> .pushsection .tramp.ftrace.text,"aw" -SYM_DATA(ftrace_ool_stub_text_end_count, .long $num_ool_stubs_text) +SYM_DATA(ftrace_ool_stub_text_end_count, .long $num_ool_stubs_text_end) SYM_CODE_START(ftrace_ool_stub_text_end) - .space $num_ool_stubs_text * FTRACE_OOL_STUB_SIZE + .space $num_ool_stubs_text_end * FTRACE_OOL_STUB_SIZE SYM_CODE_END(ftrace_ool_stub_text_end) .popsection