diff mbox series

[5/6] powerpc/64: Add support for out-of-line static calls

Message ID 20220916062330.430468-6-bgray@linux.ibm.com (mailing list archive)
State Superseded
Headers show
Series Out-of-line static calls for powerpc64 ELF V2 | expand

Commit Message

Benjamin Gray Sept. 16, 2022, 6:23 a.m. UTC 
Implement static call support for 64 bit V2 ABI. This requires
making sure the TOC is kept correct across kernel-module
boundaries. As a secondary concern, it tries to use the local
entry point of a target wherever possible. It does so by
checking if both tramp & target are kernel code, and falls
back to detecting the common global entry point patterns
if modules are involved. Detecting the global entry point is
also required for setting the local entry point as the trampoline
target: if we cannot detect the local entry point, then we need to
convservatively initialise r12 and use the global entry point.

The trampolines are marked with `.localentry NAME, 1` to make the
linker save and restore the TOC on each call to the trampoline. This
allows the trampoline to safely target functions with different TOC
values.

However this directive also implies the TOC is not initialised on entry
to the trampoline. The kernel TOC is easily found in the PACA, but not
an arbitrary module TOC. Therefore the trampoline implementation depends
on whether it's in the kernel or not. If in the kernel, we initialise
the TOC using the PACA. If in a module, we have to initialise the TOC
with zero context, so it's quite expensive.

Signed-off-by: Benjamin Gray <bgray@linux.ibm.com>
---
 arch/powerpc/Kconfig                     | 12 +++-
 arch/powerpc/include/asm/code-patching.h |  1 +
 arch/powerpc/include/asm/static_call.h   | 80 +++++++++++++++++++--
 arch/powerpc/kernel/Makefile             |  3 +-
 arch/powerpc/kernel/static_call.c        | 92 ++++++++++++++++++++++--
 5 files changed, 175 insertions(+), 13 deletions(-)

Comments

kernel test robot Sept. 16, 2022, 8:32 a.m. UTC | #1 
Hi Benjamin,

Thank you for the patch! Yet something to improve:

[auto build test ERROR on powerpc/next]
[also build test ERROR on next-20220915]
[cannot apply to linus/master v6.0-rc5]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/Benjamin-Gray/Out-of-line-static-calls-for-powerpc64-ELF-V2/20220916-142951
base:   https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git next
config: powerpc-allyesconfig (https://download.01.org/0day-ci/archive/20220916/202209161627.bZaVohGQ-lkp@intel.com/config)
compiler: powerpc-linux-gcc (GCC) 12.1.0
reproduce (this is a W=1 build):
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # https://github.com/intel-lab-lkp/linux/commit/f76701ac48849394d2a140d87c4263e796f4920f
        git remote add linux-review https://github.com/intel-lab-lkp/linux
        git fetch --no-tags linux-review Benjamin-Gray/Out-of-line-static-calls-for-powerpc64-ELF-V2/20220916-142951
        git checkout f76701ac48849394d2a140d87c4263e796f4920f
        # save the config file
        mkdir build_dir && cp config build_dir/.config
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-12.1.0 make.cross W=1 O=build_dir ARCH=powerpc SHELL=/bin/bash arch/powerpc/kernel/ kernel/trace/

If you fix the issue, kindly add following tag where applicable
Reported-by: kernel test robot <lkp@intel.com>

Note: the linux-review/Benjamin-Gray/Out-of-line-static-calls-for-powerpc64-ELF-V2/20220916-142951 HEAD d2702454a6382dfec6e4f47e4c3e49f70bec7049 builds fine.
      It only hurts bisectability.

All error/warnings (new ones prefixed by >>):

   In file included from include/linux/static_call.h:139,
                    from include/linux/tracepoint.h:22,
                    from include/trace/syscall.h:5,
                    from include/linux/syscalls.h:88,
                    from arch/powerpc/kernel/syscalls.c:19:
>> arch/powerpc/include/asm/static_call.h:94:2: error: #error "Unsupported ABI"
      94 | #error "Unsupported ABI"
         |  ^~~~~
--
   In file included from include/linux/static_call.h:139,
                    from arch/powerpc/kernel/irq.c:55:
>> arch/powerpc/include/asm/static_call.h:94:2: error: #error "Unsupported ABI"
      94 | #error "Unsupported ABI"
         |  ^~~~~
>> arch/powerpc/include/asm/static_call.h:97:73: error: expected ')' before string constant
      97 | #define ARCH_DEFINE_STATIC_CALL_TRAMP(name, func)       __PPC_SCT(name, "b " #func)
         |                                                                         ^~~~
   include/linux/static_call.h:233:9: note: in expansion of macro 'ARCH_DEFINE_STATIC_CALL_TRAMP'
     233 |         ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
         |         ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   include/linux/tracepoint.h:325:9: note: in expansion of macro 'DEFINE_STATIC_CALL'
     325 |         DEFINE_STATIC_CALL(tp_func_##_name, __traceiter_##_name);
         |         ^~~~~~~~~~~~~~~~~~
   include/linux/tracepoint.h:328:9: note: in expansion of macro 'DEFINE_TRACE_FN'
     328 |         DEFINE_TRACE_FN(name, NULL, NULL, PARAMS(proto), PARAMS(args));
         |         ^~~~~~~~~~~~~~~
   include/trace/define_trace.h:57:9: note: in expansion of macro 'DEFINE_TRACE'
      57 |         DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
         |         ^~~~~~~~~~~~
   arch/powerpc/include/asm/trace.h:29:1: note: in expansion of macro 'DEFINE_EVENT'
      29 | DEFINE_EVENT(ppc64_interrupt_class, irq_entry,
         | ^~~~~~~~~~~~
>> arch/powerpc/include/asm/static_call.h:97:73: error: expected ')' before string constant
      97 | #define ARCH_DEFINE_STATIC_CALL_TRAMP(name, func)       __PPC_SCT(name, "b " #func)
         |                                                                         ^~~~
   include/linux/static_call.h:233:9: note: in expansion of macro 'ARCH_DEFINE_STATIC_CALL_TRAMP'
     233 |         ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
         |         ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   include/linux/tracepoint.h:325:9: note: in expansion of macro 'DEFINE_STATIC_CALL'
     325 |         DEFINE_STATIC_CALL(tp_func_##_name, __traceiter_##_name);
         |         ^~~~~~~~~~~~~~~~~~
   include/linux/tracepoint.h:328:9: note: in expansion of macro 'DEFINE_TRACE_FN'
     328 |         DEFINE_TRACE_FN(name, NULL, NULL, PARAMS(proto), PARAMS(args));
         |         ^~~~~~~~~~~~~~~
   include/trace/define_trace.h:57:9: note: in expansion of macro 'DEFINE_TRACE'
      57 |         DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
         |         ^~~~~~~~~~~~
   arch/powerpc/include/asm/trace.h:36:1: note: in expansion of macro 'DEFINE_EVENT'
      36 | DEFINE_EVENT(ppc64_interrupt_class, irq_exit,
         | ^~~~~~~~~~~~
>> arch/powerpc/include/asm/static_call.h:97:73: error: expected ')' before string constant
      97 | #define ARCH_DEFINE_STATIC_CALL_TRAMP(name, func)       __PPC_SCT(name, "b " #func)
         |                                                                         ^~~~
   include/linux/static_call.h:233:9: note: in expansion of macro 'ARCH_DEFINE_STATIC_CALL_TRAMP'
     233 |         ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
         |         ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   include/linux/tracepoint.h:325:9: note: in expansion of macro 'DEFINE_STATIC_CALL'
     325 |         DEFINE_STATIC_CALL(tp_func_##_name, __traceiter_##_name);
         |         ^~~~~~~~~~~~~~~~~~
   include/linux/tracepoint.h:328:9: note: in expansion of macro 'DEFINE_TRACE_FN'
     328 |         DEFINE_TRACE_FN(name, NULL, NULL, PARAMS(proto), PARAMS(args));
         |         ^~~~~~~~~~~~~~~
   include/trace/define_trace.h:57:9: note: in expansion of macro 'DEFINE_TRACE'
      57 |         DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
         |         ^~~~~~~~~~~~
   arch/powerpc/include/asm/trace.h:43:1: note: in expansion of macro 'DEFINE_EVENT'
      43 | DEFINE_EVENT(ppc64_interrupt_class, timer_interrupt_entry,
         | ^~~~~~~~~~~~
>> arch/powerpc/include/asm/static_call.h:97:73: error: expected ')' before string constant
      97 | #define ARCH_DEFINE_STATIC_CALL_TRAMP(name, func)       __PPC_SCT(name, "b " #func)
         |                                                                         ^~~~
   include/linux/static_call.h:233:9: note: in expansion of macro 'ARCH_DEFINE_STATIC_CALL_TRAMP'
     233 |         ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
         |         ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   include/linux/tracepoint.h:325:9: note: in expansion of macro 'DEFINE_STATIC_CALL'
     325 |         DEFINE_STATIC_CALL(tp_func_##_name, __traceiter_##_name);
         |         ^~~~~~~~~~~~~~~~~~
   include/linux/tracepoint.h:328:9: note: in expansion of macro 'DEFINE_TRACE_FN'
     328 |         DEFINE_TRACE_FN(name, NULL, NULL, PARAMS(proto), PARAMS(args));
         |         ^~~~~~~~~~~~~~~
   include/trace/define_trace.h:57:9: note: in expansion of macro 'DEFINE_TRACE'
      57 |         DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
         |         ^~~~~~~~~~~~
   arch/powerpc/include/asm/trace.h:50:1: note: in expansion of macro 'DEFINE_EVENT'
      50 | DEFINE_EVENT(ppc64_interrupt_class, timer_interrupt_exit,
         | ^~~~~~~~~~~~
>> arch/powerpc/include/asm/static_call.h:97:73: error: expected ')' before string constant
      97 | #define ARCH_DEFINE_STATIC_CALL_TRAMP(name, func)       __PPC_SCT(name, "b " #func)
         |                                                                         ^~~~
   include/linux/static_call.h:233:9: note: in expansion of macro 'ARCH_DEFINE_STATIC_CALL_TRAMP'
     233 |         ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
         |         ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   include/linux/tracepoint.h:325:9: note: in expansion of macro 'DEFINE_STATIC_CALL'
     325 |         DEFINE_STATIC_CALL(tp_func_##_name, __traceiter_##_name);
         |         ^~~~~~~~~~~~~~~~~~
   include/linux/tracepoint.h:328:9: note: in expansion of macro 'DEFINE_TRACE_FN'
     328 |         DEFINE_TRACE_FN(name, NULL, NULL, PARAMS(proto), PARAMS(args));
         |         ^~~~~~~~~~~~~~~
   include/trace/define_trace.h:28:9: note: in expansion of macro 'DEFINE_TRACE'
      28 |         DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
         |         ^~~~~~~~~~~~
   arch/powerpc/include/asm/trace.h:167:1: note: in expansion of macro 'TRACE_EVENT'
     167 | TRACE_EVENT(hash_fault,
         | ^~~~~~~~~~~
>> arch/powerpc/include/asm/static_call.h:97:73: error: expected ')' before string constant
      97 | #define ARCH_DEFINE_STATIC_CALL_TRAMP(name, func)       __PPC_SCT(name, "b " #func)
         |                                                                         ^~~~
   include/linux/static_call.h:233:9: note: in expansion of macro 'ARCH_DEFINE_STATIC_CALL_TRAMP'
     233 |         ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
         |         ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   include/linux/tracepoint.h:325:9: note: in expansion of macro 'DEFINE_STATIC_CALL'
     325 |         DEFINE_STATIC_CALL(tp_func_##_name, __traceiter_##_name);
         |         ^~~~~~~~~~~~~~~~~~
   include/linux/tracepoint.h:328:9: note: in expansion of macro 'DEFINE_TRACE_FN'
     328 |         DEFINE_TRACE_FN(name, NULL, NULL, PARAMS(proto), PARAMS(args));
         |         ^~~~~~~~~~~~~~~
   include/trace/define_trace.h:28:9: note: in expansion of macro 'DEFINE_TRACE'
      28 |         DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
         |         ^~~~~~~~~~~~
   arch/powerpc/include/asm/trace.h:188:1: note: in expansion of macro 'TRACE_EVENT'
     188 | TRACE_EVENT(tlbie,
         | ^~~~~~~~~~~
>> arch/powerpc/include/asm/static_call.h:97:73: error: expected ')' before string constant
      97 | #define ARCH_DEFINE_STATIC_CALL_TRAMP(name, func)       __PPC_SCT(name, "b " #func)
         |                                                                         ^~~~
   include/linux/static_call.h:233:9: note: in expansion of macro 'ARCH_DEFINE_STATIC_CALL_TRAMP'
     233 |         ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
         |         ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   include/linux/tracepoint.h:325:9: note: in expansion of macro 'DEFINE_STATIC_CALL'
     325 |         DEFINE_STATIC_CALL(tp_func_##_name, __traceiter_##_name);
         |         ^~~~~~~~~~~~~~~~~~
   include/linux/tracepoint.h:328:9: note: in expansion of macro 'DEFINE_TRACE_FN'
     328 |         DEFINE_TRACE_FN(name, NULL, NULL, PARAMS(proto), PARAMS(args));
         |         ^~~~~~~~~~~~~~~
   include/trace/define_trace.h:28:9: note: in expansion of macro 'DEFINE_TRACE'
      28 |         DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
         |         ^~~~~~~~~~~~
   arch/powerpc/include/asm/trace.h:220:1: note: in expansion of macro 'TRACE_EVENT'
     220 | TRACE_EVENT(tlbia,
         | ^~~~~~~~~~~
   arch/powerpc/include/asm/static_call.h:99:73: error: expected ')' before string constant
      99 | #define ARCH_DEFINE_STATIC_CALL_RET0_TRAMP(name)        __PPC_SCT(name, "b 1f")
         |                                                                         ^~~~~~
   include/linux/static_call.h:247:9: note: in expansion of macro 'ARCH_DEFINE_STATIC_CALL_RET0_TRAMP'
     247 |         ARCH_DEFINE_STATIC_CALL_RET0_TRAMP(name)
         |         ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   arch/powerpc/kernel/irq.c:223:1: note: in expansion of macro 'DEFINE_STATIC_CALL_RET0'
     223 | DEFINE_STATIC_CALL_RET0(ppc_get_irq, *ppc_md.get_irq);
         | ^~~~~~~~~~~~~~~~~~~~~~~
--
   In file included from include/linux/static_call.h:139,
                    from arch/powerpc/kernel/static_call.c:7:
>> arch/powerpc/include/asm/static_call.h:94:2: error: #error "Unsupported ABI"
      94 | #error "Unsupported ABI"
         |  ^~~~~
   arch/powerpc/kernel/static_call.c: In function 'ppc_function_toc':
>> arch/powerpc/kernel/static_call.c:16:29: warning: cast from pointer to integer of different size [-Wpointer-to-int-cast]
      16 |                 u64 addr = ((u64) func + si1) + si2;
         |                             ^
>> arch/powerpc/kernel/static_call.c:21:32: warning: cast to pointer from integer of different size [-Wint-to-pointer-cast]
      21 |                         return (void*)addr;
         |                                ^
   arch/powerpc/kernel/static_call.c: In function 'get_inst_addr':
>> arch/powerpc/kernel/static_call.c:52:51: error: 'PPC_SCT_INST_KERNEL' undeclared (first use in this function)
      52 |         return tramp + (core_kernel_text(tramp) ? PPC_SCT_INST_KERNEL : PPC_SCT_INST_MODULE);
         |                                                   ^~~~~~~~~~~~~~~~~~~
   arch/powerpc/kernel/static_call.c:52:51: note: each undeclared identifier is reported only once for each function it appears in
>> arch/powerpc/kernel/static_call.c:52:73: error: 'PPC_SCT_INST_MODULE' undeclared (first use in this function)
      52 |         return tramp + (core_kernel_text(tramp) ? PPC_SCT_INST_KERNEL : PPC_SCT_INST_MODULE);
         |                                                                         ^~~~~~~~~~~~~~~~~~~
   arch/powerpc/kernel/static_call.c: In function 'get_ret0_addr':
>> arch/powerpc/kernel/static_call.c:56:51: error: 'PPC_SCT_RET0_KERNEL' undeclared (first use in this function)
      56 |         return tramp + (core_kernel_text(tramp) ? PPC_SCT_RET0_KERNEL : PPC_SCT_RET0_MODULE);
         |                                                   ^~~~~~~~~~~~~~~~~~~
>> arch/powerpc/kernel/static_call.c:56:73: error: 'PPC_SCT_RET0_MODULE' undeclared (first use in this function)
      56 |         return tramp + (core_kernel_text(tramp) ? PPC_SCT_RET0_KERNEL : PPC_SCT_RET0_MODULE);
         |                                                                         ^~~~~~~~~~~~~~~~~~~
   arch/powerpc/kernel/static_call.c: In function 'get_data_addr':
>> arch/powerpc/kernel/static_call.c:60:51: error: 'PPC_SCT_DATA_KERNEL' undeclared (first use in this function)
      60 |         return tramp + (core_kernel_text(tramp) ? PPC_SCT_DATA_KERNEL : PPC_SCT_DATA_MODULE);
         |                                                   ^~~~~~~~~~~~~~~~~~~
>> arch/powerpc/kernel/static_call.c:60:73: error: 'PPC_SCT_DATA_MODULE' undeclared (first use in this function)
      60 |         return tramp + (core_kernel_text(tramp) ? PPC_SCT_DATA_KERNEL : PPC_SCT_DATA_MODULE);
         |                                                                         ^~~~~~~~~~~~~~~~~~~
   arch/powerpc/kernel/static_call.c: In function 'get_ret0_addr':
   arch/powerpc/kernel/static_call.c:57:1: error: control reaches end of non-void function [-Werror=return-type]
      57 | }
         | ^
   arch/powerpc/kernel/static_call.c: In function 'get_inst_addr':
   arch/powerpc/kernel/static_call.c:53:1: error: control reaches end of non-void function [-Werror=return-type]
      53 | }
         | ^
   arch/powerpc/kernel/static_call.c: In function 'get_data_addr':
   arch/powerpc/kernel/static_call.c:61:1: error: control reaches end of non-void function [-Werror=return-type]
      61 | }
         | ^
   cc1: some warnings being treated as errors
--
   In file included from include/linux/static_call.h:139,
                    from include/linux/tracepoint.h:22,
                    from include/trace/events/sched.h:10,
                    from kernel/trace/ftrace.c:39:
>> arch/powerpc/include/asm/static_call.h:94:2: error: #error "Unsupported ABI"
      94 | #error "Unsupported ABI"
         |  ^~~~~
   kernel/trace/ftrace.c:299:5: warning: no previous prototype for '__register_ftrace_function' [-Wmissing-prototypes]
     299 | int __register_ftrace_function(struct ftrace_ops *ops)
         |     ^~~~~~~~~~~~~~~~~~~~~~~~~~
   kernel/trace/ftrace.c:342:5: warning: no previous prototype for '__unregister_ftrace_function' [-Wmissing-prototypes]
     342 | int __unregister_ftrace_function(struct ftrace_ops *ops)
         |     ^~~~~~~~~~~~~~~~~~~~~~~~~~~~
   kernel/trace/ftrace.c:4044:15: warning: no previous prototype for 'arch_ftrace_match_adjust' [-Wmissing-prototypes]
    4044 | char * __weak arch_ftrace_match_adjust(char *str, const char *search)
         |               ^~~~~~~~~~~~~~~~~~~~~~~~
--
   In file included from include/linux/static_call.h:139,
                    from include/linux/tracepoint.h:22,
                    from kernel/trace/trace.h:12,
                    from kernel/trace/trace.c:53:
>> arch/powerpc/include/asm/static_call.h:94:2: error: #error "Unsupported ABI"
      94 | #error "Unsupported ABI"
         |  ^~~~~
   kernel/trace/trace.c: In function 'trace_check_vprintf':
   kernel/trace/trace.c:3846:17: warning: function 'trace_check_vprintf' might be a candidate for 'gnu_printf' format attribute [-Wsuggest-attribute=format]
    3846 |                 trace_seq_vprintf(&iter->seq, iter->fmt, ap);
         |                 ^~~~~~~~~~~~~~~~~
   kernel/trace/trace.c:3913:17: warning: function 'trace_check_vprintf' might be a candidate for 'gnu_printf' format attribute [-Wsuggest-attribute=format]
    3913 |                 trace_seq_vprintf(&iter->seq, p, ap);
         |                 ^~~~~~~~~~~~~~~~~
--
   In file included from include/linux/static_call.h:139,
                    from include/linux/tracepoint.h:22,
                    from kernel/trace/trace.h:12,
                    from kernel/trace/trace_output.h:6,
                    from kernel/trace/trace_output.c:15:
>> arch/powerpc/include/asm/static_call.h:94:2: error: #error "Unsupported ABI"
      94 | #error "Unsupported ABI"
         |  ^~~~~
   kernel/trace/trace_output.c: In function 'trace_output_raw':
   kernel/trace/trace_output.c:332:9: warning: function 'trace_output_raw' might be a candidate for 'gnu_printf' format attribute [-Wsuggest-attribute=format]
     332 |         trace_seq_vprintf(s, trace_event_format(iter, fmt), ap);
         |         ^~~~~~~~~~~~~~~~~
--
   In file included from include/linux/static_call.h:139,
                    from include/linux/tracepoint.h:22,
                    from kernel/trace/trace.h:12,
                    from kernel/trace/trace_preemptirq.c:13:
>> arch/powerpc/include/asm/static_call.h:94:2: error: #error "Unsupported ABI"
      94 | #error "Unsupported ABI"
         |  ^~~~~
>> arch/powerpc/include/asm/static_call.h:97:73: error: expected ')' before string constant
      97 | #define ARCH_DEFINE_STATIC_CALL_TRAMP(name, func)       __PPC_SCT(name, "b " #func)
         |                                                                         ^~~~
   include/linux/static_call.h:233:9: note: in expansion of macro 'ARCH_DEFINE_STATIC_CALL_TRAMP'
     233 |         ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
         |         ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   include/linux/tracepoint.h:325:9: note: in expansion of macro 'DEFINE_STATIC_CALL'
     325 |         DEFINE_STATIC_CALL(tp_func_##_name, __traceiter_##_name);
         |         ^~~~~~~~~~~~~~~~~~
   include/linux/tracepoint.h:328:9: note: in expansion of macro 'DEFINE_TRACE_FN'
     328 |         DEFINE_TRACE_FN(name, NULL, NULL, PARAMS(proto), PARAMS(args));
         |         ^~~~~~~~~~~~~~~
   include/trace/define_trace.h:57:9: note: in expansion of macro 'DEFINE_TRACE'
      57 |         DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
         |         ^~~~~~~~~~~~
   include/trace/events/preemptirq.h:36:1: note: in expansion of macro 'DEFINE_EVENT'
      36 | DEFINE_EVENT(preemptirq_template, irq_disable,
         | ^~~~~~~~~~~~
>> arch/powerpc/include/asm/static_call.h:97:73: error: expected ')' before string constant
      97 | #define ARCH_DEFINE_STATIC_CALL_TRAMP(name, func)       __PPC_SCT(name, "b " #func)
         |                                                                         ^~~~
   include/linux/static_call.h:233:9: note: in expansion of macro 'ARCH_DEFINE_STATIC_CALL_TRAMP'
     233 |         ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
         |         ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   include/linux/tracepoint.h:325:9: note: in expansion of macro 'DEFINE_STATIC_CALL'
     325 |         DEFINE_STATIC_CALL(tp_func_##_name, __traceiter_##_name);
         |         ^~~~~~~~~~~~~~~~~~
   include/linux/tracepoint.h:328:9: note: in expansion of macro 'DEFINE_TRACE_FN'
     328 |         DEFINE_TRACE_FN(name, NULL, NULL, PARAMS(proto), PARAMS(args));
         |         ^~~~~~~~~~~~~~~
   include/trace/define_trace.h:57:9: note: in expansion of macro 'DEFINE_TRACE'
      57 |         DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
         |         ^~~~~~~~~~~~
   include/trace/events/preemptirq.h:40:1: note: in expansion of macro 'DEFINE_EVENT'
      40 | DEFINE_EVENT(preemptirq_template, irq_enable,
         | ^~~~~~~~~~~~
   kernel/trace/trace_preemptirq.c:88:16: warning: no previous prototype for 'trace_hardirqs_on_caller' [-Wmissing-prototypes]
      88 | __visible void trace_hardirqs_on_caller(unsigned long caller_addr)
         |                ^~~~~~~~~~~~~~~~~~~~~~~~
   kernel/trace/trace_preemptirq.c:103:16: warning: no previous prototype for 'trace_hardirqs_off_caller' [-Wmissing-prototypes]
     103 | __visible void trace_hardirqs_off_caller(unsigned long caller_addr)
         |                ^~~~~~~~~~~~~~~~~~~~~~~~~
--
   In file included from include/linux/static_call.h:139,
                    from include/linux/tracepoint.h:22,
                    from kernel/trace/trace.h:12,
                    from kernel/trace/trace_osnoise.c:27:
>> arch/powerpc/include/asm/static_call.h:94:2: error: #error "Unsupported ABI"
      94 | #error "Unsupported ABI"
         |  ^~~~~
>> arch/powerpc/include/asm/static_call.h:97:73: error: expected ')' before string constant
      97 | #define ARCH_DEFINE_STATIC_CALL_TRAMP(name, func)       __PPC_SCT(name, "b " #func)
         |                                                                         ^~~~
   include/linux/static_call.h:233:9: note: in expansion of macro 'ARCH_DEFINE_STATIC_CALL_TRAMP'
     233 |         ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
         |         ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   include/linux/tracepoint.h:325:9: note: in expansion of macro 'DEFINE_STATIC_CALL'
     325 |         DEFINE_STATIC_CALL(tp_func_##_name, __traceiter_##_name);
         |         ^~~~~~~~~~~~~~~~~~
   include/linux/tracepoint.h:328:9: note: in expansion of macro 'DEFINE_TRACE_FN'
     328 |         DEFINE_TRACE_FN(name, NULL, NULL, PARAMS(proto), PARAMS(args));
         |         ^~~~~~~~~~~~~~~
   include/trace/define_trace.h:28:9: note: in expansion of macro 'DEFINE_TRACE'
      28 |         DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
         |         ^~~~~~~~~~~~
   include/trace/events/osnoise.h:9:1: note: in expansion of macro 'TRACE_EVENT'
       9 | TRACE_EVENT(thread_noise,
         | ^~~~~~~~~~~
>> arch/powerpc/include/asm/static_call.h:97:73: error: expected ')' before string constant
      97 | #define ARCH_DEFINE_STATIC_CALL_TRAMP(name, func)       __PPC_SCT(name, "b " #func)
         |                                                                         ^~~~
   include/linux/static_call.h:233:9: note: in expansion of macro 'ARCH_DEFINE_STATIC_CALL_TRAMP'
     233 |         ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
         |         ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   include/linux/tracepoint.h:325:9: note: in expansion of macro 'DEFINE_STATIC_CALL'
     325 |         DEFINE_STATIC_CALL(tp_func_##_name, __traceiter_##_name);
         |         ^~~~~~~~~~~~~~~~~~
   include/linux/tracepoint.h:328:9: note: in expansion of macro 'DEFINE_TRACE_FN'
     328 |         DEFINE_TRACE_FN(name, NULL, NULL, PARAMS(proto), PARAMS(args));
         |         ^~~~~~~~~~~~~~~
   include/trace/define_trace.h:28:9: note: in expansion of macro 'DEFINE_TRACE'
      28 |         DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
         |         ^~~~~~~~~~~~
   include/trace/events/osnoise.h:37:1: note: in expansion of macro 'TRACE_EVENT'
      37 | TRACE_EVENT(softirq_noise,
         | ^~~~~~~~~~~
>> arch/powerpc/include/asm/static_call.h:97:73: error: expected ')' before string constant
      97 | #define ARCH_DEFINE_STATIC_CALL_TRAMP(name, func)       __PPC_SCT(name, "b " #func)
         |                                                                         ^~~~
   include/linux/static_call.h:233:9: note: in expansion of macro 'ARCH_DEFINE_STATIC_CALL_TRAMP'
     233 |         ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
         |         ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   include/linux/tracepoint.h:325:9: note: in expansion of macro 'DEFINE_STATIC_CALL'
     325 |         DEFINE_STATIC_CALL(tp_func_##_name, __traceiter_##_name);
         |         ^~~~~~~~~~~~~~~~~~
   include/linux/tracepoint.h:328:9: note: in expansion of macro 'DEFINE_TRACE_FN'
     328 |         DEFINE_TRACE_FN(name, NULL, NULL, PARAMS(proto), PARAMS(args));
         |         ^~~~~~~~~~~~~~~
   include/trace/define_trace.h:28:9: note: in expansion of macro 'DEFINE_TRACE'
      28 |         DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
         |         ^~~~~~~~~~~~
   include/trace/events/osnoise.h:63:1: note: in expansion of macro 'TRACE_EVENT'
      63 | TRACE_EVENT(irq_noise,
         | ^~~~~~~~~~~
>> arch/powerpc/include/asm/static_call.h:97:73: error: expected ')' before string constant
      97 | #define ARCH_DEFINE_STATIC_CALL_TRAMP(name, func)       __PPC_SCT(name, "b " #func)
         |                                                                         ^~~~
   include/linux/static_call.h:233:9: note: in expansion of macro 'ARCH_DEFINE_STATIC_CALL_TRAMP'
     233 |         ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
         |         ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   include/linux/tracepoint.h:325:9: note: in expansion of macro 'DEFINE_STATIC_CALL'
     325 |         DEFINE_STATIC_CALL(tp_func_##_name, __traceiter_##_name);
         |         ^~~~~~~~~~~~~~~~~~
   include/linux/tracepoint.h:328:9: note: in expansion of macro 'DEFINE_TRACE_FN'
     328 |         DEFINE_TRACE_FN(name, NULL, NULL, PARAMS(proto), PARAMS(args));
         |         ^~~~~~~~~~~~~~~
   include/trace/define_trace.h:28:9: note: in expansion of macro 'DEFINE_TRACE'
      28 |         DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
         |         ^~~~~~~~~~~~
   include/trace/events/osnoise.h:92:1: note: in expansion of macro 'TRACE_EVENT'
      92 | TRACE_EVENT(nmi_noise,
         | ^~~~~~~~~~~
>> arch/powerpc/include/asm/static_call.h:97:73: error: expected ')' before string constant
      97 | #define ARCH_DEFINE_STATIC_CALL_TRAMP(name, func)       __PPC_SCT(name, "b " #func)
         |                                                                         ^~~~
   include/linux/static_call.h:233:9: note: in expansion of macro 'ARCH_DEFINE_STATIC_CALL_TRAMP'
     233 |         ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
         |         ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   include/linux/tracepoint.h:325:9: note: in expansion of macro 'DEFINE_STATIC_CALL'
     325 |         DEFINE_STATIC_CALL(tp_func_##_name, __traceiter_##_name);
         |         ^~~~~~~~~~~~~~~~~~
   include/linux/tracepoint.h:328:9: note: in expansion of macro 'DEFINE_TRACE_FN'
     328 |         DEFINE_TRACE_FN(name, NULL, NULL, PARAMS(proto), PARAMS(args));
         |         ^~~~~~~~~~~~~~~
   include/trace/define_trace.h:28:9: note: in expansion of macro 'DEFINE_TRACE'
      28 |         DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
         |         ^~~~~~~~~~~~
   include/trace/events/osnoise.h:114:1: note: in expansion of macro 'TRACE_EVENT'
     114 | TRACE_EVENT(sample_threshold,
         | ^~~~~~~~~~~
..


vim +94 arch/powerpc/include/asm/static_call.h

    92	
    93	#else /* !CONFIG_PPC64_ELF_ABI_V2 && !PPC32 */
  > 94	#error "Unsupported ABI"
    95	#endif /* CONFIG_PPC64_ELF_ABI_V2 */
    96	
  > 97	#define ARCH_DEFINE_STATIC_CALL_TRAMP(name, func)	__PPC_SCT(name, "b " #func)
    98	#define ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name)	__PPC_SCT(name, "blr")
    99	#define ARCH_DEFINE_STATIC_CALL_RET0_TRAMP(name)	__PPC_SCT(name, "b 1f")
   100
diff mbox series

Patch

diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index 4c466acdc70d..d3bb61e0de43 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -248,7 +248,7 @@  config PPC
 	select HAVE_SOFTIRQ_ON_OWN_STACK
 	select HAVE_STACKPROTECTOR		if PPC32 && $(cc-option,-mstack-protector-guard=tls -mstack-protector-guard-reg=r2)
 	select HAVE_STACKPROTECTOR		if PPC64 && $(cc-option,-mstack-protector-guard=tls -mstack-protector-guard-reg=r13)
-	select HAVE_STATIC_CALL			if PPC32
+	select HAVE_STATIC_CALL			if PPC_ENABLE_STATIC_CALL
 	select HAVE_SYSCALL_TRACEPOINTS
 	select HAVE_VIRT_CPU_ACCOUNTING
 	select HUGETLB_PAGE_SIZE_VARIABLE	if PPC_BOOK3S_64 && HUGETLB_PAGE
@@ -1023,6 +1023,16 @@  config PPC_RTAS_FILTER
 	  Say Y unless you know what you are doing and the filter is causing
 	  problems for you.
 
+config PPC_ENABLE_STATIC_CALL
+	bool "Enable static calls"
+	default y
+	depends on PPC32 || PPC64_ELF_ABI_V2
+	help
+	  PowerPC static calls with the ELF V2 ABI must ensure the TOC remains
+	  consistent. The benefit is only noticeable when indirect branch prediction
+	  is disabled. Otherwise, the generic implementation is better as it is
+	  inlined and effectively a direct branch with a hot predictor.
+
 endmenu
 
 config ISA_DMA_API
diff --git a/arch/powerpc/include/asm/code-patching.h b/arch/powerpc/include/asm/code-patching.h
index 6a52c19dae46..1ab7f3522da2 100644
--- a/arch/powerpc/include/asm/code-patching.h
+++ b/arch/powerpc/include/asm/code-patching.h
@@ -126,6 +126,7 @@  int translate_branch(ppc_inst_t *instr, const u32 *dest, const u32 *src);
 bool is_conditional_branch(ppc_inst_t instr);
 
 #define OP_RT_RA_MASK	0xffff0000UL
+#define OP_SI_MASK	0x0000ffffUL
 #define LIS_R2		(PPC_RAW_LIS(_R2, 0))
 #define ADDIS_R2_R12	(PPC_RAW_ADDIS(_R2, _R12, 0))
 #define ADDI_R2_R2	(PPC_RAW_ADDI(_R2, _R2, 0))
diff --git a/arch/powerpc/include/asm/static_call.h b/arch/powerpc/include/asm/static_call.h
index de1018cc522b..5e763cf16e9c 100644
--- a/arch/powerpc/include/asm/static_call.h
+++ b/arch/powerpc/include/asm/static_call.h
@@ -2,12 +2,75 @@ 
 #ifndef _ASM_POWERPC_STATIC_CALL_H
 #define _ASM_POWERPC_STATIC_CALL_H
 
+#ifdef CONFIG_PPC64_ELF_ABI_V2
+
+#ifdef MODULE
+
+#define __PPC_SCT(name, inst)					\
+	asm(".pushsection .text, \"ax\"				\n"	\
+	    ".align 6						\n"	\
+	    ".globl " STATIC_CALL_TRAMP_STR(name) "		\n"	\
+	    ".localentry " STATIC_CALL_TRAMP_STR(name) ", 1	\n"	\
+	    STATIC_CALL_TRAMP_STR(name) ":			\n"	\
+	    "	mflr	11					\n"	\
+	    "	bcl	20, 31, $+4				\n"	\
+	    "0:	mflr	12					\n"	\
+	    "	mtlr	11					\n"	\
+	    "	addi	12, 12, (" STATIC_CALL_TRAMP_STR(name) " - 0b)	\n"	\
+	    "	addis 2, 12, (.TOC.-" STATIC_CALL_TRAMP_STR(name) ")@ha	\n"	\
+	    "	addi 2, 2, (.TOC.-" STATIC_CALL_TRAMP_STR(name) ")@l	\n"	\
+	    "	" inst "					\n"	\
+	    "	ld	12, (2f - " STATIC_CALL_TRAMP_STR(name) ")(12)	\n"	\
+	    "	mtctr	12					\n"	\
+	    "	bctr						\n"	\
+	    "1:	li	3, 0					\n"	\
+	    "	blr						\n"	\
+	    ".balign 8						\n"	\
+	    "2:	.8byte 0					\n"	\
+	    ".type " STATIC_CALL_TRAMP_STR(name) ", @function	\n"	\
+	    ".size " STATIC_CALL_TRAMP_STR(name) ", . - " STATIC_CALL_TRAMP_STR(name) " \n" \
+	    ".popsection					\n")
+
+#else /* KERNEL */
+
+#define __PPC_SCT(name, inst)					\
+	asm(".pushsection .text, \"ax\"				\n"	\
+	    ".align 5						\n"	\
+	    ".globl " STATIC_CALL_TRAMP_STR(name) "		\n"	\
+	    ".localentry " STATIC_CALL_TRAMP_STR(name) ", 1	\n"	\
+	    STATIC_CALL_TRAMP_STR(name) ":			\n"	\
+	    "	ld	2, 16(13)				\n"	\
+	    "	" inst "					\n"	\
+	    "	addis	12, 2, 2f@toc@ha			\n"	\
+	    "	ld	12, 2f@toc@l(12)			\n"	\
+	    "	mtctr	12					\n"	\
+	    "	bctr						\n"	\
+	    "1:	li	3, 0					\n"	\
+	    "	blr						\n"	\
+	    ".balign 8						\n"	\
+	    "2:	.8byte 0					\n"	\
+	    ".type " STATIC_CALL_TRAMP_STR(name) ", @function	\n"	\
+	    ".size " STATIC_CALL_TRAMP_STR(name) ", . - " STATIC_CALL_TRAMP_STR(name) " \n" \
+	    ".popsection					\n")
+
+#endif /* MODULE */
+
+#define PPC_SCT_INST_MODULE		28		/* Offset of instruction to update */
+#define PPC_SCT_RET0_MODULE		44		/* Offset of label 1 */
+#define PPC_SCT_DATA_MODULE		56		/* Offset of label 2 (aligned) */
+
+#define PPC_SCT_INST_KERNEL		4		/* Offset of instruction to update */
+#define PPC_SCT_RET0_KERNEL		24		/* Offset of label 1 */
+#define PPC_SCT_DATA_KERNEL		32		/* Offset of label 2 (aligned) */
+
+#elif defined(PPC32)
+
 #define __PPC_SCT(name, inst)					\
 	asm(".pushsection .text, \"ax\"				\n"	\
 	    ".align 5						\n"	\
 	    ".globl " STATIC_CALL_TRAMP_STR(name) "		\n"	\
 	    STATIC_CALL_TRAMP_STR(name) ":			\n"	\
-	    inst "						\n"	\
+	    "	" inst "					\n"	\
 	    "	lis	12,2f@ha				\n"	\
 	    "	lwz	12,2f@l(12)				\n"	\
 	    "	mtctr	12					\n"	\
@@ -19,11 +82,20 @@ 
 	    ".size " STATIC_CALL_TRAMP_STR(name) ", . - " STATIC_CALL_TRAMP_STR(name) " \n" \
 	    ".popsection					\n")
 
-#define PPC_SCT_RET0		20		/* Offset of label 1 */
-#define PPC_SCT_DATA		28		/* Offset of label 2 */
+#define PPC_SCT_INST_MODULE		0		/* Offset of instruction to update */
+#define PPC_SCT_RET0_MODULE		20		/* Offset of label 1 */
+#define PPC_SCT_DATA_MODULE		28		/* Offset of label 2 */
+
+#define PPC_SCT_INST_KERNEL		PPC_SCT_INST_MODULE
+#define PPC_SCT_RET0_KERNEL		PPC_SCT_RET0_MODULE
+#define PPC_SCT_DATA_KERNEL		PPC_SCT_DATA_MODULE
+
+#else /* !CONFIG_PPC64_ELF_ABI_V2 && !PPC32 */
+#error "Unsupported ABI"
+#endif /* CONFIG_PPC64_ELF_ABI_V2 */
 
 #define ARCH_DEFINE_STATIC_CALL_TRAMP(name, func)	__PPC_SCT(name, "b " #func)
 #define ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name)	__PPC_SCT(name, "blr")
-#define ARCH_DEFINE_STATIC_CALL_RET0_TRAMP(name)	__PPC_SCT(name, "b .+20")
+#define ARCH_DEFINE_STATIC_CALL_RET0_TRAMP(name)	__PPC_SCT(name, "b 1f")
 
 #endif /* _ASM_POWERPC_STATIC_CALL_H */
diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile
index 06d2d1f78f71..a30d0d0f5499 100644
--- a/arch/powerpc/kernel/Makefile
+++ b/arch/powerpc/kernel/Makefile
@@ -128,8 +128,9 @@  extra-y				+= vmlinux.lds
 
 obj-$(CONFIG_RELOCATABLE)	+= reloc_$(BITS).o
 
-obj-$(CONFIG_PPC32)		+= entry_32.o setup_32.o early_32.o static_call.o
+obj-$(CONFIG_PPC32)		+= entry_32.o setup_32.o early_32.o
 obj-$(CONFIG_PPC64)		+= dma-iommu.o iommu.o
+obj-$(CONFIG_HAVE_STATIC_CALL)	+= static_call.o
 obj-$(CONFIG_KGDB)		+= kgdb.o
 obj-$(CONFIG_BOOTX_TEXT)	+= btext.o
 obj-$(CONFIG_SMP)		+= smp.o
diff --git a/arch/powerpc/kernel/static_call.c b/arch/powerpc/kernel/static_call.c
index 863a7aa24650..d2d9b496381d 100644
--- a/arch/powerpc/kernel/static_call.c
+++ b/arch/powerpc/kernel/static_call.c
@@ -1,33 +1,111 @@ 
 // SPDX-License-Identifier: GPL-2.0
+#include <asm/code-patching.h>
+#include <linux/export.h>
+#include <linux/kconfig.h>
+#include <linux/kernel.h>
 #include <linux/memory.h>
 #include <linux/static_call.h>
+#include <linux/syscalls.h>
 
-#include <asm/code-patching.h>
+static void* ppc_function_toc(u32 *func) {
+	if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2)) {
+		u32 insn1 = *func;
+		u32 insn2 = *(func+1);
+		u64 si1 = sign_extend64((insn1 & OP_SI_MASK) << 16, 31);
+		u64 si2 = sign_extend64(insn2 & OP_SI_MASK, 15);
+		u64 addr = ((u64) func + si1) + si2;
+
+		if ((((insn1 & OP_RT_RA_MASK) == ADDIS_R2_R12) ||
+		     ((insn1 & OP_RT_RA_MASK) == LIS_R2)) &&
+		    ((insn2 & OP_RT_RA_MASK) == ADDI_R2_R2))
+			return (void*)addr;
+	}
+
+	return NULL;
+}
+
+static bool shares_toc(void *func1, void *func2) {
+	if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2)) {
+		void* func1_toc;
+		void* func2_toc;
+
+		if (func1 == NULL || func2 == NULL)
+			return false;
+
+		/* Assume the kernel only uses a single TOC */
+		if (core_kernel_text((unsigned long)func1) &&
+		    core_kernel_text((unsigned long)func2))
+			return true;
+
+		/* Fall back to calculating the TOC from common patterns
+		 * if modules are involved
+		 */
+		func1_toc = ppc_function_toc(func1);
+		func2_toc = ppc_function_toc(func2);
+		return func1_toc != NULL && func2_toc != NULL && (func1_toc == func2_toc);
+	}
+
+	return true;
+}
+
+static unsigned long get_inst_addr(unsigned long tramp) {
+	return tramp + (core_kernel_text(tramp) ? PPC_SCT_INST_KERNEL : PPC_SCT_INST_MODULE);
+}
+
+static unsigned long get_ret0_addr(unsigned long tramp) {
+	return tramp + (core_kernel_text(tramp) ? PPC_SCT_RET0_KERNEL : PPC_SCT_RET0_MODULE);
+}
+
+static unsigned long get_data_addr(unsigned long tramp) {
+	return tramp + (core_kernel_text(tramp) ? PPC_SCT_DATA_KERNEL : PPC_SCT_DATA_MODULE);
+}
 
 void arch_static_call_transform(void *site, void *tramp, void *func, bool tail)
 {
 	int err;
 	bool is_ret0 = (func == __static_call_return0);
-	unsigned long target = (unsigned long)(is_ret0 ? tramp + PPC_SCT_RET0 : func);
-	bool is_short = is_offset_in_branch_range((long)target - (long)tramp);
+	unsigned long target = is_ret0
+		? get_ret0_addr((unsigned long)tramp)
+		: (unsigned long)func;
+	bool is_short;
+	void* tramp_inst;
 
 	if (!tramp)
 		return;
 
+	tramp_inst = (void*)get_inst_addr((unsigned long)tramp);
+
+	if (is_ret0) {
+		is_short = true;
+	} else if (shares_toc(tramp, (void*)target)) {
+		/* Confirm that the local entry point is in range */
+		is_short = is_offset_in_branch_range(
+			(long)ppc_function_entry((void*)target) - (long)tramp_inst);
+	} else {
+		/* Combine out-of-range with not sharing a TOC. Though it's possible an
+		 * out-of-range target shares a TOC, handling this separately complicates
+		 * the trampoline. It's simpler to always use the global entry point
+		 * in this case.
+		 */
+		is_short = false;
+	}
+
 	mutex_lock(&text_mutex);
 
 	if (func && !is_short) {
-		err = patch_instruction(tramp + PPC_SCT_DATA, ppc_inst(target));
+		err = patch_text_data(
+			(void*)get_data_addr((unsigned long)tramp), &target, sizeof(target));
 		if (err)
 			goto out;
 	}
 
 	if (!func)
-		err = patch_instruction(tramp, ppc_inst(PPC_RAW_BLR()));
+		err = patch_instruction(tramp_inst, ppc_inst(PPC_RAW_BLR()));
 	else if (is_short)
-		err = patch_branch(tramp, target, 0);
+		err = patch_branch(tramp_inst, ppc_function_entry((void*)target), 0);
 	else
-		err = patch_instruction(tramp, ppc_inst(PPC_RAW_NOP()));
+		err = patch_instruction(tramp_inst, ppc_inst(PPC_RAW_NOP()));
+
 out:
 	mutex_unlock(&text_mutex);