| Message ID | 5d8154cf64f6c0d7b09dbac44b763c97dcb408fe.1701967183.git.szabolcs.nagy@arm.com |
|---|---|
| State | New |
| Headers | show |
| Series | libgcc: aarch64: Add SME runtime | expand |
Szabolcs Nagy <szabolcs.nagy@arm.com> writes: > The call ABI for SME (Scalable Matrix Extension) requires a number of > helper routines which are added to libgcc so they are tied to the > compiler version instead of the libc version. See > https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst#sme-support-routines > > The routines are in shared libgcc and static libgcc eh, even though > they are not related to exception handling. This is to avoid linking > a copy of the routines into dynamic linked binaries, because TPIDR2_EL0 > block can be extended in the future which is better to handle in a > single place per process. > > The support routines have to decide if SME is accessible or not. Linux > tells userspace if SME is accessible via AT_HWCAP2, otherwise a new > __aarch64_sme_accessible symbol was introduced that a libc can define. > Due to libgcc and libc build order, the symbol availability cannot be > checked so for __aarch64_sme_accessible an unistd.h feature test macro > is used while such detection mechanism is not available for __getauxval > so we rely on configure checks based on the target triplet. > > Asm helper code is added to make writing the routines easier. > > libgcc/ChangeLog: > > * config/aarch64/t-aarch64: Add sources to the build. > * config/aarch64/__aarch64_have_sme.c: New file. > * config/aarch64/__arm_sme_state.S: New file. > * config/aarch64/__arm_tpidr2_restore.S: New file. > * config/aarch64/__arm_tpidr2_save.S: New file. > * config/aarch64/__arm_za_disable.S: New file. > * config/aarch64/aarch64-asm.h: New file. > * config/aarch64/libgcc-sme.ver: New file. > --- > libgcc/config/aarch64/__aarch64_have_sme.c | 71 +++++++++++++ > libgcc/config/aarch64/__arm_sme_state.S | 55 ++++++++++ > libgcc/config/aarch64/__arm_tpidr2_restore.S | 89 ++++++++++++++++ > libgcc/config/aarch64/__arm_tpidr2_save.S | 101 +++++++++++++++++++ > libgcc/config/aarch64/__arm_za_disable.S | 66 ++++++++++++ > libgcc/config/aarch64/aarch64-asm.h | 98 ++++++++++++++++++ > libgcc/config/aarch64/libgcc-sme.ver | 24 +++++ > libgcc/config/aarch64/t-aarch64 | 10 ++ > 8 files changed, 514 insertions(+) > create mode 100644 libgcc/config/aarch64/__aarch64_have_sme.c > create mode 100644 libgcc/config/aarch64/__arm_sme_state.S > create mode 100644 libgcc/config/aarch64/__arm_tpidr2_restore.S > create mode 100644 libgcc/config/aarch64/__arm_tpidr2_save.S > create mode 100644 libgcc/config/aarch64/__arm_za_disable.S > create mode 100644 libgcc/config/aarch64/aarch64-asm.h > create mode 100644 libgcc/config/aarch64/libgcc-sme.ver > > diff --git a/libgcc/config/aarch64/__aarch64_have_sme.c b/libgcc/config/aarch64/__aarch64_have_sme.c > new file mode 100644 > index 00000000000..2dc6be63ce9 > --- /dev/null > +++ b/libgcc/config/aarch64/__aarch64_have_sme.c > @@ -0,0 +1,71 @@ > +/* Initializer for SME support. > + Copyright (C) 2023 Free Software Foundation, Inc. > + > + This file is part of GCC. > + > + GCC is free software; you can redistribute it and/or modify it > + under the terms of the GNU General Public License as published > + by the Free Software Foundation; either version 3, or (at your > + option) any later version. > + > + GCC is distributed in the hope that it will be useful, but WITHOUT > + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY > + or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public > + License for more details. > + > + Under Section 7 of GPL version 3, you are granted additional > + permissions described in the GCC Runtime Library Exception, version > + 3.1, as published by the Free Software Foundation. > + > + You should have received a copy of the GNU General Public License and > + a copy of the GCC Runtime Library Exception along with this program; > + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see > + <http://www.gnu.org/licenses/>. */ > + > +#include "auto-target.h" > +#include <unistd.h> > + > +#if __ARM_FEATURE_SME > +/* Avoid runtime SME detection if libgcc is built with SME. */ > +# define HAVE_SME_CONST const > +# define HAVE_SME_VALUE 1 > +#elif HAVE___GETAUXVAL > +/* SME access detection on Linux. */ > +# define HAVE_SME_CONST > +# define HAVE_SME_VALUE 0 > +# define HAVE_SME_CTOR sme_accessible () > + > +# define AT_HWCAP2 26 > +# define HWCAP2_SME (1 << 23) > +unsigned long int __getauxval (unsigned long int); > + > +static _Bool > +sme_accessible (void) > +{ > + unsigned long hwcap2 = __getauxval (AT_HWCAP2); > + return (hwcap2 & HWCAP2_SME) != 0; > +} > +#elif __LIBC___AARCH64_SME_ACCESSIBLE > +/* Alternative SME access detection. */ > +# define HAVE_SME_CONST > +# define HAVE_SME_VALUE 0 > +# define HAVE_SME_CTOR __aarch64_sme_accessible () > +_Bool __aarch64_sme_accessible (void); > +#else > +# define HAVE_SME_CONST const > +# define HAVE_SME_VALUE 0 > +#endif > + > +/* Define the symbol gating SME support in libgcc. */ > +HAVE_SME_CONST _Bool __aarch64_have_sme > + __attribute__((visibility("hidden"), nocommon)) = HAVE_SME_VALUE; > + > +#ifdef HAVE_SME_CTOR > +/* Use a higher priority to ensure it runs before user constructors > + with priority 100. */ > +static void __attribute__((constructor (90))) > +init_have_sme (void) > +{ > + __aarch64_have_sme = HAVE_SME_CTOR; > +} > +#endif > diff --git a/libgcc/config/aarch64/__arm_sme_state.S b/libgcc/config/aarch64/__arm_sme_state.S > new file mode 100644 > index 00000000000..c4e16cac00d > --- /dev/null > +++ b/libgcc/config/aarch64/__arm_sme_state.S > @@ -0,0 +1,55 @@ > +/* Support routine for SME. > + Copyright (C) 2023 Free Software Foundation, Inc. > + > + This file is part of GCC. > + > + GCC is free software; you can redistribute it and/or modify it > + under the terms of the GNU General Public License as published > + by the Free Software Foundation; either version 3, or (at your > + option) any later version. > + > + GCC is distributed in the hope that it will be useful, but WITHOUT > + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY > + or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public > + License for more details. > + > + Under Section 7 of GPL version 3, you are granted additional > + permissions described in the GCC Runtime Library Exception, version > + 3.1, as published by the Free Software Foundation. > + > + You should have received a copy of the GNU General Public License and > + a copy of the GCC Runtime Library Exception along with this program; > + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see > + <http://www.gnu.org/licenses/>. */ > + > +#include "aarch64-asm.h" > + > +/* Query SME state. Call ABI: > + - Private ZA, streaming-compatible. > + - x2-x15, x19-x29, sp and fp regs are call preserved. > + - Takes no argument. > + - Returns SME state in x0 and TPIDR2_EL0 in x1. */ > + > +.hidden __aarch64_have_sme > + > +variant_pcs (__arm_sme_state) > + > +ENTRY (__arm_sme_state) > + /* Check if SME is available. */ > + adrp x1, __aarch64_have_sme > + ldrb w1, [x1, :lo12:__aarch64_have_sme] > + cbz w1, L(nosme) > + > + /* Expose the bottom 2 bits of svcr (SM, ZA) in x0 and set the > + top 2 bits indicating that SME and TPIDR2_EL0 are available. */ > + .inst 0xd53b4240 /* mrs x0, svcr */ > + .inst 0xd53bd0a1 /* mrs x1, tpidr2_el0 */ > + and x0, x0, 3 > + orr x0, x0, 0xc000000000000000 > + ret > + > +L(nosme): > + mov x0, 0 > + mov x1, 0 > + ret > +END (__arm_sme_state) > diff --git a/libgcc/config/aarch64/__arm_tpidr2_restore.S b/libgcc/config/aarch64/__arm_tpidr2_restore.S > new file mode 100644 > index 00000000000..4569d04a2d0 > --- /dev/null > +++ b/libgcc/config/aarch64/__arm_tpidr2_restore.S > @@ -0,0 +1,89 @@ > +/* Support routine for SME. > + Copyright (C) 2023 Free Software Foundation, Inc. > + > + This file is part of GCC. > + > + GCC is free software; you can redistribute it and/or modify it > + under the terms of the GNU General Public License as published > + by the Free Software Foundation; either version 3, or (at your > + option) any later version. > + > + GCC is distributed in the hope that it will be useful, but WITHOUT > + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY > + or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public > + License for more details. > + > + Under Section 7 of GPL version 3, you are granted additional > + permissions described in the GCC Runtime Library Exception, version > + 3.1, as published by the Free Software Foundation. > + > + You should have received a copy of the GNU General Public License and > + a copy of the GCC Runtime Library Exception along with this program; > + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see > + <http://www.gnu.org/licenses/>. */ > + > +#include "aarch64-asm.h" > + > +/* Used for lazy ZA restore. Call ABI: > + - Shared ZA, streaming-compatible. > + - x0 is a pointer to a TPIDR2 block. > + - x0-x13, x19-x29, sp and fp regs are call preserved. > + - Does not return a value. > + - Can abort on failure (then registers are not preserved). */ > + > +variant_pcs (__arm_tpidr2_restore) > + > +ENTRY (__arm_tpidr2_restore) > + .inst 0xd53bd0ae /* mrs x14, tpidr2_el0 */ > + cbnz x14, L(fail) > + > + /* check reserved bytes. */ > + ldrh w15, [x0, 10] > + ldr w16, [x0, 12] > + orr w15, w15, w16 > + cbnz w15, L(fail) > + > + ldr x16, [x0] > + cbz x16, L(end) > + ldrh w17, [x0, 8] > + cbz w17, L(end) > + > + /* x0: blk, x14: 0, x15: 0, > + x16: za_save_buffer, x17: num_za_save_slices. */ > + > +L(restore_loop): > + .inst 0xe1006200 /* ldr za[w15, 0], [x16] */ > + .inst 0xe1006201 /* ldr za[w15, 1], [x16, 1, mul vl] */ > + .inst 0xe1006202 /* ldr za[w15, 2], [x16, 2, mul vl] */ > + .inst 0xe1006203 /* ldr za[w15, 3], [x16, 3, mul vl] */ > + .inst 0xe1006204 /* ldr za[w15, 4], [x16, 4, mul vl] */ > + .inst 0xe1006205 /* ldr za[w15, 5], [x16, 5, mul vl] */ > + .inst 0xe1006206 /* ldr za[w15, 6], [x16, 6, mul vl] */ > + .inst 0xe1006207 /* ldr za[w15, 7], [x16, 7, mul vl] */ > + .inst 0xe1006208 /* ldr za[w15, 8], [x16, 8, mul vl] */ > + .inst 0xe1006209 /* ldr za[w15, 9], [x16, 9, mul vl] */ > + .inst 0xe100620a /* ldr za[w15, 10], [x16, 10, mul vl] */ > + .inst 0xe100620b /* ldr za[w15, 11], [x16, 11, mul vl] */ > + .inst 0xe100620c /* ldr za[w15, 12], [x16, 12, mul vl] */ > + .inst 0xe100620d /* ldr za[w15, 13], [x16, 13, mul vl] */ > + .inst 0xe100620e /* ldr za[w15, 14], [x16, 14, mul vl] */ > + .inst 0xe100620f /* ldr za[w15, 15], [x16, 15, mul vl] */ > + add w15, w15, 16 > + .inst 0x04305a10 /* addsvl x16, x16, 16 */ > + cmp w17, w15 > + bhi L(restore_loop) > +L(end): > + ret > +L(fail): > + PACIASP > + stp x29, x30, [sp, -32]! > + .cfi_adjust_cfa_offset 32 > + .cfi_rel_offset x29, 0 > + .cfi_rel_offset x30, 8 > + mov x29, sp > + .inst 0x04e0e3f0 /* cntd x16 */ > + str x16, [sp, 16] > + .cfi_rel_offset 46, 16 > + .inst 0xd503467f /* smstop */ > + bl abort > +END (__arm_tpidr2_restore) > diff --git a/libgcc/config/aarch64/__arm_tpidr2_save.S b/libgcc/config/aarch64/__arm_tpidr2_save.S > new file mode 100644 > index 00000000000..879cf798079 > --- /dev/null > +++ b/libgcc/config/aarch64/__arm_tpidr2_save.S > @@ -0,0 +1,101 @@ > +/* Support routine for SME. > + Copyright (C) 2023 Free Software Foundation, Inc. > + > + This file is part of GCC. > + > + GCC is free software; you can redistribute it and/or modify it > + under the terms of the GNU General Public License as published > + by the Free Software Foundation; either version 3, or (at your > + option) any later version. > + > + GCC is distributed in the hope that it will be useful, but WITHOUT > + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY > + or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public > + License for more details. > + > + Under Section 7 of GPL version 3, you are granted additional > + permissions described in the GCC Runtime Library Exception, version > + 3.1, as published by the Free Software Foundation. > + > + You should have received a copy of the GNU General Public License and > + a copy of the GCC Runtime Library Exception along with this program; > + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see > + <http://www.gnu.org/licenses/>. */ > + > +#include "aarch64-asm.h" > + > +/* Used for lazy ZA save. Call ABI: > + - Private ZA, streaming-compatible. > + - x0-x13, x19-x29, sp and fp regs are call preserved. > + - Takes no argument. > + - Does not return a value. > + - Can abort on failure (then registers are not preserved). */ > + > +.hidden __aarch64_have_sme > + > +variant_pcs (__arm_tpidr2_save) > + > +ENTRY (__arm_tpidr2_save) > + /* Check if SME is available. */ > + adrp x14, __aarch64_have_sme > + ldrb w14, [x14, :lo12:__aarch64_have_sme] > + cbz w14, L(end) > + > + .inst 0xd53bd0ae /* mrs x14, tpidr2_el0 */ > + cbz x14, L(end) > + > + /* check reserved bytes. */ > + ldrh w15, [x14, 10] > + ldr w16, [x14, 12] > + orr w15, w15, w16 > + cbnz w15, L(fail) > + > + ldr x16, [x14] > + cbz x16, L(end) > + ldrh w17, [x14, 8] > + cbz w17, L(end) > + > + /* x14: tpidr2, x15: 0, > + x16: za_save_buffer, x17: num_za_save_slices. */ > + > +L(save_loop): > + .inst 0xe1206200 /* str za[w15, 0], [x16] */ > + .inst 0xe1206201 /* str za[w15, 1], [x16, 1, mul vl] */ > + .inst 0xe1206202 /* str za[w15, 2], [x16, 2, mul vl] */ > + .inst 0xe1206203 /* str za[w15, 3], [x16, 3, mul vl] */ > + .inst 0xe1206204 /* str za[w15, 4], [x16, 4, mul vl] */ > + .inst 0xe1206205 /* str za[w15, 5], [x16, 5, mul vl] */ > + .inst 0xe1206206 /* str za[w15, 6], [x16, 6, mul vl] */ > + .inst 0xe1206207 /* str za[w15, 7], [x16, 7, mul vl] */ > + .inst 0xe1206208 /* str za[w15, 8], [x16, 8, mul vl] */ > + .inst 0xe1206209 /* str za[w15, 9], [x16, 9, mul vl] */ > + .inst 0xe120620a /* str za[w15, 10], [x16, 10, mul vl] */ > + .inst 0xe120620b /* str za[w15, 11], [x16, 11, mul vl] */ > + .inst 0xe120620c /* str za[w15, 12], [x16, 12, mul vl] */ > + .inst 0xe120620d /* str za[w15, 13], [x16, 13, mul vl] */ > + .inst 0xe120620e /* str za[w15, 14], [x16, 14, mul vl] */ > + .inst 0xe120620f /* str za[w15, 15], [x16, 15, mul vl] */ > + add w15, w15, 16 > + .inst 0x04305a10 /* addsvl x16, x16, 16 */ > + cmp w17, w15 > + bhi L(save_loop) > +L(end): > + ret > +L(fail): > + PACIASP > + stp x29, x30, [sp, -32]! > + .cfi_adjust_cfa_offset 32 > + .cfi_rel_offset x29, 0 > + .cfi_rel_offset x30, 8 > + mov x29, sp > + .inst 0x04e0e3f0 /* cntd x16 */ > + str x16, [sp, 16] > + .cfi_rel_offset 46, 16 > + .inst 0xd503467f /* smstop */ > + bl abort > +END (__arm_tpidr2_save) > + > +/* Hidden alias used by __arm_za_disable. */ > +.global __libgcc_arm_tpidr2_save > +.hidden __libgcc_arm_tpidr2_save > +.set __libgcc_arm_tpidr2_save, __arm_tpidr2_save > diff --git a/libgcc/config/aarch64/__arm_za_disable.S b/libgcc/config/aarch64/__arm_za_disable.S > new file mode 100644 > index 00000000000..7a888a98d49 > --- /dev/null > +++ b/libgcc/config/aarch64/__arm_za_disable.S > @@ -0,0 +1,66 @@ > +/* Support routine for SME. > + Copyright (C) 2023 Free Software Foundation, Inc. > + > + This file is part of GCC. > + > + GCC is free software; you can redistribute it and/or modify it > + under the terms of the GNU General Public License as published > + by the Free Software Foundation; either version 3, or (at your > + option) any later version. > + > + GCC is distributed in the hope that it will be useful, but WITHOUT > + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY > + or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public > + License for more details. > + > + Under Section 7 of GPL version 3, you are granted additional > + permissions described in the GCC Runtime Library Exception, version > + 3.1, as published by the Free Software Foundation. > + > + You should have received a copy of the GNU General Public License and > + a copy of the GCC Runtime Library Exception along with this program; > + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see > + <http://www.gnu.org/licenses/>. */ > + > +#include "aarch64-asm.h" > + > +/* Disable ZA. Call ABI: > + - Private ZA, streaming-compatible. > + - x0-x13, x19-x29, sp and fp regs are call preserved. > + - Takes no argument. > + - Does not return a value. > + - Can abort on failure (then registers are not preserved). */ > + > +.hidden __aarch64_have_sme > + > +.hidden __libgcc_arm_tpidr2_save > + > +variant_pcs (__arm_za_disable) > + > +ENTRY (__arm_za_disable) > + /* Check if SME is available. */ > + adrp x14, __aarch64_have_sme > + ldrb w14, [x14, :lo12:__aarch64_have_sme] > + cbz w14, L(end) > + > + .inst 0xd53bd0ae /* mrs x14, tpidr2_el0 */ > + cbz x14, L(end) > + > + PACIASP > + stp x29, x30, [sp, -16]! > + .cfi_adjust_cfa_offset 16 > + .cfi_rel_offset x29, 0 > + .cfi_rel_offset x30, 8 > + mov x29, sp > + bl __libgcc_arm_tpidr2_save > + mov x14, 0 > + .inst 0xd51bd0ae /* msr tpidr2_el0, x14 */ Sorry, just noticed that this could use msr tpidr2_el0, xzr, without the preceding move. OK for the series with that change, or as-is if you prefer. Thanks, Richard > + .inst 0xd503447f /* smstop za */ > + ldp x29, x30, [sp], 16 > + .cfi_adjust_cfa_offset -16 > + .cfi_restore x29 > + .cfi_restore x30 > + AUTIASP > +L(end): > + ret > +END (__arm_za_disable) > diff --git a/libgcc/config/aarch64/aarch64-asm.h b/libgcc/config/aarch64/aarch64-asm.h > new file mode 100644 > index 00000000000..8969b06b09c > --- /dev/null > +++ b/libgcc/config/aarch64/aarch64-asm.h > @@ -0,0 +1,98 @@ > +/* AArch64 asm definitions. > + Copyright (C) 2023 Free Software Foundation, Inc. > + > + This file is part of GCC. > + > + GCC is free software; you can redistribute it and/or modify it > + under the terms of the GNU General Public License as published > + by the Free Software Foundation; either version 3, or (at your > + option) any later version. > + > + GCC is distributed in the hope that it will be useful, but WITHOUT > + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY > + or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public > + License for more details. > + > + Under Section 7 of GPL version 3, you are granted additional > + permissions described in the GCC Runtime Library Exception, version > + 3.1, as published by the Free Software Foundation. > + > + You should have received a copy of the GNU General Public License and > + a copy of the GCC Runtime Library Exception along with this program; > + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see > + <http://www.gnu.org/licenses/>. */ > + > +#include "auto-target.h" > + > +#define L(label) .L ## label > + > +/* Marking variant PCS symbol references is important for PLT calls > + otherwise it is for documenting the PCS in the symbol table. */ > +#ifdef HAVE_AS_VARIANT_PCS > +# define variant_pcs(name) .variant_pcs name > +#else > +# define variant_pcs(name) > +#endif > + > +/* GNU_PROPERTY_AARCH64_* macros from elf.h for use in asm code. */ > +#define FEATURE_1_AND 0xc0000000 > +#define FEATURE_1_BTI 1 > +#define FEATURE_1_PAC 2 > + > +/* Supported features based on the code generation options. */ > +#if defined(__ARM_FEATURE_BTI_DEFAULT) > +# define BTI_FLAG FEATURE_1_BTI > +# define BTI_C hint 34 > +#else > +# define BTI_FLAG 0 > +# define BTI_C > +#endif > + > +#if __ARM_FEATURE_PAC_DEFAULT & 3 > +# define PAC_FLAG FEATURE_1_PAC > +# define PACIASP hint 25; .cfi_window_save > +# define AUTIASP hint 29; .cfi_window_save > +#else > +# define PAC_FLAG 0 > +# define PACIASP > +# define AUTIASP > +#endif > + > +/* Add a NT_GNU_PROPERTY_TYPE_0 note. */ > +#define GNU_PROPERTY(type, value) \ > + .section .note.gnu.property, "a"; \ > + .p2align 3; \ > + .word 4; \ > + .word 16; \ > + .word 5; \ > + .asciz "GNU"; \ > + .word type; \ > + .word 4; \ > + .word value; \ > + .word 0; \ > + .previous > + > +#if defined(__linux__) || defined(__FreeBSD__) > +/* Do not require executable stack. */ > +.section .note.GNU-stack, "", %progbits > +.previous > + > +/* Add GNU property note if built with branch protection. */ > +# if (BTI_FLAG|PAC_FLAG) != 0 > +GNU_PROPERTY (FEATURE_1_AND, BTI_FLAG|PAC_FLAG) > +# endif > +#endif > + > +#define ENTRY_ALIGN(name, align) \ > + .global name; \ > + .type name,%function; \ > + .balign align; \ > + name: \ > + .cfi_startproc; \ > + BTI_C > + > +#define ENTRY(name) ENTRY_ALIGN(name, 16) > + > +#define END(name) \ > + .cfi_endproc; \ > + .size name, .-name > diff --git a/libgcc/config/aarch64/libgcc-sme.ver b/libgcc/config/aarch64/libgcc-sme.ver > new file mode 100644 > index 00000000000..da889c6c09e > --- /dev/null > +++ b/libgcc/config/aarch64/libgcc-sme.ver > @@ -0,0 +1,24 @@ > +# Copyright (C) 2023 Free Software Foundation, Inc. > +# > +# This file is part of GCC. > +# > +# GCC is free software; you can redistribute it and/or modify > +# it under the terms of the GNU General Public License as published by > +# the Free Software Foundation; either version 3, or (at your option) > +# any later version. > +# > +# GCC is distributed in the hope that it will be useful, > +# but WITHOUT ANY WARRANTY; without even the implied warranty of > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > +# GNU General Public License for more details. > +# > +# You should have received a copy of the GNU General Public License > +# along with GCC; see the file COPYING3. If not see > +# <http://www.gnu.org/licenses/>. > + > +GCC_14.0 { > + __arm_sme_state > + __arm_tpidr2_restore > + __arm_tpidr2_save > + __arm_za_disable > +} > diff --git a/libgcc/config/aarch64/t-aarch64 b/libgcc/config/aarch64/t-aarch64 > index a40b6241c86..7b852022a4d 100644 > --- a/libgcc/config/aarch64/t-aarch64 > +++ b/libgcc/config/aarch64/t-aarch64 > @@ -19,3 +19,13 @@ > # <http://www.gnu.org/licenses/>. > > LIB2ADD += $(srcdir)/config/aarch64/sync-cache.c > + > +# Add sme runtime to shared libgcc > +LIB2ADDEH += \ > + $(srcdir)/config/aarch64/__aarch64_have_sme.c \ > + $(srcdir)/config/aarch64/__arm_sme_state.S \ > + $(srcdir)/config/aarch64/__arm_tpidr2_restore.S \ > + $(srcdir)/config/aarch64/__arm_tpidr2_save.S \ > + $(srcdir)/config/aarch64/__arm_za_disable.S > + > +SHLIB_MAPFILES += $(srcdir)/config/aarch64/libgcc-sme.ver
The 12/07/2023 17:36, Richard Sandiford wrote: > Szabolcs Nagy <szabolcs.nagy@arm.com> writes: > > + > > +#include "auto-target.h" > > +#include <unistd.h> > > + sorry, this seems to fail when building --without-headers i will respin this, handling the 'inhibit_libc' case.
diff --git a/libgcc/config/aarch64/__aarch64_have_sme.c b/libgcc/config/aarch64/__aarch64_have_sme.c new file mode 100644 index 00000000000..2dc6be63ce9 --- /dev/null +++ b/libgcc/config/aarch64/__aarch64_have_sme.c @@ -0,0 +1,71 @@ +/* Initializer for SME support. + Copyright (C) 2023 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GCC is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public + License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + <http://www.gnu.org/licenses/>. */ + +#include "auto-target.h" +#include <unistd.h> + +#if __ARM_FEATURE_SME +/* Avoid runtime SME detection if libgcc is built with SME. */ +# define HAVE_SME_CONST const +# define HAVE_SME_VALUE 1 +#elif HAVE___GETAUXVAL +/* SME access detection on Linux. */ +# define HAVE_SME_CONST +# define HAVE_SME_VALUE 0 +# define HAVE_SME_CTOR sme_accessible () + +# define AT_HWCAP2 26 +# define HWCAP2_SME (1 << 23) +unsigned long int __getauxval (unsigned long int); + +static _Bool +sme_accessible (void) +{ + unsigned long hwcap2 = __getauxval (AT_HWCAP2); + return (hwcap2 & HWCAP2_SME) != 0; +} +#elif __LIBC___AARCH64_SME_ACCESSIBLE +/* Alternative SME access detection. */ +# define HAVE_SME_CONST +# define HAVE_SME_VALUE 0 +# define HAVE_SME_CTOR __aarch64_sme_accessible () +_Bool __aarch64_sme_accessible (void); +#else +# define HAVE_SME_CONST const +# define HAVE_SME_VALUE 0 +#endif + +/* Define the symbol gating SME support in libgcc. */ +HAVE_SME_CONST _Bool __aarch64_have_sme + __attribute__((visibility("hidden"), nocommon)) = HAVE_SME_VALUE; + +#ifdef HAVE_SME_CTOR +/* Use a higher priority to ensure it runs before user constructors + with priority 100. */ +static void __attribute__((constructor (90))) +init_have_sme (void) +{ + __aarch64_have_sme = HAVE_SME_CTOR; +} +#endif diff --git a/libgcc/config/aarch64/__arm_sme_state.S b/libgcc/config/aarch64/__arm_sme_state.S new file mode 100644 index 00000000000..c4e16cac00d --- /dev/null +++ b/libgcc/config/aarch64/__arm_sme_state.S @@ -0,0 +1,55 @@ +/* Support routine for SME. + Copyright (C) 2023 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GCC is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public + License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + <http://www.gnu.org/licenses/>. */ + +#include "aarch64-asm.h" + +/* Query SME state. Call ABI: + - Private ZA, streaming-compatible. + - x2-x15, x19-x29, sp and fp regs are call preserved. + - Takes no argument. + - Returns SME state in x0 and TPIDR2_EL0 in x1. */ + +.hidden __aarch64_have_sme + +variant_pcs (__arm_sme_state) + +ENTRY (__arm_sme_state) + /* Check if SME is available. */ + adrp x1, __aarch64_have_sme + ldrb w1, [x1, :lo12:__aarch64_have_sme] + cbz w1, L(nosme) + + /* Expose the bottom 2 bits of svcr (SM, ZA) in x0 and set the + top 2 bits indicating that SME and TPIDR2_EL0 are available. */ + .inst 0xd53b4240 /* mrs x0, svcr */ + .inst 0xd53bd0a1 /* mrs x1, tpidr2_el0 */ + and x0, x0, 3 + orr x0, x0, 0xc000000000000000 + ret + +L(nosme): + mov x0, 0 + mov x1, 0 + ret +END (__arm_sme_state) diff --git a/libgcc/config/aarch64/__arm_tpidr2_restore.S b/libgcc/config/aarch64/__arm_tpidr2_restore.S new file mode 100644 index 00000000000..4569d04a2d0 --- /dev/null +++ b/libgcc/config/aarch64/__arm_tpidr2_restore.S @@ -0,0 +1,89 @@ +/* Support routine for SME. + Copyright (C) 2023 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GCC is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public + License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + <http://www.gnu.org/licenses/>. */ + +#include "aarch64-asm.h" + +/* Used for lazy ZA restore. Call ABI: + - Shared ZA, streaming-compatible. + - x0 is a pointer to a TPIDR2 block. + - x0-x13, x19-x29, sp and fp regs are call preserved. + - Does not return a value. + - Can abort on failure (then registers are not preserved). */ + +variant_pcs (__arm_tpidr2_restore) + +ENTRY (__arm_tpidr2_restore) + .inst 0xd53bd0ae /* mrs x14, tpidr2_el0 */ + cbnz x14, L(fail) + + /* check reserved bytes. */ + ldrh w15, [x0, 10] + ldr w16, [x0, 12] + orr w15, w15, w16 + cbnz w15, L(fail) + + ldr x16, [x0] + cbz x16, L(end) + ldrh w17, [x0, 8] + cbz w17, L(end) + + /* x0: blk, x14: 0, x15: 0, + x16: za_save_buffer, x17: num_za_save_slices. */ + +L(restore_loop): + .inst 0xe1006200 /* ldr za[w15, 0], [x16] */ + .inst 0xe1006201 /* ldr za[w15, 1], [x16, 1, mul vl] */ + .inst 0xe1006202 /* ldr za[w15, 2], [x16, 2, mul vl] */ + .inst 0xe1006203 /* ldr za[w15, 3], [x16, 3, mul vl] */ + .inst 0xe1006204 /* ldr za[w15, 4], [x16, 4, mul vl] */ + .inst 0xe1006205 /* ldr za[w15, 5], [x16, 5, mul vl] */ + .inst 0xe1006206 /* ldr za[w15, 6], [x16, 6, mul vl] */ + .inst 0xe1006207 /* ldr za[w15, 7], [x16, 7, mul vl] */ + .inst 0xe1006208 /* ldr za[w15, 8], [x16, 8, mul vl] */ + .inst 0xe1006209 /* ldr za[w15, 9], [x16, 9, mul vl] */ + .inst 0xe100620a /* ldr za[w15, 10], [x16, 10, mul vl] */ + .inst 0xe100620b /* ldr za[w15, 11], [x16, 11, mul vl] */ + .inst 0xe100620c /* ldr za[w15, 12], [x16, 12, mul vl] */ + .inst 0xe100620d /* ldr za[w15, 13], [x16, 13, mul vl] */ + .inst 0xe100620e /* ldr za[w15, 14], [x16, 14, mul vl] */ + .inst 0xe100620f /* ldr za[w15, 15], [x16, 15, mul vl] */ + add w15, w15, 16 + .inst 0x04305a10 /* addsvl x16, x16, 16 */ + cmp w17, w15 + bhi L(restore_loop) +L(end): + ret +L(fail): + PACIASP + stp x29, x30, [sp, -32]! + .cfi_adjust_cfa_offset 32 + .cfi_rel_offset x29, 0 + .cfi_rel_offset x30, 8 + mov x29, sp + .inst 0x04e0e3f0 /* cntd x16 */ + str x16, [sp, 16] + .cfi_rel_offset 46, 16 + .inst 0xd503467f /* smstop */ + bl abort +END (__arm_tpidr2_restore) diff --git a/libgcc/config/aarch64/__arm_tpidr2_save.S b/libgcc/config/aarch64/__arm_tpidr2_save.S new file mode 100644 index 00000000000..879cf798079 --- /dev/null +++ b/libgcc/config/aarch64/__arm_tpidr2_save.S @@ -0,0 +1,101 @@ +/* Support routine for SME. + Copyright (C) 2023 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GCC is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public + License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + <http://www.gnu.org/licenses/>. */ + +#include "aarch64-asm.h" + +/* Used for lazy ZA save. Call ABI: + - Private ZA, streaming-compatible. + - x0-x13, x19-x29, sp and fp regs are call preserved. + - Takes no argument. + - Does not return a value. + - Can abort on failure (then registers are not preserved). */ + +.hidden __aarch64_have_sme + +variant_pcs (__arm_tpidr2_save) + +ENTRY (__arm_tpidr2_save) + /* Check if SME is available. */ + adrp x14, __aarch64_have_sme + ldrb w14, [x14, :lo12:__aarch64_have_sme] + cbz w14, L(end) + + .inst 0xd53bd0ae /* mrs x14, tpidr2_el0 */ + cbz x14, L(end) + + /* check reserved bytes. */ + ldrh w15, [x14, 10] + ldr w16, [x14, 12] + orr w15, w15, w16 + cbnz w15, L(fail) + + ldr x16, [x14] + cbz x16, L(end) + ldrh w17, [x14, 8] + cbz w17, L(end) + + /* x14: tpidr2, x15: 0, + x16: za_save_buffer, x17: num_za_save_slices. */ + +L(save_loop): + .inst 0xe1206200 /* str za[w15, 0], [x16] */ + .inst 0xe1206201 /* str za[w15, 1], [x16, 1, mul vl] */ + .inst 0xe1206202 /* str za[w15, 2], [x16, 2, mul vl] */ + .inst 0xe1206203 /* str za[w15, 3], [x16, 3, mul vl] */ + .inst 0xe1206204 /* str za[w15, 4], [x16, 4, mul vl] */ + .inst 0xe1206205 /* str za[w15, 5], [x16, 5, mul vl] */ + .inst 0xe1206206 /* str za[w15, 6], [x16, 6, mul vl] */ + .inst 0xe1206207 /* str za[w15, 7], [x16, 7, mul vl] */ + .inst 0xe1206208 /* str za[w15, 8], [x16, 8, mul vl] */ + .inst 0xe1206209 /* str za[w15, 9], [x16, 9, mul vl] */ + .inst 0xe120620a /* str za[w15, 10], [x16, 10, mul vl] */ + .inst 0xe120620b /* str za[w15, 11], [x16, 11, mul vl] */ + .inst 0xe120620c /* str za[w15, 12], [x16, 12, mul vl] */ + .inst 0xe120620d /* str za[w15, 13], [x16, 13, mul vl] */ + .inst 0xe120620e /* str za[w15, 14], [x16, 14, mul vl] */ + .inst 0xe120620f /* str za[w15, 15], [x16, 15, mul vl] */ + add w15, w15, 16 + .inst 0x04305a10 /* addsvl x16, x16, 16 */ + cmp w17, w15 + bhi L(save_loop) +L(end): + ret +L(fail): + PACIASP + stp x29, x30, [sp, -32]! + .cfi_adjust_cfa_offset 32 + .cfi_rel_offset x29, 0 + .cfi_rel_offset x30, 8 + mov x29, sp + .inst 0x04e0e3f0 /* cntd x16 */ + str x16, [sp, 16] + .cfi_rel_offset 46, 16 + .inst 0xd503467f /* smstop */ + bl abort +END (__arm_tpidr2_save) + +/* Hidden alias used by __arm_za_disable. */ +.global __libgcc_arm_tpidr2_save +.hidden __libgcc_arm_tpidr2_save +.set __libgcc_arm_tpidr2_save, __arm_tpidr2_save diff --git a/libgcc/config/aarch64/__arm_za_disable.S b/libgcc/config/aarch64/__arm_za_disable.S new file mode 100644 index 00000000000..7a888a98d49 --- /dev/null +++ b/libgcc/config/aarch64/__arm_za_disable.S @@ -0,0 +1,66 @@ +/* Support routine for SME. + Copyright (C) 2023 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GCC is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public + License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + <http://www.gnu.org/licenses/>. */ + +#include "aarch64-asm.h" + +/* Disable ZA. Call ABI: + - Private ZA, streaming-compatible. + - x0-x13, x19-x29, sp and fp regs are call preserved. + - Takes no argument. + - Does not return a value. + - Can abort on failure (then registers are not preserved). */ + +.hidden __aarch64_have_sme + +.hidden __libgcc_arm_tpidr2_save + +variant_pcs (__arm_za_disable) + +ENTRY (__arm_za_disable) + /* Check if SME is available. */ + adrp x14, __aarch64_have_sme + ldrb w14, [x14, :lo12:__aarch64_have_sme] + cbz w14, L(end) + + .inst 0xd53bd0ae /* mrs x14, tpidr2_el0 */ + cbz x14, L(end) + + PACIASP + stp x29, x30, [sp, -16]! + .cfi_adjust_cfa_offset 16 + .cfi_rel_offset x29, 0 + .cfi_rel_offset x30, 8 + mov x29, sp + bl __libgcc_arm_tpidr2_save + mov x14, 0 + .inst 0xd51bd0ae /* msr tpidr2_el0, x14 */ + .inst 0xd503447f /* smstop za */ + ldp x29, x30, [sp], 16 + .cfi_adjust_cfa_offset -16 + .cfi_restore x29 + .cfi_restore x30 + AUTIASP +L(end): + ret +END (__arm_za_disable) diff --git a/libgcc/config/aarch64/aarch64-asm.h b/libgcc/config/aarch64/aarch64-asm.h new file mode 100644 index 00000000000..8969b06b09c --- /dev/null +++ b/libgcc/config/aarch64/aarch64-asm.h @@ -0,0 +1,98 @@ +/* AArch64 asm definitions. + Copyright (C) 2023 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GCC is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public + License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + <http://www.gnu.org/licenses/>. */ + +#include "auto-target.h" + +#define L(label) .L ## label + +/* Marking variant PCS symbol references is important for PLT calls + otherwise it is for documenting the PCS in the symbol table. */ +#ifdef HAVE_AS_VARIANT_PCS +# define variant_pcs(name) .variant_pcs name +#else +# define variant_pcs(name) +#endif + +/* GNU_PROPERTY_AARCH64_* macros from elf.h for use in asm code. */ +#define FEATURE_1_AND 0xc0000000 +#define FEATURE_1_BTI 1 +#define FEATURE_1_PAC 2 + +/* Supported features based on the code generation options. */ +#if defined(__ARM_FEATURE_BTI_DEFAULT) +# define BTI_FLAG FEATURE_1_BTI +# define BTI_C hint 34 +#else +# define BTI_FLAG 0 +# define BTI_C +#endif + +#if __ARM_FEATURE_PAC_DEFAULT & 3 +# define PAC_FLAG FEATURE_1_PAC +# define PACIASP hint 25; .cfi_window_save +# define AUTIASP hint 29; .cfi_window_save +#else +# define PAC_FLAG 0 +# define PACIASP +# define AUTIASP +#endif + +/* Add a NT_GNU_PROPERTY_TYPE_0 note. */ +#define GNU_PROPERTY(type, value) \ + .section .note.gnu.property, "a"; \ + .p2align 3; \ + .word 4; \ + .word 16; \ + .word 5; \ + .asciz "GNU"; \ + .word type; \ + .word 4; \ + .word value; \ + .word 0; \ + .previous + +#if defined(__linux__) || defined(__FreeBSD__) +/* Do not require executable stack. */ +.section .note.GNU-stack, "", %progbits +.previous + +/* Add GNU property note if built with branch protection. */ +# if (BTI_FLAG|PAC_FLAG) != 0 +GNU_PROPERTY (FEATURE_1_AND, BTI_FLAG|PAC_FLAG) +# endif +#endif + +#define ENTRY_ALIGN(name, align) \ + .global name; \ + .type name,%function; \ + .balign align; \ + name: \ + .cfi_startproc; \ + BTI_C + +#define ENTRY(name) ENTRY_ALIGN(name, 16) + +#define END(name) \ + .cfi_endproc; \ + .size name, .-name diff --git a/libgcc/config/aarch64/libgcc-sme.ver b/libgcc/config/aarch64/libgcc-sme.ver new file mode 100644 index 00000000000..da889c6c09e --- /dev/null +++ b/libgcc/config/aarch64/libgcc-sme.ver @@ -0,0 +1,24 @@ +# Copyright (C) 2023 Free Software Foundation, Inc. +# +# This file is part of GCC. +# +# GCC is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# GCC is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with GCC; see the file COPYING3. If not see +# <http://www.gnu.org/licenses/>. + +GCC_14.0 { + __arm_sme_state + __arm_tpidr2_restore + __arm_tpidr2_save + __arm_za_disable +} diff --git a/libgcc/config/aarch64/t-aarch64 b/libgcc/config/aarch64/t-aarch64 index a40b6241c86..7b852022a4d 100644 --- a/libgcc/config/aarch64/t-aarch64 +++ b/libgcc/config/aarch64/t-aarch64 @@ -19,3 +19,13 @@ # <http://www.gnu.org/licenses/>. LIB2ADD += $(srcdir)/config/aarch64/sync-cache.c + +# Add sme runtime to shared libgcc +LIB2ADDEH += \ + $(srcdir)/config/aarch64/__aarch64_have_sme.c \ + $(srcdir)/config/aarch64/__arm_sme_state.S \ + $(srcdir)/config/aarch64/__arm_tpidr2_restore.S \ + $(srcdir)/config/aarch64/__arm_tpidr2_save.S \ + $(srcdir)/config/aarch64/__arm_za_disable.S + +SHLIB_MAPFILES += $(srcdir)/config/aarch64/libgcc-sme.ver