@@ -25,6 +25,7 @@
#endif
/* There is some commonality. */
+#include <sysdeps/unix/sysv/linux/sysdep.h>
#include <sysdeps/unix/alpha/sysdep.h>
#include <tls.h>
@@ -21,6 +21,7 @@
#define _LINUX_ARM_SYSDEP_H 1
/* There is some commonality. */
+#include <sysdeps/unix/sysv/linux/sysdep.h>
#include <sysdeps/unix/arm/sysdep.h>
/* Defines RTLD_PRIVATE_ERRNO and USE_DL_SYSINFO. */
@@ -19,6 +19,7 @@
#include <bits/wordsize.h>
#include <kernel-features.h>
#include <sysdeps/unix/sysdep.h>
+#include <sysdeps/unix/sysv/linux/sysdep.h>
/* Provide the common name to allow more code reuse. */
#define __NR__llseek __NR_llseek
@@ -22,6 +22,7 @@
#define _LINUX_HPPA_SYSDEP_H 1
#include <sysdeps/unix/sysdep.h>
+#include <sysdeps/unix/sysv/linux/sysdep.h>
#include <sysdeps/hppa/sysdep.h>
/* Defines RTLD_PRIVATE_ERRNO. */
@@ -27,3 +27,17 @@ endif
ifeq ($(subdir),stdlib)
gen-as-const-headers += ucontext_i.sym
endif
+
+ifeq ($(subdir),csu)
+sysdep-dl-routines += sysdep
+endif
+
+ifeq ($(subdir),nptl)
+# pull in __syscall_error routine
+libpthread-routines += sysdep
+endif
+
+ifeq ($(subdir),rt)
+# pull in __syscall_error routine
+librt-routines += sysdep
+endif
@@ -47,19 +47,11 @@ ENTRY (__clone)
/* Sanity check arguments. */
movl $-EINVAL,%eax
movl FUNC(%esp),%ecx /* no NULL function pointers */
-#ifdef PIC
- jecxz SYSCALL_ERROR_LABEL
-#else
testl %ecx,%ecx
jz SYSCALL_ERROR_LABEL
-#endif
movl STACK(%esp),%ecx /* no NULL stack pointers */
-#ifdef PIC
- jecxz SYSCALL_ERROR_LABEL
-#else
testl %ecx,%ecx
jz SYSCALL_ERROR_LABEL
-#endif
/* Insert the argument onto the new stack. Make sure the new
thread is started with an alignment of (mod 16). */
similarity index 53%
rename from sysdeps/unix/sysv/linux/i386/sysdep.S
rename to sysdeps/unix/sysv/linux/i386/sysdep.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 1995-2015 Free Software Foundation, Inc.
+/* Copyright (C) 2015 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
@@ -15,26 +15,16 @@
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
+#include <errno.h>
#include <sysdep.h>
-/* The following code is only used in the shared library when we
- compile the reentrant version. Otherwise each system call defines
- each own version. */
-
-#ifndef PIC
-
-/* The syscall stubs jump here when they detect an error.
- The code for Linux is almost identical to the canonical Unix/i386
- code, except that the error number in %eax is negated. */
-
-#undef CALL_MCOUNT
-#define CALL_MCOUNT /* Don't insert the profiling call, it clobbers %eax. */
-
- .text
-ENTRY (__syscall_error)
- negl %eax
-
-#define __syscall_error __syscall_error_1
-#include <sysdeps/unix/i386/sysdep.S>
-
-#endif /* !PIC */
+/* This routine is jumped to by all the syscall handlers, to stash
+ an error number into errno. ERROR is the negative error number
+ returned from the x86 kernel. */
+int
+__attribute__ ((__regparm__ (1)))
+__syscall_error (int error)
+{
+ __set_errno (-error);
+ return -1;
+}
@@ -20,6 +20,7 @@
#define _LINUX_I386_SYSDEP_H 1
/* There is some commonality. */
+#include <sysdeps/unix/sysv/linux/sysdep.h>
#include <sysdeps/unix/i386/sysdep.h>
/* Defines RTLD_PRIVATE_ERRNO and USE_DL_SYSINFO. */
#include <dl-sysdep.h>
@@ -55,11 +56,7 @@
/* We don't want the label for the error handle to be global when we define
it here. */
-#ifdef PIC
-# define SYSCALL_ERROR_LABEL 0f
-#else
-# define SYSCALL_ERROR_LABEL syscall_error
-#endif
+#define SYSCALL_ERROR_LABEL __syscall_error
#undef PSEUDO
#define PSEUDO(name, syscall_name, args) \
@@ -100,55 +97,7 @@
#define ret_ERRVAL ret
-#ifndef PIC
-# define SYSCALL_ERROR_HANDLER /* Nothing here; code in sysdep.S is used. */
-#else
-
-# if RTLD_PRIVATE_ERRNO
-# define SYSCALL_ERROR_HANDLER \
-0:SETUP_PIC_REG(cx); \
- addl $_GLOBAL_OFFSET_TABLE_, %ecx; \
- negl %eax; \
- movl %eax, rtld_errno@GOTOFF(%ecx); \
- orl $-1, %eax; \
- ret;
-
-# elif defined _LIBC_REENTRANT
-
-# if IS_IN (libc)
-# define SYSCALL_ERROR_ERRNO __libc_errno
-# else
-# define SYSCALL_ERROR_ERRNO errno
-# endif
-# define SYSCALL_ERROR_HANDLER \
-0:SETUP_PIC_REG (cx); \
- addl $_GLOBAL_OFFSET_TABLE_, %ecx; \
- movl SYSCALL_ERROR_ERRNO@GOTNTPOFF(%ecx), %ecx; \
- negl %eax; \
- SYSCALL_ERROR_HANDLER_TLS_STORE (%eax, %ecx); \
- orl $-1, %eax; \
- ret;
-# ifndef NO_TLS_DIRECT_SEG_REFS
-# define SYSCALL_ERROR_HANDLER_TLS_STORE(src, destoff) \
- movl src, %gs:(destoff)
-# else
-# define SYSCALL_ERROR_HANDLER_TLS_STORE(src, destoff) \
- addl %gs:0, destoff; \
- movl src, (destoff)
-# endif
-# else
-/* Store (- %eax) into errno through the GOT. */
-# define SYSCALL_ERROR_HANDLER \
-0:SETUP_PIC_REG(cx); \
- addl $_GLOBAL_OFFSET_TABLE_, %ecx; \
- negl %eax; \
- movl errno@GOT(%ecx), %ecx; \
- movl %eax, (%ecx); \
- orl $-1, %eax; \
- ret;
-# endif /* _LIBC_REENTRANT */
-#endif /* PIC */
-
+#define SYSCALL_ERROR_HANDLER /* Nothing here; code in sysdep.c is used. */
/* The original calling convention for system calls on Linux/i386 is
to use int $0x80. */
@@ -275,6 +224,9 @@
#else /* !__ASSEMBLER__ */
+extern int __syscall_error (int)
+ attribute_hidden __attribute__ ((__regparm__ (1)));
+
/* We need some help from the assembler to generate optimal code. We
define some macros here which later will be used. */
asm (".L__X'%ebx = 1\n\t"
@@ -318,7 +270,15 @@ struct libc_do_syscall_args
/* Define a macro which expands inline into the wrapper code for a system
call. */
#undef INLINE_SYSCALL
-#define INLINE_SYSCALL(name, nr, args...) \
+#if IS_IN (libc)
+# define INLINE_SYSCALL(name, nr, args...) \
+ ({ \
+ unsigned int resultvar = INTERNAL_SYSCALL (name, , nr, args); \
+ __glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (resultvar, )) \
+ ? __syscall_error (-INTERNAL_SYSCALL_ERRNO (resultvar, )) \
+ : (int) resultvar; })
+#else
+# define INLINE_SYSCALL(name, nr, args...) \
({ \
unsigned int resultvar = INTERNAL_SYSCALL (name, , nr, args); \
if (__glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (resultvar, ))) \
@@ -327,6 +287,14 @@ struct libc_do_syscall_args
resultvar = 0xffffffff; \
} \
(int) resultvar; })
+#endif
+
+/* Set error number and return -1. Return the internal function,
+ __syscall_error, which sets errno from the negative error number
+ and returns -1, to avoid PIC. */
+#undef INLINE_SYSCALL_ERROR_RETURN
+#define INLINE_SYSCALL_ERROR_RETURN(resultvar) \
+ __syscall_error (-(resultvar))
/* List of system calls which are supported as vsyscalls. */
# define HAVE_CLOCK_GETTIME_VSYSCALL 1
@@ -21,6 +21,7 @@
#define _LINUX_IA64_SYSDEP_H 1
#include <sysdeps/unix/sysdep.h>
+#include <sysdeps/unix/sysv/linux/sysdep.h>
#include <sysdeps/ia64/sysdep.h>
#include <dl-sysdep.h>
#include <tls.h>
@@ -17,6 +17,7 @@
License along with the GNU C Library. If not, see
<http://www.gnu.org/licenses/>. */
+#include <sysdeps/unix/sysv/linux/sysdep.h>
#include <tls.h>
/* Defines RTLD_PRIVATE_ERRNO. */
@@ -20,6 +20,7 @@
#define _LINUX_MICROBLAZE_SYSDEP_H 1
#include <sysdeps/unix/sysdep.h>
+#include <sysdeps/unix/sysv/linux/sysdep.h>
#include <sysdeps/microblaze/sysdep.h>
/* Defines RTLD_PRIVATE_ERRNO. */
@@ -19,6 +19,7 @@
#define _LINUX_MIPS_MIPS32_SYSDEP_H 1
/* There is some commonality. */
+#include <sysdeps/unix/sysv/linux/sysdep.h>
#include <sysdeps/unix/mips/mips32/sysdep.h>
#include <tls.h>
@@ -19,6 +19,7 @@
#define _LINUX_MIPS_SYSDEP_H 1
/* There is some commonality. */
+#include <sysdeps/unix/sysv/linux/sysdep.h>
#include <sysdeps/unix/mips/mips64/n32/sysdep.h>
#include <tls.h>
@@ -19,6 +19,7 @@
#define _LINUX_MIPS_SYSDEP_H 1
/* There is some commonality. */
+#include <sysdeps/unix/sysv/linux/sysdep.h>
#include <sysdeps/unix/mips/mips64/n64/sysdep.h>
#include <tls.h>
@@ -18,6 +18,7 @@
#ifndef _LINUX_POWERPC_SYSDEP_H
#define _LINUX_POWERPC_SYSDEP_H 1
+#include <sysdeps/unix/sysv/linux/sysdep.h>
#include <sysdeps/unix/powerpc/sysdep.h>
#include <tls.h>
@@ -20,6 +20,7 @@
#ifndef _LINUX_POWERPC_SYSDEP_H
#define _LINUX_POWERPC_SYSDEP_H 1
+#include <sysdeps/unix/sysv/linux/sysdep.h>
#include <sysdeps/unix/powerpc/sysdep.h>
#include <tls.h>
@@ -21,6 +21,7 @@
#include <sysdeps/s390/s390-32/sysdep.h>
#include <sysdeps/unix/sysdep.h>
+#include <sysdeps/unix/sysv/linux/sysdep.h>
#include <dl-sysdep.h> /* For RTLD_PRIVATE_ERRNO. */
#include <tls.h>
@@ -22,6 +22,7 @@
#include <sysdeps/s390/s390-64/sysdep.h>
#include <sysdeps/unix/sysdep.h>
+#include <sysdeps/unix/sysv/linux/sysdep.h>
#include <dl-sysdep.h> /* For RTLD_PRIVATE_ERRNO. */
#include <tls.h>
@@ -21,6 +21,7 @@
#define _LINUX_SH_SYSDEP_H 1
/* There is some commonality. */
+#include <sysdeps/unix/sysv/linux/sysdep.h>
#include <sysdeps/unix/sh/sysdep.h>
#include <tls.h>
@@ -20,6 +20,7 @@
#define _LINUX_SPARC_SYSDEP_H 1
#include <sysdeps/unix/sysdep.h>
+#include <sysdeps/unix/sysv/linux/sysdep.h>
#include <sysdeps/sparc/sysdep.h>
#ifdef __ASSEMBLER__
new file mode 100644
@@ -0,0 +1,24 @@
+/* Copyright (C) 2015 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+/* Set error number and return -1. A target may choose to return the
+ internal function, __syscall_error, which sets errno and returns -1. */
+#define INLINE_SYSCALL_ERROR_RETURN(err) \
+ ({ \
+ __set_errno (err); \
+ -1; \
+ })
@@ -19,6 +19,7 @@
#define _LINUX_X86_64_SYSDEP_H 1
/* There is some commonality. */
+#include <sysdeps/unix/sysv/linux/sysdep.h>
#include <sysdeps/unix/x86_64/sysdep.h>
#include <tls.h>
For ia32 PIC, the first thing of many syscalls does is to call __x86.get_pc_thunk.reg to load PC into reg in case there is an error, which is required for setting errno. In most cases, there are no errors. But we still call __x86.get_pc_thunk.reg. This patch adds INLINE_SYSCALL_ERROR_RETURN so that i386 can optimize setting errno by branching to the internal __syscall_error without PLT. With i386 INLINE_SYSCALL_ERROR_RETURN and i386 syscall inlining optimization for GCC 5, for sysdeps/unix/sysv/linux/fchmodat.c with -O2 -march=i686 -mtune=generic, GCC 5.2 now generates: <fchmodat>: 0: push %ebx 1: mov 0x14(%esp),%eax 5: mov 0x8(%esp),%ebx 9: mov 0xc(%esp),%ecx d: mov 0x10(%esp),%edx 11: test $0xfffffeff,%eax 16: jne 38 <fchmodat+0x38> 18: test $0x1,%ah 1b: jne 48 <fchmodat+0x48> 1d: mov $0x132,%eax 22: call *%gs:0x10 29: cmp $0xfffff000,%eax 2e: ja 58 <fchmodat+0x58> 30: pop %ebx 31: ret 32: lea 0x0(%esi),%esi 38: pop %ebx 39: mov $0xffffffea,%eax 3e: jmp 3f <fchmodat+0x3f> 3f: R_386_PC32 __syscall_error 43: nop 44: lea 0x0(%esi,%eiz,1),%esi 48: pop %ebx 49: mov $0xffffffa1,%eax 4e: jmp 4f <fchmodat+0x4f> 4f: R_386_PC32 __syscall_error 53: nop 54: lea 0x0(%esi,%eiz,1),%esi 58: pop %ebx 59: jmp 5a <fchmodat+0x5a> 5a: R_386_PC32 __syscall_error instead of <fchmodat>: 0: sub $0x8,%esp 3: mov 0x18(%esp),%eax 7: mov %ebx,(%esp) a: call b <fchmodat+0xb> b: R_386_PC32 __x86.get_pc_thunk.bx f: add $0x2,%ebx 11: R_386_GOTPC _GLOBAL_OFFSET_TABLE_ 15: mov %edi,0x4(%esp) 19: test $0xfffffeff,%eax 1e: jne 70 <fchmodat+0x70> 20: test $0x1,%ah 23: jne 88 <fchmodat+0x88> 25: mov 0x14(%esp),%edx 29: mov 0x10(%esp),%ecx 2d: mov 0xc(%esp),%edi 31: xchg %ebx,%edi 33: mov $0x132,%eax 38: call *%gs:0x10 3f: xchg %edi,%ebx 41: cmp $0xfffff000,%eax 46: ja 58 <fchmodat+0x58> 48: mov (%esp),%ebx 4b: mov 0x4(%esp),%edi 4f: add $0x8,%esp 52: ret 53: nop 54: lea 0x0(%esi,%eiz,1),%esi 58: mov 0x0(%ebx),%edx 5a: R_386_TLS_GOTIE __libc_errno 5e: neg %eax 60: mov %eax,%gs:(%edx) 63: mov $0xffffffff,%eax 68: jmp 48 <fchmodat+0x48> 6a: lea 0x0(%esi),%esi 70: mov 0x0(%ebx),%eax 72: R_386_TLS_GOTIE __libc_errno 76: movl $0x16,%gs:(%eax) 7d: mov $0xffffffff,%eax 82: jmp 48 <fchmodat+0x48> 84: lea 0x0(%esi,%eiz,1),%esi 88: mov 0x0(%ebx),%eax 8a: R_386_TLS_GOTIE __libc_errno 8e: movl $0x5f,%gs:(%eax) 95: mov $0xffffffff,%eax 9a: jmp 48 <fchmodat+0x48> * sysdeps/unix/sysv/linux/sysdep.h: New file. * sysdeps/unix/sysv/linux/i386/sysdep.c: Likewise. * sysdeps/unix/sysv/linux/alpha/sysdep.h: Include <sysdeps/unix/sysv/linux/sysdep.h>. * sysdeps/unix/sysv/linux/arm/sysdep.h: Likewise. * sysdeps/unix/sysv/linux/generic/sysdep.h: Likewise. * sysdeps/unix/sysv/linux/hppa/sysdep.h: Likewise. * sysdeps/unix/sysv/linux/ia64/sysdep.h: Likewise. * sysdeps/unix/sysv/linux/m68k/sysdep.h: Likewise. * sysdeps/unix/sysv/linux/microblaze/sysdep.h: Likewise. * sysdeps/unix/sysv/linux/mips/mips32/sysdep.h: Likewise. * sysdeps/unix/sysv/linux/mips/mips64/n32/sysdep.h: Likewise. * sysdeps/unix/sysv/linux/mips/mips64/n64/sysdep.h: Likewise. * sysdeps/unix/sysv/linux/powerpc/powerpc32/sysdep.h: Likewise. * sysdeps/unix/sysv/linux/powerpc/powerpc64/sysdep.h: Likewise. * sysdeps/unix/sysv/linux/s390/s390-32/sysdep.h: Likewise. * sysdeps/unix/sysv/linux/s390/s390-64/sysdep.h: Likewise. * sysdeps/unix/sysv/linux/sh/sysdep.h: Likewise. * sysdeps/unix/sysv/linux/sparc/sysdep.h: Likewise. * sysdeps/unix/sysv/linux/x86_64/sysdep.h: Likewise. * sysdeps/unix/sysv/linux/i386/Makefile [$(subdir) == csu] (sysdep-dl-routines): Add sysdep. [$(subdir) == nptl] (libpthread-routines): Likewise. [$(subdir) == rt] (librt-routines): Likewise. * sysdeps/unix/sysv/linux/i386/clone.S (__clone): Don't check PIC when branching to SYSCALL_ERROR_LABEL. * sysdeps/unix/sysv/linux/i386/sysdep.S: Removed. * sysdeps/unix/sysv/linux/i386/sysdep.h: Include <sysdeps/unix/sysv/linux/sysdep.h>. (SYSCALL_ERROR_LABEL): Changed to __syscall_error. (SYSCALL_ERROR_ERRNO): Removed. (SYSCALL_ERROR_HANDLER): Changed to empty. (SYSCALL_ERROR_HANDLER_TLS_STORE): Likewise. (__syscall_error): New prototype. [IS_IN (libc)] (INLINE_SYSCALL): New macro. (INLINE_SYSCALL_ERROR_RETURN): Likewise. --- sysdeps/unix/sysv/linux/alpha/sysdep.h | 1 + sysdeps/unix/sysv/linux/arm/sysdep.h | 1 + sysdeps/unix/sysv/linux/generic/sysdep.h | 1 + sysdeps/unix/sysv/linux/hppa/sysdep.h | 1 + sysdeps/unix/sysv/linux/i386/Makefile | 14 ++++ sysdeps/unix/sysv/linux/i386/clone.S | 8 --- .../unix/sysv/linux/i386/{sysdep.S => sysdep.c} | 34 ++++------ sysdeps/unix/sysv/linux/i386/sysdep.h | 78 +++++++--------------- sysdeps/unix/sysv/linux/ia64/sysdep.h | 1 + sysdeps/unix/sysv/linux/m68k/sysdep.h | 1 + sysdeps/unix/sysv/linux/microblaze/sysdep.h | 1 + sysdeps/unix/sysv/linux/mips/mips32/sysdep.h | 1 + sysdeps/unix/sysv/linux/mips/mips64/n32/sysdep.h | 1 + sysdeps/unix/sysv/linux/mips/mips64/n64/sysdep.h | 1 + sysdeps/unix/sysv/linux/powerpc/powerpc32/sysdep.h | 1 + sysdeps/unix/sysv/linux/powerpc/powerpc64/sysdep.h | 1 + sysdeps/unix/sysv/linux/s390/s390-32/sysdep.h | 1 + sysdeps/unix/sysv/linux/s390/s390-64/sysdep.h | 1 + sysdeps/unix/sysv/linux/sh/sysdep.h | 1 + sysdeps/unix/sysv/linux/sparc/sysdep.h | 1 + sysdeps/unix/sysv/linux/sysdep.h | 24 +++++++ sysdeps/unix/sysv/linux/x86_64/sysdep.h | 1 + 22 files changed, 90 insertions(+), 85 deletions(-) rename sysdeps/unix/sysv/linux/i386/{sysdep.S => sysdep.c} (53%) create mode 100644 sysdeps/unix/sysv/linux/sysdep.h