diff -r 711e085ceb33 gcc/config/i386/i386-protos.h
--- a/gcc/config/i386/i386-protos.h	Wed Sep 22 11:55:54 2010 +0200
+++ b/gcc/config/i386/i386-protos.h	Thu Sep 23 10:22:16 2010 +0200
@@ -140,7 +140,6 @@
 extern bool ix86_function_arg_regno_p (int);
 extern void ix86_asm_output_function_label (FILE *, const char *, tree);
 extern int ix86_function_arg_boundary (enum machine_mode, const_tree);
-extern bool ix86_solaris_return_in_memory (const_tree, const_tree);
 extern rtx ix86_force_to_memory (enum machine_mode, rtx);
 extern void ix86_free_from_memory (enum machine_mode);
 extern void ix86_call_abi_override (const_tree);
diff -r 711e085ceb33 gcc/config/i386/i386.c
--- a/gcc/config/i386/i386.c	Wed Sep 22 11:55:54 2010 +0200
+++ b/gcc/config/i386/i386.c	Thu Sep 23 10:22:16 2010 +0200
@@ -2627,6 +2627,7 @@
     { "-msseregparm",			MASK_SSEREGPARM },
     { "-mstack-arg-probe",		MASK_STACK_PROBE },
     { "-mtls-direct-seg-refs",		MASK_TLS_DIRECT_SEG_REFS },
+    { "-mvect8-ret-in-mem",		MASK_VECT8_RETURNS },
     { "-m8bit-idiv",			MASK_USE_8BIT_IDIV },
   };
 
@@ -6840,9 +6841,9 @@
 	return false;
 
       /* MMX/3dNow values are returned in MM0,
-	 except when it doesn't exits.  */
+	 except when it doesn't exits or the ABI prescribes otherwise.  */
       if (size == 8)
-	return !TARGET_MMX;
+	return !TARGET_MMX || TARGET_VECT8_RETURNS;
 
       /* SSE values are returned in XMM0, except when it doesn't exist.  */
       if (size == 16)
@@ -6906,43 +6907,6 @@
 #endif
 }
 
-/* Return false iff TYPE is returned in memory.  This version is used
-   on Solaris 2.  It is similar to the generic ix86_return_in_memory,
-   but differs notably in that when MMX is available, 8-byte vectors
-   are returned in memory, rather than in MMX registers.  */
-
-bool
-ix86_solaris_return_in_memory (const_tree type, const_tree fntype ATTRIBUTE_UNUSED)
-{
-  int size;
-  enum machine_mode mode = type_natural_mode (type, NULL);
-
-  if (TARGET_64BIT)
-    return return_in_memory_64 (type, mode);
-
-  if (mode == BLKmode)
-    return 1;
-
-  size = int_size_in_bytes (type);
-
-  if (VECTOR_MODE_P (mode))
-    {
-      /* Return in memory only if MMX registers *are* available.  This
-	 seems backwards, but it is consistent with the existing
-	 Solaris x86 ABI.  */
-      if (size == 8)
-	return TARGET_MMX;
-      if (size == 16)
-	return !TARGET_SSE;
-    }
-  else if (mode == TImode)
-    return !TARGET_SSE;
-  else if (mode == XFmode)
-    return 0;
-
-  return size > 12;
-}
-
 /* When returning SSE vector types, we have a choice of either
      (1) being abi incompatible with a -march switch, or
      (2) generating an error.
diff -r 711e085ceb33 gcc/config/i386/i386.opt
--- a/gcc/config/i386/i386.opt	Wed Sep 22 11:55:54 2010 +0200
+++ b/gcc/config/i386/i386.opt	Thu Sep 23 10:22:16 2010 +0200
@@ -244,6 +244,10 @@
 Target RejectNegative Joined Var(ix86_veclibabi_string)
 Vector library ABI to use
 
+mvect8-ret-in-mem
+Target Report Mask(VECT8_RETURNS) Save
+Return 8-byte vectors in memory
+
 mrecip
 Target Report Mask(RECIP) Save
 Generate reciprocals instead of divss and sqrtss.
diff -r 711e085ceb33 gcc/config/i386/sol2-10.h
--- a/gcc/config/i386/sol2-10.h	Wed Sep 22 11:55:54 2010 +0200
+++ b/gcc/config/i386/sol2-10.h	Thu Sep 23 10:22:16 2010 +0200
@@ -89,9 +89,12 @@
     }								\
   while (0)
 
+/* Override i386/sol2.h version: return 8-byte vectors in MMX registers if
+   possible, matching Sun Studio 12 Update 1+ compilers and other x86
+   targets.  */
 #undef TARGET_SUBTARGET_DEFAULT
-#define TARGET_SUBTARGET_DEFAULT (MASK_80387 | MASK_IEEE_FP	\
-				  | MASK_FLOAT_RETURNS)
+#define TARGET_SUBTARGET_DEFAULT \
+	(MASK_80387 | MASK_IEEE_FP | MASK_FLOAT_RETURNS)
 
 #define SUBTARGET_OPTIMIZATION_OPTIONS			\
   do							\
diff -r 711e085ceb33 gcc/config/i386/sol2.h
--- a/gcc/config/i386/sol2.h	Wed Sep 22 11:55:54 2010 +0200
+++ b/gcc/config/i386/sol2.h	Thu Sep 23 10:22:16 2010 +0200
@@ -140,9 +140,15 @@
 /* Register the Solaris-specific #pragma directives.  */
 #define REGISTER_SUBTARGET_PRAGMAS() solaris_register_pragmas ()
 
+/* Undo i386/sysv4.h version.  */
 #undef SUBTARGET_RETURN_IN_MEMORY
-#define SUBTARGET_RETURN_IN_MEMORY(TYPE, FNTYPE) \
-	ix86_solaris_return_in_memory (TYPE, FNTYPE)
+
+/* Augment i386/unix.h version to return 8-byte vectors in memory, matching
+   Sun Studio compilers until version 12, the only ones supported on
+   Solaris 8 and 9.  */
+#undef TARGET_SUBTARGET_DEFAULT
+#define TARGET_SUBTARGET_DEFAULT \
+	(MASK_80387 | MASK_IEEE_FP | MASK_FLOAT_RETURNS | MASK_VECT8_RETURNS)
 
 /* Output a simple call for .init/.fini.  */
 #define ASM_OUTPUT_CALL(FILE, FN)				\
diff -r 711e085ceb33 gcc/config/i386/vx-common.h
--- a/gcc/config/i386/vx-common.h	Wed Sep 22 11:55:54 2010 +0200
+++ b/gcc/config/i386/vx-common.h	Thu Sep 23 10:22:16 2010 +0200
@@ -20,7 +20,8 @@
 #define ASM_OUTPUT_ALIGNED_BSS(FILE, DECL, NAME, SIZE, ALIGN) \
   asm_output_aligned_bss (FILE, DECL, NAME, SIZE, ALIGN)
 
-/* VxWorks uses the same ABI as Solaris 2.  */
+/* VxWorks uses the same ABI as Solaris 2, so use i386/sol2.h version.  */
 
-#define SUBTARGET_RETURN_IN_MEMORY(TYPE, FNTYPE) \
-	ix86_solaris_return_in_memory (TYPE, FNTYPE)
+#undef TARGET_SUBTARGET_DEFAULT
+#define TARGET_SUBTARGET_DEFAULT \
+	(MASK_80387 | MASK_IEEE_FP | MASK_FLOAT_RETURNS | MASK_VECT8_RETURNS)
diff -r 711e085ceb33 gcc/doc/invoke.texi
--- a/gcc/doc/invoke.texi	Wed Sep 22 11:55:54 2010 +0200
+++ b/gcc/doc/invoke.texi	Thu Sep 23 10:22:16 2010 +0200
@@ -598,7 +598,8 @@
 -minline-stringops-dynamically -mstringop-strategy=@var{alg} @gol
 -mpush-args  -maccumulate-outgoing-args  -m128bit-long-double @gol
 -m96bit-long-double  -mregparm=@var{num}  -msseregparm @gol
--mveclibabi=@var{type} -mpc32 -mpc64 -mpc80 -mstackrealign @gol
+-mveclibabi=@var{type} -mvect8-ret-in-mem @gol
+-mpc32 -mpc64 -mpc80 -mstackrealign @gol
 -momit-leaf-frame-pointer  -mno-red-zone -mno-tls-direct-seg-refs @gol
 -mcmodel=@var{code-model} -mabi=@var{name} @gol
 -m32  -m64 -mlarge-data-threshold=@var{num} @gol
@@ -12319,6 +12320,16 @@
 modules with the same value, including any libraries.  This includes
 the system libraries and startup modules.
 
+@item -mvect8-ret-in-mem
+@opindex mvect8-ret-in-mem
+Return 8-byte vectors in memory instead of MMX registers.  This is the
+default on Solaris~8 and 9 and VxWorks to match the ABI of the Sun
+Studio compilers until version 12.  Later compiler versions (starting
+with Studio 12 Update~1) follow the ABI used by other x86 targets, which
+is the default on Solaris~10 and later.  @emph{Only} use this option if
+yo need to remain compatible with existing code produced by those
+previous compiler versions or older versions of GCC.
+
 @item -mpc32
 @itemx -mpc64
 @itemx -mpc80
diff -r 711e085ceb33 gcc/testsuite/gcc.target/i386/pr22076.c
--- a/gcc/testsuite/gcc.target/i386/pr22076.c	Wed Sep 22 11:55:54 2010 +0200
+++ b/gcc/testsuite/gcc.target/i386/pr22076.c	Thu Sep 23 10:22:16 2010 +0200
@@ -1,5 +1,6 @@
 /* { dg-do compile } */
 /* { dg-options "-O2 -fomit-frame-pointer -flax-vector-conversions -mmmx" } */
+/* { dg-options "-O2 -fomit-frame-pointer -flax-vector-conversions -mmmx -mno-vect8-ret-in-mem" { target i?86-*-solaris2.[89] *-*-vxworks* } } */
 
 #include <mmintrin.h>
 
diff -r 711e085ceb33 gcc/testsuite/gcc.target/i386/pr22152.c
--- a/gcc/testsuite/gcc.target/i386/pr22152.c	Wed Sep 22 11:55:54 2010 +0200
+++ b/gcc/testsuite/gcc.target/i386/pr22152.c	Thu Sep 23 10:22:16 2010 +0200
@@ -1,5 +1,6 @@
 /* { dg-do compile } */
 /* { dg-options "-O2 -msse2" } */
+/* { dg-options "-O2 -msse2 -mno-vect8-ret-in-mem" { target i?86-*-solaris2.[89] *-*-vxworks* } } */
 
 #include <mmintrin.h>
 
diff -r 711e085ceb33 gcc/testsuite/gcc.target/i386/vect8-ret.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/gcc/testsuite/gcc.target/i386/vect8-ret.c	Thu Sep 23 10:22:16 2010 +0200
@@ -0,0 +1,13 @@
+/* { dg-do compile { target ilp32 } } */
+/* { dg-options "-mmmx" { target i?86-*-solaris2.[89] *-*-vxworks* } } */
+/* { dg-options "-mmmx -mvect8-ret-in-mem" } */
+
+#include <mmintrin.h>
+
+__m64
+vecret (__m64 vect)
+{
+  return vect;
+}
+
+/* { dg-final { scan-assembler-times "movq" 1 } } */


2010-09-26  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>

	gcc/testsuite:
	PR target/44452
	* gcc.target/i386/abi-2.c: XFAIL on i?86-*-solaris2* && ilp32.
	* gcc.target/i386/pr22076.c: Likewise.

diff -r 87f36be56257 gcc/testsuite/gcc.target/i386/abi-2.c
--- a/gcc/testsuite/gcc.target/i386/abi-2.c	Fri Sep 17 10:22:17 2010 +0000
+++ b/gcc/testsuite/gcc.target/i386/abi-2.c	Sun Sep 26 14:58:07 2010 +0200
@@ -6,4 +6,5 @@
 typedef long long __m256i __attribute__ ((__vector_size__ (32)));
 __m256i foo (void) { return (__m256i){ 1, 2, 3, 4 }; }
 
-/* { dg-final { scan-assembler-times "ymm0" 1 } } */
+/* Fails on 32-bit Solaris 2/x86: PR target/44452 */
+/* { dg-final { scan-assembler-times "ymm0" 1 { xfail { i?86-*-solaris2* && ilp32 } } } } */
diff -r 87f36be56257 gcc/testsuite/gcc.target/i386/pr22076.c
--- a/gcc/testsuite/gcc.target/i386/pr22076.c	Fri Sep 17 10:22:17 2010 +0000
+++ b/gcc/testsuite/gcc.target/i386/pr22076.c	Sun Sep 26 14:58:07 2010 +0200
@@ -14,5 +14,6 @@
   return x;
 }
 
-/* { dg-final { scan-assembler-times "movq" 3 } } */
-/* { dg-final { scan-assembler-not "movl" { target nonpic } } } */
+/* Fails on 32-bit Solaris 2/x86: PR target/44452 */
+/* { dg-final { scan-assembler-times "movq" 3 { xfail { i?86-*-solaris2* && ilp32 } } } } */
+/* { dg-final { if [ istarget i?86-*-solaris2* ] { scan-assembler-not "movl" { xfail { i?86-*-solaris2* && ilp32 } } } else { scan-assembler-not "movl" { target nonpic } } } } */
