diff mbox series

[13/18] Support -mevex512 for AVX512F intrins

Message ID 20230921072013.2124750-14-lin1.hu@intel.com
State New
Headers show
Series Support -mevex512 for AVX512 | expand

Commit Message

Hu, Lin1 Sept. 21, 2023, 7:20 a.m. UTC 
From: Haochen Jiang <haochen.jiang@intel.com>

gcc/ChangeLog:

	* config/i386/i386-builtins.cc
	(ix86_vectorize_builtin_gather): Disable 512 bit gather
	when !TARGET_EVEX512.
	* config/i386/i386-expand.cc (ix86_valid_mask_cmp_mode):
	Add TARGET_EVEX512.
	(ix86_expand_int_sse_cmp): Ditto.
	(ix86_expand_vector_init_one_nonzero): Disable subroutine
	when !TARGET_EVEX512.
	(ix86_emit_swsqrtsf): Add TARGET_EVEX512.
	(ix86_vectorize_vec_perm_const): Disable subroutine when
	!TARGET_EVEX512.
	* config/i386/i386.cc
	(standard_sse_constant_p): Add TARGET_EVEX512.
	(standard_sse_constant_opcode): Ditto.
	(ix86_get_ssemov): Ditto.
	(ix86_legitimate_constant_p): Ditto.
	(ix86_vectorize_builtin_scatter): Diable 512 bit scatter
	when !TARGET_EVEX512.
	* config/i386/i386.md (avx512f_512): New.
	(movxi): Add TARGET_EVEX512.
	(*movxi_internal_avx512f): Ditto.
	(*movdi_internal): Change alternative 12 to ?Yv. Adjust mode
	for alternative 13.
	(*movsi_internal): Change alternative 8 to ?Yv. Adjust mode for
	alternative 9.
	(*movhi_internal): Change alternative 11 to *Yv.
	(*movdf_internal): Change alternative 12 to Yv.
	(*movsf_internal): Change alternative 5 to Yv. Adjust mode for
	alternative 5 and 6.
	(*mov<mode>_internal): Change alternative 4 to Yv.
	(define_split for convert SF to DF): Add TARGET_EVEX512.
	(extendbfsf2_1): Ditto.
	* config/i386/predicates.md (bcst_mem_operand): Disable predicate
	for 512 bit when !TARGET_EVEX512.
	* config/i386/sse.md (VMOVE): Add TARGET_EVEX512.
	(V48_AVX512VL): Ditto.
	(V48_256_512_AVX512VL): Ditto.
	(V48H_AVX512VL): Ditto.
	(VI12_AVX512VL): Ditto.
	(V): Ditto.
	(V_512): Ditto.
	(V_256_512): Ditto.
	(VF): Ditto.
	(VF1_VF2_AVX512DQ): Ditto.
	(VFH): Ditto.
	(VFB): Ditto.
	(VF1): Ditto.
	(VF1_AVX2): Ditto.
	(VF2): Ditto.
	(VF2H): Ditto.
	(VF2_512_256): Ditto.
	(VF2_512_256VL): Ditto.
	(VF_512): Ditto.
	(VFB_512): Ditto.
	(VI48_AVX512VL): Ditto.
	(VI1248_AVX512VLBW): Ditto.
	(VF_AVX512VL): Ditto.
	(VFH_AVX512VL): Ditto.
	(VF1_AVX512VL): Ditto.
	(VI): Ditto.
	(VIHFBF): Ditto.
	(VI_AVX2): Ditto.
	(VI8): Ditto.
	(VI8_AVX512VL): Ditto.
	(VI2_AVX512F): Ditto.
	(VI4_AVX512F): Ditto.
	(VI4_AVX512VL): Ditto.
	(VI48_AVX512F_AVX512VL): Ditto.
	(VI8_AVX2_AVX512F): Ditto.
	(VI8_AVX_AVX512F): Ditto.
	(V8FI): Ditto.
	(V16FI): Ditto.
	(VI124_AVX2_24_AVX512F_1_AVX512BW): Ditto.
	(VI248_AVX512VLBW): Ditto.
	(VI248_AVX2_8_AVX512F_24_AVX512BW): Ditto.
	(VI248_AVX512BW): Ditto.
	(VI248_AVX512BW_AVX512VL): Ditto.
	(VI48_AVX512F): Ditto.
	(VI48_AVX_AVX512F): Ditto.
	(VI12_AVX_AVX512F): Ditto.
	(VI148_512): Ditto.
	(VI124_256_AVX512F_AVX512BW): Ditto.
	(VI48_512): Ditto.
	(VI_AVX512BW): Ditto.
	(VIHFBF_AVX512BW): Ditto.
	(VI4F_256_512): Ditto.
	(VI48F_256_512): Ditto.
	(VI48F): Ditto.
	(VI12_VI48F_AVX512VL): Ditto.
	(V32_512): Ditto.
	(AVX512MODE2P): Ditto.
	(STORENT_MODE): Ditto.
	(REDUC_PLUS_MODE): Ditto.
	(REDUC_SMINMAX_MODE): Ditto.
	(*andnot<mode>3): Change isa attribute to avx512f_512.
	(*andnot<mode>3): Ditto.
	(<code><mode>3): Ditto.
	(<code>tf3): Ditto.
	(FMAMODEM): Add TARGET_EVEX512.
	(FMAMODE_AVX512): Ditto.
	(VFH_SF_AVX512VL): Ditto.
	(avx512f_fix_notruncv16sfv16si<mask_name><round_name>): Ditto.
	(fix<fixunssuffix>_truncv16sfv16si2<mask_name><round_saeonly_name>):
	Ditto.
	(avx512f_cvtdq2pd512_2): Ditto.
	(avx512f_cvtpd2dq512<mask_name><round_name>): Ditto.
	(fix<fixunssuffix>_truncv8dfv8si2<mask_name><round_saeonly_name>):
	Ditto.
	(<mask_codefor>avx512f_cvtpd2ps512<mask_name><round_name>): Ditto.
	(vec_unpacks_lo_v16sf): Ditto.
	(vec_unpacks_hi_v16sf): Ditto.
	(vec_unpacks_float_hi_v16si): Ditto.
	(vec_unpacks_float_lo_v16si): Ditto.
	(vec_unpacku_float_hi_v16si): Ditto.
	(vec_unpacku_float_lo_v16si): Ditto.
	(vec_pack_sfix_trunc_v8df): Ditto.
	(avx512f_vec_pack_sfix_v8df): Ditto.
	(<mask_codefor>avx512f_unpckhps512<mask_name>): Ditto.
	(<mask_codefor>avx512f_unpcklps512<mask_name>): Ditto.
	(<mask_codefor>avx512f_movshdup512<mask_name>): Ditto.
	(<mask_codefor>avx512f_movsldup512<mask_name>): Ditto.
	(AVX512_VEC): Ditto.
	(AVX512_VEC_2): Ditto.
	(vec_extract_lo_v64qi): Ditto.
	(vec_extract_hi_v64qi): Ditto.
	(VEC_EXTRACT_MODE): Ditto.
	(<mask_codefor>avx512f_unpckhpd512<mask_name>): Ditto.
	(avx512f_movddup512<mask_name>): Ditto.
	(avx512f_unpcklpd512<mask_name>): Ditto.
	(*<avx512>_vternlog<mode>_all): Ditto.
	(*<avx512>_vpternlog<mode>_1): Ditto.
	(*<avx512>_vpternlog<mode>_2): Ditto.
	(*<avx512>_vpternlog<mode>_3): Ditto.
	(avx512f_shufps512_mask): Ditto.
	(avx512f_shufps512_1<mask_name>): Ditto.
	(avx512f_shufpd512_mask): Ditto.
	(avx512f_shufpd512_1<mask_name>): Ditto.
	(<mask_codefor>avx512f_interleave_highv8di<mask_name>): Ditto.
	(<mask_codefor>avx512f_interleave_lowv8di<mask_name>): Ditto.
	(vec_dupv2df<mask_name>): Ditto.
	(trunc<pmov_src_lower><mode>2): Ditto.
	(*avx512f_<code><pmov_src_lower><mode>2): Ditto.
	(*avx512f_vpermvar_truncv8div8si_1): Ditto.
	(avx512f_<code><pmov_src_lower><mode>2_mask): Ditto.
	(avx512f_<code><pmov_src_lower><mode>2_mask_store): Ditto.
	(truncv8div8qi2): Ditto.
	(avx512f_<code>v8div16qi2): Ditto.
	(*avx512f_<code>v8div16qi2_store_1): Ditto.
	(*avx512f_<code>v8div16qi2_store_2): Ditto.
	(avx512f_<code>v8div16qi2_mask): Ditto.
	(*avx512f_<code>v8div16qi2_mask_1): Ditto.
	(*avx512f_<code>v8div16qi2_mask_store_1): Ditto.
	(avx512f_<code>v8div16qi2_mask_store_2): Ditto.
	(vec_widen_umult_even_v16si<mask_name>): Ditto.
	(*vec_widen_umult_even_v16si<mask_name>): Ditto.
	(vec_widen_smult_even_v16si<mask_name>): Ditto.
	(*vec_widen_smult_even_v16si<mask_name>): Ditto.
	(VEC_PERM_AVX2): Ditto.
	(one_cmpl<mode>2): Ditto.
	(<mask_codefor>one_cmpl<mode>2<mask_name>): Ditto.
	(*one_cmpl<mode>2_pternlog_false_dep): Ditto.
	(define_split to xor): Ditto.
	(*andnot<mode>3): Ditto.
	(define_split for ior): Ditto.
	(*iornot<mode>3): Ditto.
	(*xnor<mode>3): Ditto.
	(*<nlogic><mode>3): Ditto.
	(<mask_codefor>avx512f_interleave_highv16si<mask_name>): Ditto.
	(<mask_codefor>avx512f_interleave_lowv16si<mask_name>): Ditto.
	(avx512f_pshufdv3_mask): Ditto.
	(avx512f_pshufd_1<mask_name>): Ditto.
	(*vec_extractv4ti): Ditto.
	(VEXTRACTI128_MODE): Ditto.
	(define_split to vec_extract): Ditto.
	(VI1248_AVX512VL_AVX512BW): Ditto.
	(<mask_codefor>avx512f_<code>v16qiv16si2<mask_name>): Ditto.
	(<insn>v16qiv16si2): Ditto.
	(avx512f_<code>v16hiv16si2<mask_name>): Ditto.
	(<insn>v16hiv16si2): Ditto.
	(avx512f_zero_extendv16hiv16si2_1): Ditto.
	(avx512f_<code>v8qiv8di2<mask_name>): Ditto.
	(*avx512f_<code>v8qiv8di2<mask_name>_1): Ditto.
	(*avx512f_<code>v8qiv8di2<mask_name>_2): Ditto.
	(<insn>v8qiv8di2): Ditto.
	(avx512f_<code>v8hiv8di2<mask_name>): Ditto.
	(<insn>v8hiv8di2): Ditto.
	(avx512f_<code>v8siv8di2<mask_name>): Ditto.
	(*avx512f_zero_extendv8siv8di2_1): Ditto.
	(*avx512f_zero_extendv8siv8di2_2): Ditto.
	(<insn>v8siv8di2): Ditto.
	(avx512f_roundps512_sfix): Ditto.
	(vashrv8di3): Ditto.
	(vashrv16si3): Ditto.
	(pbroadcast_evex_isa): Change isa attribute to avx512f_512.
	(vec_dupv4sf): Add TARGET_EVEX512.
	(*vec_dupv4si): Ditto.
	(*vec_dupv2di): Ditto.
	(vec_dup<mode>): Change isa attribute to avx512f_512.
	(VPERMI2): Add TARGET_EVEX512.
	(VPERMI2I): Ditto.
	(VEC_INIT_MODE): Ditto.
	(VEC_INIT_HALF_MODE): Ditto.
	(<mask_codefor>avx512f_vcvtph2ps512<mask_name><round_saeonly_name>):
	Ditto.
	(avx512f_vcvtps2ph512_mask_sae): Ditto.
	(<mask_codefor>avx512f_vcvtps2ph512<mask_name><round_saeonly_name>):
	Ditto.
	(*avx512f_vcvtps2ph512<merge_mask_name>): Ditto.
	(INT_BROADCAST_MODE): Ditto.

gcc/testsuite/ChangeLog:

	* gcc.target/i386/pr89229-5b.c: Modify message of
	scan-assembler.
	* gcc.target/i386/pr89229-6b.c: Ditto.
	* gcc.target/i386/pr89229-7b.c: Ditto.
---
 gcc/config/i386/i386-builtins.cc           |  24 +-
 gcc/config/i386/i386-expand.cc             |  12 +-
 gcc/config/i386/i386.cc                    | 101 ++--
 gcc/config/i386/i386.md                    |  81 ++-
 gcc/config/i386/predicates.md              |   3 +-
 gcc/config/i386/sse.md                     | 553 +++++++++++----------
 gcc/testsuite/gcc.target/i386/pr89229-5b.c |   2 +-
 gcc/testsuite/gcc.target/i386/pr89229-6b.c |   2 +-
 gcc/testsuite/gcc.target/i386/pr89229-7b.c |   2 +-
 9 files changed, 445 insertions(+), 335 deletions(-)
diff mbox series

Patch

diff --git a/gcc/config/i386/i386-builtins.cc b/gcc/config/i386/i386-builtins.cc
index e1d1dac2ba2..70538fbe17b 100644
--- a/gcc/config/i386/i386-builtins.cc
+++ b/gcc/config/i386/i386-builtins.cc
@@ -1675,6 +1675,10 @@  ix86_vectorize_builtin_gather (const_tree mem_vectype,
 {
   bool si;
   enum ix86_builtins code;
+  const machine_mode mode = TYPE_MODE (TREE_TYPE (mem_vectype));
+
+  if ((!TARGET_AVX512F || !TARGET_EVEX512) && GET_MODE_SIZE (mode) == 64)
+    return NULL_TREE;
 
   if (! TARGET_AVX2
       || (known_eq (TYPE_VECTOR_SUBPARTS (mem_vectype), 2u)
@@ -1755,28 +1759,16 @@  ix86_vectorize_builtin_gather (const_tree mem_vectype,
 	code = si ? IX86_BUILTIN_GATHERSIV8SI : IX86_BUILTIN_GATHERALTDIV8SI;
       break;
     case E_V8DFmode:
-      if (TARGET_AVX512F)
-	code = si ? IX86_BUILTIN_GATHER3ALTSIV8DF : IX86_BUILTIN_GATHER3DIV8DF;
-      else
-	return NULL_TREE;
+      code = si ? IX86_BUILTIN_GATHER3ALTSIV8DF : IX86_BUILTIN_GATHER3DIV8DF;
       break;
     case E_V8DImode:
-      if (TARGET_AVX512F)
-	code = si ? IX86_BUILTIN_GATHER3ALTSIV8DI : IX86_BUILTIN_GATHER3DIV8DI;
-      else
-	return NULL_TREE;
+      code = si ? IX86_BUILTIN_GATHER3ALTSIV8DI : IX86_BUILTIN_GATHER3DIV8DI;
       break;
     case E_V16SFmode:
-      if (TARGET_AVX512F)
-	code = si ? IX86_BUILTIN_GATHER3SIV16SF : IX86_BUILTIN_GATHER3ALTDIV16SF;
-      else
-	return NULL_TREE;
+      code = si ? IX86_BUILTIN_GATHER3SIV16SF : IX86_BUILTIN_GATHER3ALTDIV16SF;
       break;
     case E_V16SImode:
-      if (TARGET_AVX512F)
-	code = si ? IX86_BUILTIN_GATHER3SIV16SI : IX86_BUILTIN_GATHER3ALTDIV16SI;
-      else
-	return NULL_TREE;
+      code = si ? IX86_BUILTIN_GATHER3SIV16SI : IX86_BUILTIN_GATHER3ALTDIV16SI;
       break;
     default:
       return NULL_TREE;
diff --git a/gcc/config/i386/i386-expand.cc b/gcc/config/i386/i386-expand.cc
index 6eedcb384c0..0705e08d38c 100644
--- a/gcc/config/i386/i386-expand.cc
+++ b/gcc/config/i386/i386-expand.cc
@@ -3943,7 +3943,7 @@  ix86_valid_mask_cmp_mode (machine_mode mode)
   if ((inner_mode == QImode || inner_mode == HImode) && !TARGET_AVX512BW)
     return false;
 
-  return vector_size == 64 || TARGET_AVX512VL;
+  return (vector_size == 64 && TARGET_EVEX512) || TARGET_AVX512VL;
 }
 
 /* Return true if integer mask comparison should be used.  */
@@ -4773,7 +4773,7 @@  ix86_expand_int_sse_cmp (rtx dest, enum rtx_code code, rtx cop0, rtx cop1,
 	      && GET_MODE_SIZE (GET_MODE_INNER (mode)) >= 4
 	      /* Don't do it if not using integer masks and we'd end up with
 		 the right values in the registers though.  */
-	      && (GET_MODE_SIZE (mode) == 64
+	      && ((GET_MODE_SIZE (mode) == 64 && TARGET_EVEX512)
 		  || !vector_all_ones_operand (optrue, data_mode)
 		  || opfalse != CONST0_RTX (data_mode))))
 	{
@@ -15668,6 +15668,9 @@  ix86_expand_vector_init_one_nonzero (bool mmx_ok, machine_mode mode,
   bool use_vector_set = false;
   rtx (*gen_vec_set_0) (rtx, rtx, rtx) = NULL;
 
+  if (GET_MODE_SIZE (mode) == 64 && !TARGET_EVEX512)
+    return false;
+
   switch (mode)
     {
     case E_V2DImode:
@@ -18288,7 +18291,7 @@  ix86_emit_swsqrtsf (rtx res, rtx a, machine_mode mode, bool recip)
 
   unsigned vector_size = GET_MODE_SIZE (mode);
   if (TARGET_FMA
-      || (TARGET_AVX512F && vector_size == 64)
+      || (TARGET_AVX512F && TARGET_EVEX512 && vector_size == 64)
       || (TARGET_AVX512VL && (vector_size == 32 || vector_size == 16)))
     emit_insn (gen_rtx_SET (e2,
 			    gen_rtx_FMA (mode, e0, x0, mthree)));
@@ -23005,6 +23008,9 @@  ix86_vectorize_vec_perm_const (machine_mode vmode, machine_mode op_mode,
   unsigned int i, nelt, which;
   bool two_args;
 
+  if (GET_MODE_SIZE (vmode) == 64 && !TARGET_EVEX512)
+    return false;
+
   /* For HF mode vector, convert it to HI using subreg.  */
   if (GET_MODE_INNER (vmode) == HFmode)
     {
diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc
index 0df3bf10547..635dd85e764 100644
--- a/gcc/config/i386/i386.cc
+++ b/gcc/config/i386/i386.cc
@@ -5263,7 +5263,7 @@  standard_sse_constant_p (rtx x, machine_mode pred_mode)
       switch (GET_MODE_SIZE (mode))
 	{
 	case 64:
-	  if (TARGET_AVX512F)
+	  if (TARGET_AVX512F && TARGET_EVEX512)
 	    return 2;
 	  break;
 	case 32:
@@ -5313,9 +5313,14 @@  standard_sse_constant_opcode (rtx_insn *insn, rtx *operands)
 	case MODE_XI:
 	case MODE_OI:
 	  if (EXT_REX_SSE_REG_P (operands[0]))
-	    return (TARGET_AVX512VL
-		    ? "vpxord\t%x0, %x0, %x0"
-		    : "vpxord\t%g0, %g0, %g0");
+	    {
+	      if (TARGET_AVX512VL)
+		return "vpxord\t%x0, %x0, %x0";
+	      else if (TARGET_EVEX512)
+		return "vpxord\t%g0, %g0, %g0";
+	      else
+		gcc_unreachable ();
+	    }
 	  return "vpxor\t%x0, %x0, %x0";
 
 	case MODE_V2DF:
@@ -5324,16 +5329,23 @@  standard_sse_constant_opcode (rtx_insn *insn, rtx *operands)
 	  /* FALLTHRU */
 	case MODE_V8DF:
 	case MODE_V4DF:
-	  if (!EXT_REX_SSE_REG_P (operands[0]))
-	    return "vxorpd\t%x0, %x0, %x0";
-	  else if (TARGET_AVX512DQ)
-	    return (TARGET_AVX512VL
-		    ? "vxorpd\t%x0, %x0, %x0"
-		    : "vxorpd\t%g0, %g0, %g0");
-	  else
-	    return (TARGET_AVX512VL
-		    ? "vpxorq\t%x0, %x0, %x0"
-		    : "vpxorq\t%g0, %g0, %g0");
+	  if (EXT_REX_SSE_REG_P (operands[0]))
+	    {
+	      if (TARGET_AVX512DQ)
+		return (TARGET_AVX512VL
+			? "vxorpd\t%x0, %x0, %x0"
+			: "vxorpd\t%g0, %g0, %g0");
+	      else
+		{
+		  if (TARGET_AVX512VL)
+		    return "vpxorq\t%x0, %x0, %x0";
+		  else if (TARGET_EVEX512)
+		    return "vpxorq\t%g0, %g0, %g0";
+		  else
+		    gcc_unreachable ();
+		}
+	    }
+	  return "vxorpd\t%x0, %x0, %x0";
 
 	case MODE_V4SF:
 	  if (!EXT_REX_SSE_REG_P (operands[0]))
@@ -5341,16 +5353,23 @@  standard_sse_constant_opcode (rtx_insn *insn, rtx *operands)
 	  /* FALLTHRU */
 	case MODE_V16SF:
 	case MODE_V8SF:
-	  if (!EXT_REX_SSE_REG_P (operands[0]))
-	    return "vxorps\t%x0, %x0, %x0";
-	  else if (TARGET_AVX512DQ)
-	    return (TARGET_AVX512VL
-		    ? "vxorps\t%x0, %x0, %x0"
-		    : "vxorps\t%g0, %g0, %g0");
-	  else
-	    return (TARGET_AVX512VL
-		    ? "vpxord\t%x0, %x0, %x0"
-		    : "vpxord\t%g0, %g0, %g0");
+	  if (EXT_REX_SSE_REG_P (operands[0]))
+	    {
+	      if (TARGET_AVX512DQ)
+		return (TARGET_AVX512VL
+			? "vxorps\t%x0, %x0, %x0"
+			: "vxorps\t%g0, %g0, %g0");
+	      else
+		{
+		  if (TARGET_AVX512VL)
+		    return "vpxord\t%x0, %x0, %x0";
+		  else if (TARGET_EVEX512)
+		    return "vpxord\t%g0, %g0, %g0";
+		  else
+		    gcc_unreachable ();
+		}
+	    }
+	  return "vxorps\t%x0, %x0, %x0";
 
 	default:
 	  gcc_unreachable ();
@@ -5368,7 +5387,7 @@  standard_sse_constant_opcode (rtx_insn *insn, rtx *operands)
 	case MODE_XI:
 	case MODE_V8DF:
 	case MODE_V16SF:
-	  gcc_assert (TARGET_AVX512F);
+	  gcc_assert (TARGET_AVX512F && TARGET_EVEX512);
 	  return "vpternlogd\t{$0xFF, %g0, %g0, %g0|%g0, %g0, %g0, 0xFF}";
 
 	case MODE_OI:
@@ -5380,14 +5399,18 @@  standard_sse_constant_opcode (rtx_insn *insn, rtx *operands)
 	case MODE_V2DF:
 	case MODE_V4SF:
 	  gcc_assert (TARGET_SSE2);
-	  if (!EXT_REX_SSE_REG_P (operands[0]))
-	    return (TARGET_AVX
-		    ? "vpcmpeqd\t%0, %0, %0"
-		    : "pcmpeqd\t%0, %0");
-	  else if (TARGET_AVX512VL)
-	    return "vpternlogd\t{$0xFF, %0, %0, %0|%0, %0, %0, 0xFF}";
-	  else
-	    return "vpternlogd\t{$0xFF, %g0, %g0, %g0|%g0, %g0, %g0, 0xFF}";
+	  if (EXT_REX_SSE_REG_P (operands[0]))
+	    {
+	      if (TARGET_AVX512VL)
+		return "vpternlogd\t{$0xFF, %0, %0, %0|%0, %0, %0, 0xFF}";
+	      else if (TARGET_EVEX512)
+		return "vpternlogd\t{$0xFF, %g0, %g0, %g0|%g0, %g0, %g0, 0xFF}";
+	      else
+		gcc_unreachable ();
+	    }
+	  return (TARGET_AVX
+		  ? "vpcmpeqd\t%0, %0, %0"
+		  : "pcmpeqd\t%0, %0");
 
 	default:
 	  gcc_unreachable ();
@@ -5397,7 +5420,7 @@  standard_sse_constant_opcode (rtx_insn *insn, rtx *operands)
     {
       if (GET_MODE_SIZE (mode) == 64)
 	{
-	  gcc_assert (TARGET_AVX512F);
+	  gcc_assert (TARGET_AVX512F && TARGET_EVEX512);
 	  return "vpcmpeqd\t%t0, %t0, %t0";
 	}
       else if (GET_MODE_SIZE (mode) == 32)
@@ -5409,7 +5432,7 @@  standard_sse_constant_opcode (rtx_insn *insn, rtx *operands)
     }
   else if (vector_all_ones_zero_extend_quarter_operand (x, mode))
     {
-      gcc_assert (TARGET_AVX512F);
+      gcc_assert (TARGET_AVX512F && TARGET_EVEX512);
       return "vpcmpeqd\t%x0, %x0, %x0";
     }
 
@@ -5511,6 +5534,8 @@  ix86_get_ssemov (rtx *operands, unsigned size,
 	  || memory_operand (operands[1], mode))
 	gcc_unreachable ();
       size = 64;
+      /* We need TARGET_EVEX512 to move into zmm register.  */
+      gcc_assert (TARGET_EVEX512);
       switch (type)
 	{
 	case opcode_int:
@@ -10727,7 +10752,7 @@  ix86_legitimate_constant_p (machine_mode mode, rtx x)
 	case E_OImode:
 	case E_XImode:
 	  if (!standard_sse_constant_p (x, mode)
-	      && GET_MODE_SIZE (TARGET_AVX512F
+	      && GET_MODE_SIZE (TARGET_AVX512F && TARGET_EVEX512
 				? XImode
 				: (TARGET_AVX
 				   ? OImode
@@ -19195,10 +19220,14 @@  ix86_vectorize_builtin_scatter (const_tree vectype,
 {
   bool si;
   enum ix86_builtins code;
+  const machine_mode mode = TYPE_MODE (TREE_TYPE (vectype));
 
   if (!TARGET_AVX512F)
     return NULL_TREE;
 
+  if (!TARGET_EVEX512 && GET_MODE_SIZE (mode) == 64)
+    return NULL_TREE;
+
   if (known_eq (TYPE_VECTOR_SUBPARTS (vectype), 2u)
       ? !TARGET_USE_SCATTER_2PARTS
       : (known_eq (TYPE_VECTOR_SUBPARTS (vectype), 4u)
diff --git a/gcc/config/i386/i386.md b/gcc/config/i386/i386.md
index eef8a0e01eb..6eb4e540140 100644
--- a/gcc/config/i386/i386.md
+++ b/gcc/config/i386/i386.md
@@ -535,10 +535,11 @@ 
 (define_attr "isa" "base,x64,nox64,x64_sse2,x64_sse4,x64_sse4_noavx,
 		    x64_avx,x64_avx512bw,x64_avx512dq,aes,
 		    sse_noavx,sse2,sse2_noavx,sse3,sse3_noavx,sse4,sse4_noavx,
-		    avx,noavx,avx2,noavx2,bmi,bmi2,fma4,fma,avx512f,noavx512f,
-		    avx512bw,noavx512bw,avx512dq,noavx512dq,fma_or_avx512vl,
-		    avx512vl,noavx512vl,avxvnni,avx512vnnivl,avx512fp16,avxifma,
-		    avx512ifmavl,avxneconvert,avx512bf16vl,vpclmulqdqvl"
+		    avx,noavx,avx2,noavx2,bmi,bmi2,fma4,fma,avx512f,avx512f_512,
+		    noavx512f,avx512bw,noavx512bw,avx512dq,noavx512dq,
+		    fma_or_avx512vl,avx512vl,noavx512vl,avxvnni,avx512vnnivl,
+		    avx512fp16,avxifma,avx512ifmavl,avxneconvert,avx512bf16vl,
+		    vpclmulqdqvl"
   (const_string "base"))
 
 ;; The (bounding maximum) length of an instruction immediate.
@@ -899,6 +900,8 @@ 
 	 (eq_attr "isa" "fma_or_avx512vl")
 	   (symbol_ref "TARGET_FMA || TARGET_AVX512VL")
 	 (eq_attr "isa" "avx512f") (symbol_ref "TARGET_AVX512F")
+	 (eq_attr "isa" "avx512f_512")
+	   (symbol_ref "TARGET_AVX512F && TARGET_EVEX512")
 	 (eq_attr "isa" "noavx512f") (symbol_ref "!TARGET_AVX512F")
 	 (eq_attr "isa" "avx512bw") (symbol_ref "TARGET_AVX512BW")
 	 (eq_attr "isa" "noavx512bw") (symbol_ref "!TARGET_AVX512BW")
@@ -2281,7 +2284,7 @@ 
 (define_expand "movxi"
   [(set (match_operand:XI 0 "nonimmediate_operand")
 	(match_operand:XI 1 "general_operand"))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "ix86_expand_vector_move (XImode, operands); DONE;")
 
 (define_expand "movoi"
@@ -2357,7 +2360,7 @@ 
 (define_insn "*movxi_internal_avx512f"
   [(set (match_operand:XI 0 "nonimmediate_operand"		"=v,v ,v ,m")
 	(match_operand:XI 1 "nonimmediate_or_sse_const_operand" " C,BC,vm,v"))]
-  "TARGET_AVX512F
+  "TARGET_AVX512F && TARGET_EVEX512
    && (register_operand (operands[0], XImode)
        || register_operand (operands[1], XImode))"
 {
@@ -2485,9 +2488,9 @@ 
 
 (define_insn "*movdi_internal"
   [(set (match_operand:DI 0 "nonimmediate_operand"
-    "=r  ,o  ,r,r  ,r,m ,*y,*y,?*y,?m,?r,?*y,?v,?v,?v,m ,m,?r ,?*Yd,?r,?v,?*y,?*x,*k,*k  ,*r,*m,*k")
+    "=r  ,o  ,r,r  ,r,m ,*y,*y,?*y,?m,?r,?*y,?Yv,?v,?v,m ,m,?r ,?*Yd,?r,?v,?*y,?*x,*k,*k  ,*r,*m,*k")
 	(match_operand:DI 1 "general_operand"
-    "riFo,riF,Z,rem,i,re,C ,*y,Bk ,*y,*y,r  ,C ,?v,Bk,?v,v,*Yd,r   ,?v,r  ,*x ,*y ,*r,*kBk,*k,*k,CBC"))]
+    "riFo,riF,Z,rem,i,re,C ,*y,Bk ,*y,*y,r  ,C  ,?v,Bk,?v,v,*Yd,r   ,?v,r  ,*x ,*y ,*r,*kBk,*k,*k,CBC"))]
   "!(MEM_P (operands[0]) && MEM_P (operands[1]))
    && ix86_hardreg_mov_ok (operands[0], operands[1])"
 {
@@ -2605,7 +2608,7 @@ 
    (set (attr "mode")
      (cond [(eq_attr "alternative" "2")
 	      (const_string "SI")
-	    (eq_attr "alternative" "12,13")
+	    (eq_attr "alternative" "12")
 	      (cond [(match_test "TARGET_AVX")
 		       (const_string "TI")
 		     (ior (not (match_test "TARGET_SSE2"))
@@ -2613,6 +2616,18 @@ 
 		       (const_string "V4SF")
 		    ]
 		    (const_string "TI"))
+	    (eq_attr "alternative" "13")
+	      (cond [(match_test "TARGET_AVX512VL")
+		       (const_string "TI")
+		     (match_test "TARGET_AVX512F")
+		       (const_string "DF")
+		     (match_test "TARGET_AVX")
+		       (const_string "TI")
+		     (ior (not (match_test "TARGET_SSE2"))
+			  (match_test "optimize_function_for_size_p (cfun)"))
+		       (const_string "V4SF")
+		    ]
+		    (const_string "TI"))
 
 	    (and (eq_attr "alternative" "14,15,16")
 		 (not (match_test "TARGET_SSE2")))
@@ -2706,9 +2721,9 @@ 
 
 (define_insn "*movsi_internal"
   [(set (match_operand:SI 0 "nonimmediate_operand"
-    "=r,m ,*y,*y,?*y,?m,?r,?*y,?v,?v,?v,m ,?r,?v,*k,*k  ,*rm,*k")
+    "=r,m ,*y,*y,?*y,?m,?r,?*y,?Yv,?v,?v,m ,?r,?v,*k,*k  ,*rm,*k")
 	(match_operand:SI 1 "general_operand"
-    "g ,re,C ,*y,Bk ,*y,*y,r  ,C ,?v,Bk,?v,?v,r  ,*r,*kBk,*k ,CBC"))]
+    "g ,re,C ,*y,Bk ,*y,*y,r  ,C  ,?v,Bk,?v,?v,r  ,*r,*kBk,*k ,CBC"))]
   "!(MEM_P (operands[0]) && MEM_P (operands[1]))
    && ix86_hardreg_mov_ok (operands[0], operands[1])"
 {
@@ -2793,7 +2808,7 @@ 
    (set (attr "mode")
      (cond [(eq_attr "alternative" "2,3")
 	      (const_string "DI")
-	    (eq_attr "alternative" "8,9")
+	    (eq_attr "alternative" "8")
 	      (cond [(match_test "TARGET_AVX")
 		       (const_string "TI")
 		     (ior (not (match_test "TARGET_SSE2"))
@@ -2801,6 +2816,18 @@ 
 		       (const_string "V4SF")
 		    ]
 		    (const_string "TI"))
+	    (eq_attr "alternative" "9")
+	      (cond [(match_test "TARGET_AVX512VL")
+		       (const_string "TI")
+		     (match_test "TARGET_AVX512F")
+		       (const_string "SF")
+		     (match_test "TARGET_AVX")
+		       (const_string "TI")
+		     (ior (not (match_test "TARGET_SSE2"))
+			  (match_test "optimize_function_for_size_p (cfun)"))
+		       (const_string "V4SF")
+		    ]
+		    (const_string "TI"))
 
 	    (and (eq_attr "alternative" "10,11")
 	         (not (match_test "TARGET_SSE2")))
@@ -2849,9 +2876,9 @@ 
 
 (define_insn "*movhi_internal"
   [(set (match_operand:HI 0 "nonimmediate_operand"
-    "=r,r,r,m ,*k,*k ,r ,m ,*k ,?r,?*v,*v,*v,*v,m")
+    "=r,r,r,m ,*k,*k ,r ,m ,*k ,?r,?*v,*Yv,*v,*v,m")
 	(match_operand:HI 1 "general_operand"
-    "r ,n,m,rn,r ,*km,*k,*k,CBC,*v,r  ,C ,*v,m ,*v"))]
+    "r ,n,m,rn,r ,*km,*k,*k,CBC,*v,r  ,C  ,*v,m ,*v"))]
   "!(MEM_P (operands[0]) && MEM_P (operands[1]))
    && ix86_hardreg_mov_ok (operands[0], operands[1])"
 {
@@ -3993,9 +4020,9 @@ 
 ;; Possible store forwarding (partial memory) stall in alternatives 4, 6 and 7.
 (define_insn "*movdf_internal"
   [(set (match_operand:DF 0 "nonimmediate_operand"
-    "=Yf*f,m   ,Yf*f,?r ,!o,?*r ,!o,!o,?r,?m,?r,?r,v,v,v,m,*x,*x,*x,m ,?r,?v,r  ,o ,r  ,m")
+    "=Yf*f,m   ,Yf*f,?r ,!o,?*r ,!o,!o,?r,?m,?r,?r,Yv,v,v,m,*x,*x,*x,m ,?r,?v,r  ,o ,r  ,m")
 	(match_operand:DF 1 "general_operand"
-    "Yf*fm,Yf*f,G   ,roF,r ,*roF,*r,F ,rm,rC,C ,F ,C,v,m,v,C ,*x,m ,*x, v, r,roF,rF,rmF,rC"))]
+    "Yf*fm,Yf*f,G   ,roF,r ,*roF,*r,F ,rm,rC,C ,F ,C ,v,m,v,C ,*x,m ,*x, v, r,roF,rF,rmF,rC"))]
   "!(MEM_P (operands[0]) && MEM_P (operands[1]))
    && (lra_in_progress || reload_completed
        || !CONST_DOUBLE_P (operands[1])
@@ -4170,9 +4197,9 @@ 
 
 (define_insn "*movsf_internal"
   [(set (match_operand:SF 0 "nonimmediate_operand"
-	  "=Yf*f,m   ,Yf*f,?r ,?m,v,v,v,m,?r,?v,!*y,!*y,!m,!r,!*y,r  ,m")
+	  "=Yf*f,m   ,Yf*f,?r ,?m,Yv,v,v,m,?r,?v,!*y,!*y,!m,!r,!*y,r  ,m")
 	(match_operand:SF 1 "general_operand"
-	  "Yf*fm,Yf*f,G   ,rmF,rF,C,v,m,v,v ,r ,*y ,m  ,*y,*y,r  ,rmF,rF"))]
+	  "Yf*fm,Yf*f,G   ,rmF,rF,C ,v,m,v,v ,r ,*y ,m  ,*y,*y,r  ,rmF,rF"))]
   "!(MEM_P (operands[0]) && MEM_P (operands[1]))
    && (lra_in_progress || reload_completed
        || !CONST_DOUBLE_P (operands[1])
@@ -4247,7 +4274,7 @@ 
 	       (eq_attr "alternative" "11")
 		 (const_string "DI")
 	       (eq_attr "alternative" "5")
-		 (cond [(and (match_test "TARGET_AVX512F")
+		 (cond [(and (match_test "TARGET_AVX512F && TARGET_EVEX512")
 			     (not (match_test "TARGET_PREFER_AVX256")))
 			  (const_string "V16SF")
 			(match_test "TARGET_AVX")
@@ -4271,7 +4298,11 @@ 
 		  better to maintain the whole registers in single format
 		  to avoid problems on using packed logical operations.  */
 	       (eq_attr "alternative" "6")
-		 (cond [(ior (match_test "TARGET_SSE_PARTIAL_REG_DEPENDENCY")
+		 (cond [(match_test "TARGET_AVX512VL")
+			  (const_string "V4SF")
+			(match_test "TARGET_AVX512F")
+			  (const_string "SF")
+			(ior (match_test "TARGET_SSE_PARTIAL_REG_DEPENDENCY")
 			     (match_test "TARGET_SSE_SPLIT_REGS"))
 			  (const_string "V4SF")
 		       ]
@@ -4301,9 +4332,9 @@ 
 
 (define_insn "*mov<mode>_internal"
  [(set (match_operand:HFBF 0 "nonimmediate_operand"
-	 "=?r,?r,?r,?m,v,v,?r,m,?v,v")
+	 "=?r,?r,?r,?m		 ,Yv,v,?r,m,?v,v")
        (match_operand:HFBF 1 "general_operand"
-	 "r  ,F ,m ,r<hfbfconstf>,C,v, v,v,r ,m"))]
+	 "r  ,F ,m ,r<hfbfconstf>,C ,v, v,v,r ,m"))]
  "!(MEM_P (operands[0]) && MEM_P (operands[1]))
   && (lra_in_progress
       || reload_completed
@@ -5144,7 +5175,7 @@ 
    && optimize_insn_for_speed_p ()
    && reload_completed
    && (!EXT_REX_SSE_REG_P (operands[0])
-       || TARGET_AVX512VL)"
+       || TARGET_AVX512VL || TARGET_EVEX512)"
    [(set (match_dup 2)
 	 (float_extend:V2DF
 	   (vec_select:V2SF
@@ -5287,8 +5318,8 @@ 
    (set_attr "memory" "none")
    (set (attr "enabled")
      (if_then_else (eq_attr "alternative" "2")
-       (symbol_ref "TARGET_AVX512F && !TARGET_AVX512VL
-		    && !TARGET_PREFER_AVX256")
+       (symbol_ref "TARGET_AVX512F && TARGET_EVEX512
+		    && !TARGET_AVX512VL && !TARGET_PREFER_AVX256")
        (const_string "*")))])
 
 (define_expand "extend<mode>xf2"
diff --git a/gcc/config/i386/predicates.md b/gcc/config/i386/predicates.md
index 37d20c6303a..ef49efdbde5 100644
--- a/gcc/config/i386/predicates.md
+++ b/gcc/config/i386/predicates.md
@@ -1276,7 +1276,8 @@ 
   (and (match_code "vec_duplicate")
        (and (match_test "TARGET_AVX512F")
 	    (ior (match_test "TARGET_AVX512VL")
-		 (match_test "GET_MODE_SIZE (GET_MODE (op)) == 64")))
+		 (and (match_test "GET_MODE_SIZE (GET_MODE (op)) == 64")
+		      (match_test "TARGET_EVEX512"))))
        (match_test "VALID_BCST_MODE_P (GET_MODE_INNER (GET_MODE (op)))")
        (match_test "GET_MODE (XEXP (op, 0))
 		    == GET_MODE_INNER (GET_MODE (op))")
diff --git a/gcc/config/i386/sse.md b/gcc/config/i386/sse.md
index 80b43fd7db7..8d1b75b43e0 100644
--- a/gcc/config/i386/sse.md
+++ b/gcc/config/i386/sse.md
@@ -253,43 +253,43 @@ 
 
 ;; All vector modes including V?TImode, used in move patterns.
 (define_mode_iterator VMOVE
-  [(V64QI "TARGET_AVX512F") (V32QI "TARGET_AVX") V16QI
-   (V32HI "TARGET_AVX512F") (V16HI "TARGET_AVX") V8HI
-   (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX") V4SI
-   (V8DI "TARGET_AVX512F")  (V4DI "TARGET_AVX") V2DI
-   (V4TI "TARGET_AVX512F") (V2TI "TARGET_AVX") V1TI
-   (V32HF "TARGET_AVX512F") (V16HF "TARGET_AVX") V8HF
-   (V32BF "TARGET_AVX512F") (V16BF "TARGET_AVX") V8BF
-   (V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF
-   (V8DF "TARGET_AVX512F")  (V4DF "TARGET_AVX") V2DF])
+  [(V64QI "TARGET_AVX512F && TARGET_EVEX512") (V32QI "TARGET_AVX") V16QI
+   (V32HI "TARGET_AVX512F && TARGET_EVEX512") (V16HI "TARGET_AVX") V8HI
+   (V16SI "TARGET_AVX512F && TARGET_EVEX512") (V8SI "TARGET_AVX") V4SI
+   (V8DI "TARGET_AVX512F && TARGET_EVEX512")  (V4DI "TARGET_AVX") V2DI
+   (V4TI "TARGET_AVX512F && TARGET_EVEX512") (V2TI "TARGET_AVX") V1TI
+   (V32HF "TARGET_AVX512F && TARGET_EVEX512") (V16HF "TARGET_AVX") V8HF
+   (V32BF "TARGET_AVX512F && TARGET_EVEX512") (V16BF "TARGET_AVX") V8BF
+   (V16SF "TARGET_AVX512F && TARGET_EVEX512") (V8SF "TARGET_AVX") V4SF
+   (V8DF "TARGET_AVX512F && TARGET_EVEX512")  (V4DF "TARGET_AVX") V2DF])
 
 ;; All AVX-512{F,VL} vector modes without HF. Supposed TARGET_AVX512F baseline.
 (define_mode_iterator V48_AVX512VL
-  [V16SI (V8SI "TARGET_AVX512VL") (V4SI "TARGET_AVX512VL")
-   V8DI  (V4DI "TARGET_AVX512VL") (V2DI "TARGET_AVX512VL")
-   V16SF (V8SF "TARGET_AVX512VL") (V4SF "TARGET_AVX512VL")
-   V8DF  (V4DF "TARGET_AVX512VL") (V2DF "TARGET_AVX512VL")])
+  [(V16SI "TARGET_EVEX512") (V8SI "TARGET_AVX512VL") (V4SI "TARGET_AVX512VL")
+   (V8DI "TARGET_EVEX512") (V4DI "TARGET_AVX512VL") (V2DI "TARGET_AVX512VL")
+   (V16SF "TARGET_EVEX512") (V8SF "TARGET_AVX512VL") (V4SF "TARGET_AVX512VL")
+   (V8DF "TARGET_EVEX512") (V4DF "TARGET_AVX512VL") (V2DF "TARGET_AVX512VL")])
 
 (define_mode_iterator V48_256_512_AVX512VL
-  [V16SI (V8SI "TARGET_AVX512VL")
-   V8DI  (V4DI "TARGET_AVX512VL")
-   V16SF (V8SF "TARGET_AVX512VL")
-   V8DF  (V4DF "TARGET_AVX512VL")])
+  [(V16SI "TARGET_EVEX512") (V8SI "TARGET_AVX512VL")
+   (V8DI "TARGET_EVEX512") (V4DI "TARGET_AVX512VL")
+   (V16SF "TARGET_EVEX512") (V8SF "TARGET_AVX512VL")
+   (V8DF "TARGET_EVEX512") (V4DF "TARGET_AVX512VL")])
 
 ;; All AVX-512{F,VL} vector modes. Supposed TARGET_AVX512F baseline.
 (define_mode_iterator V48H_AVX512VL
-  [V16SI (V8SI "TARGET_AVX512VL") (V4SI "TARGET_AVX512VL")
-   V8DI  (V4DI "TARGET_AVX512VL") (V2DI "TARGET_AVX512VL")
+  [(V16SI "TARGET_EVEX512") (V8SI "TARGET_AVX512VL") (V4SI "TARGET_AVX512VL")
+   (V8DI "TARGET_EVEX512") (V4DI "TARGET_AVX512VL") (V2DI "TARGET_AVX512VL")
    (V32HF "TARGET_AVX512FP16")
    (V16HF "TARGET_AVX512FP16 && TARGET_AVX512VL")
    (V8HF "TARGET_AVX512FP16 && TARGET_AVX512VL")
-   V16SF (V8SF "TARGET_AVX512VL") (V4SF "TARGET_AVX512VL")
-   V8DF  (V4DF "TARGET_AVX512VL") (V2DF "TARGET_AVX512VL")])
+   (V16SF "TARGET_EVEX512") (V8SF "TARGET_AVX512VL") (V4SF "TARGET_AVX512VL")
+   (V8DF "TARGET_EVEX512") (V4DF "TARGET_AVX512VL") (V2DF "TARGET_AVX512VL")])
 
 ;; 1,2 byte AVX-512{BW,VL} vector modes. Supposed TARGET_AVX512BW baseline.
 (define_mode_iterator VI12_AVX512VL
-  [V64QI (V16QI "TARGET_AVX512VL") (V32QI "TARGET_AVX512VL")
-   V32HI (V16HI "TARGET_AVX512VL") (V8HI "TARGET_AVX512VL")])
+  [(V64QI "TARGET_EVEX512") (V16QI "TARGET_AVX512VL") (V32QI "TARGET_AVX512VL")
+   (V32HI "TARGET_EVEX512") (V16HI "TARGET_AVX512VL") (V8HI "TARGET_AVX512VL")])
 
 (define_mode_iterator VI12HFBF_AVX512VL
   [V64QI (V16QI "TARGET_AVX512VL") (V32QI "TARGET_AVX512VL")
@@ -302,13 +302,13 @@ 
 
 ;; All vector modes
 (define_mode_iterator V
-  [(V64QI "TARGET_AVX512F") (V32QI "TARGET_AVX") V16QI
-   (V32HI "TARGET_AVX512F") (V16HI "TARGET_AVX") V8HI
-   (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX") V4SI
-   (V8DI "TARGET_AVX512F")  (V4DI "TARGET_AVX") V2DI
-   (V32HF "TARGET_AVX512F") (V16HF "TARGET_AVX") V8HF
-   (V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF
-   (V8DF "TARGET_AVX512F")  (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")])
+  [(V64QI "TARGET_AVX512F && TARGET_EVEX512") (V32QI "TARGET_AVX") V16QI
+   (V32HI "TARGET_AVX512F && TARGET_EVEX512") (V16HI "TARGET_AVX") V8HI
+   (V16SI "TARGET_AVX512F && TARGET_EVEX512") (V8SI "TARGET_AVX") V4SI
+   (V8DI "TARGET_AVX512F && TARGET_EVEX512")  (V4DI "TARGET_AVX") V2DI
+   (V32HF "TARGET_AVX512F && TARGET_EVEX512") (V16HF "TARGET_AVX") V8HF
+   (V16SF "TARGET_AVX512F && TARGET_EVEX512") (V8SF "TARGET_AVX") V4SF
+   (V8DF "TARGET_AVX512F && TARGET_EVEX512")  (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")])
 
 ;; All 128bit vector modes
 (define_mode_iterator V_128
@@ -324,22 +324,32 @@ 
    V16HF V8HF V8SF V4SF V4DF V2DF])
 
 ;; All 512bit vector modes
-(define_mode_iterator V_512 [V64QI V32HI V16SI V8DI V16SF V8DF V32HF V32BF])
+(define_mode_iterator V_512
+  [(V64QI "TARGET_EVEX512") (V32HI "TARGET_EVEX512")
+   (V16SI "TARGET_EVEX512") (V8DI "TARGET_EVEX512")
+   (V16SF "TARGET_EVEX512") (V8DF "TARGET_EVEX512")
+   (V32HF "TARGET_EVEX512") (V32BF "TARGET_EVEX512")])
 
 ;; All 256bit and 512bit vector modes
 (define_mode_iterator V_256_512
   [V32QI V16HI V16HF V16BF V8SI V4DI V8SF V4DF
-   (V64QI "TARGET_AVX512F") (V32HI "TARGET_AVX512F") (V32HF "TARGET_AVX512F")
-   (V32BF "TARGET_AVX512F") (V16SI "TARGET_AVX512F") (V8DI "TARGET_AVX512F")
-   (V16SF "TARGET_AVX512F") (V8DF "TARGET_AVX512F")])
+   (V64QI "TARGET_AVX512F && TARGET_EVEX512")
+   (V32HI "TARGET_AVX512F && TARGET_EVEX512")
+   (V32HF "TARGET_AVX512F && TARGET_EVEX512")
+   (V32BF "TARGET_AVX512F && TARGET_EVEX512")
+   (V16SI "TARGET_AVX512F && TARGET_EVEX512")
+   (V8DI "TARGET_AVX512F && TARGET_EVEX512")
+   (V16SF "TARGET_AVX512F && TARGET_EVEX512")
+   (V8DF "TARGET_AVX512F && TARGET_EVEX512")])
 
 ;; All vector float modes
 (define_mode_iterator VF
-  [(V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF
-   (V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")])
+  [(V16SF "TARGET_AVX512F && TARGET_EVEX512") (V8SF "TARGET_AVX") V4SF
+   (V8DF "TARGET_AVX512F && TARGET_EVEX512") (V4DF "TARGET_AVX")
+   (V2DF "TARGET_SSE2")])
 
 (define_mode_iterator VF1_VF2_AVX512DQ
-  [(V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF
+  [(V16SF "TARGET_AVX512F && TARGET_EVEX512") (V8SF "TARGET_AVX") V4SF
    (V8DF "TARGET_AVX512DQ") (V4DF "TARGET_AVX512DQ && TARGET_AVX512VL")
    (V2DF "TARGET_AVX512DQ && TARGET_AVX512VL")])
 
@@ -347,14 +357,17 @@ 
   [(V32HF "TARGET_AVX512FP16")
    (V16HF "TARGET_AVX512FP16 && TARGET_AVX512VL")
    (V8HF "TARGET_AVX512FP16 && TARGET_AVX512VL")
-   (V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF
-   (V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")])
+   (V16SF "TARGET_AVX512F && TARGET_EVEX512") (V8SF "TARGET_AVX") V4SF
+   (V8DF "TARGET_AVX512F && TARGET_EVEX512") (V4DF "TARGET_AVX")
+   (V2DF "TARGET_SSE2")])
 
 ;; 128-, 256- and 512-bit float vector modes for bitwise operations
 (define_mode_iterator VFB
-  [(V32HF "TARGET_AVX512F") (V16HF "TARGET_AVX") (V8HF "TARGET_SSE2")
-   (V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF
-   (V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")])
+  [(V32HF "TARGET_AVX512F && TARGET_EVEX512")
+   (V16HF "TARGET_AVX") (V8HF "TARGET_SSE2")
+   (V16SF "TARGET_AVX512F && TARGET_EVEX512") (V8SF "TARGET_AVX") V4SF
+   (V8DF "TARGET_AVX512F && TARGET_EVEX512")
+   (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")])
 
 ;; 128- and 256-bit float vector modes
 (define_mode_iterator VF_128_256
@@ -369,10 +382,10 @@ 
 
 ;; All SFmode vector float modes
 (define_mode_iterator VF1
-  [(V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF])
+  [(V16SF "TARGET_AVX512F && TARGET_EVEX512") (V8SF "TARGET_AVX") V4SF])
 
 (define_mode_iterator VF1_AVX2
-  [(V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX2") V4SF])
+  [(V16SF "TARGET_AVX512F && TARGET_EVEX512") (V8SF "TARGET_AVX2") V4SF])
 
 ;; 128- and 256-bit SF vector modes
 (define_mode_iterator VF1_128_256
@@ -383,24 +396,24 @@ 
 
 ;; All DFmode vector float modes
 (define_mode_iterator VF2
-  [(V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX") V2DF])
+  [(V8DF "TARGET_AVX512F && TARGET_EVEX512") (V4DF "TARGET_AVX") V2DF])
 
 ;; All DFmode & HFmode vector float modes
 (define_mode_iterator VF2H
   [(V32HF "TARGET_AVX512FP16")
    (V16HF "TARGET_AVX512FP16 && TARGET_AVX512VL")
    (V8HF "TARGET_AVX512FP16 && TARGET_AVX512VL")
-   (V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX") V2DF])
+   (V8DF "TARGET_AVX512F && TARGET_EVEX512") (V4DF "TARGET_AVX") V2DF])
 
 ;; 128- and 256-bit DF vector modes
 (define_mode_iterator VF2_128_256
   [(V4DF "TARGET_AVX") V2DF])
 
 (define_mode_iterator VF2_512_256
-  [(V8DF "TARGET_AVX512F") V4DF])
+  [(V8DF "TARGET_AVX512F && TARGET_EVEX512") V4DF])
 
 (define_mode_iterator VF2_512_256VL
-  [V8DF (V4DF "TARGET_AVX512VL")])
+  [(V8DF "TARGET_EVEX512") (V4DF "TARGET_AVX512VL")])
 
 ;; All 128bit vector SF/DF modes
 (define_mode_iterator VF_128
@@ -417,30 +430,30 @@ 
 
 ;; All 512bit vector float modes
 (define_mode_iterator VF_512
-  [V16SF V8DF])
+  [(V16SF "TARGET_EVEX512") (V8DF "TARGET_EVEX512")])
 
 ;; All 512bit vector float modes for bitwise operations
 (define_mode_iterator VFB_512
-  [V32HF V16SF V8DF])
+  [(V32HF "TARGET_EVEX512") (V16SF "TARGET_EVEX512") (V8DF "TARGET_EVEX512")])
 
 (define_mode_iterator V4SF_V8HF
   [V4SF V8HF])
 
 (define_mode_iterator VI48_AVX512VL
-  [V16SI (V8SI  "TARGET_AVX512VL") (V4SI  "TARGET_AVX512VL")
-   V8DI  (V4DI  "TARGET_AVX512VL") (V2DI  "TARGET_AVX512VL")])
+  [(V16SI "TARGET_EVEX512") (V8SI "TARGET_AVX512VL") (V4SI "TARGET_AVX512VL")
+   (V8DI "TARGET_EVEX512") (V4DI "TARGET_AVX512VL") (V2DI "TARGET_AVX512VL")])
 
 (define_mode_iterator VI1248_AVX512VLBW
   [(V64QI "TARGET_AVX512BW") (V32QI "TARGET_AVX512VL && TARGET_AVX512BW")
    (V16QI "TARGET_AVX512VL && TARGET_AVX512BW")
    (V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX512VL && TARGET_AVX512BW")
    (V8HI "TARGET_AVX512VL && TARGET_AVX512BW")
-   V16SI (V8SI "TARGET_AVX512VL") (V4SI "TARGET_AVX512VL")
-   V8DI (V4DI "TARGET_AVX512VL") (V2DI "TARGET_AVX512VL")])
+   (V16SI "TARGET_EVEX512") (V8SI "TARGET_AVX512VL") (V4SI "TARGET_AVX512VL")
+   (V8DI "TARGET_EVEX512") (V4DI "TARGET_AVX512VL") (V2DI "TARGET_AVX512VL")])
 
 (define_mode_iterator VF_AVX512VL
-  [V16SF (V8SF "TARGET_AVX512VL") (V4SF "TARGET_AVX512VL")
-   V8DF (V4DF "TARGET_AVX512VL") (V2DF "TARGET_AVX512VL")])
+  [(V16SF "TARGET_EVEX512") (V8SF "TARGET_AVX512VL") (V4SF "TARGET_AVX512VL")
+   (V8DF "TARGET_EVEX512") (V4DF "TARGET_AVX512VL") (V2DF "TARGET_AVX512VL")])
 
 ;; AVX512ER SF plus 128- and 256-bit SF vector modes
 (define_mode_iterator VF1_AVX512ER_128_256
@@ -450,14 +463,14 @@ 
   [(V32HF "TARGET_AVX512FP16")
    (V16HF "TARGET_AVX512FP16 && TARGET_AVX512VL")
    (V8HF "TARGET_AVX512FP16 && TARGET_AVX512VL")
-   V16SF (V8SF "TARGET_AVX512VL") (V4SF "TARGET_AVX512VL")
-   V8DF (V4DF "TARGET_AVX512VL") (V2DF "TARGET_AVX512VL")])
+   (V16SF "TARGET_EVEX512") (V8SF "TARGET_AVX512VL") (V4SF "TARGET_AVX512VL")
+   (V8DF "TARGET_EVEX512") (V4DF "TARGET_AVX512VL") (V2DF "TARGET_AVX512VL")])
 
 (define_mode_iterator VF2_AVX512VL
   [V8DF (V4DF "TARGET_AVX512VL") (V2DF "TARGET_AVX512VL")])
 
 (define_mode_iterator VF1_AVX512VL
-  [V16SF (V8SF "TARGET_AVX512VL") (V4SF "TARGET_AVX512VL")])
+  [(V16SF "TARGET_EVEX512") (V8SF "TARGET_AVX512VL") (V4SF "TARGET_AVX512VL")])
 
 (define_mode_iterator VHFBF [V32HF V16HF V8HF V32BF V16BF V8BF])
 (define_mode_iterator VHFBF_256 [V16HF V16BF])
@@ -472,7 +485,8 @@ 
 
 ;; All vector integer modes
 (define_mode_iterator VI
-  [(V16SI "TARGET_AVX512F") (V8DI "TARGET_AVX512F")
+  [(V16SI "TARGET_AVX512F && TARGET_EVEX512")
+   (V8DI "TARGET_AVX512F && TARGET_EVEX512")
    (V64QI "TARGET_AVX512BW") (V32QI "TARGET_AVX") V16QI
    (V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX") V8HI
    (V8SI "TARGET_AVX") V4SI
@@ -480,7 +494,8 @@ 
 
 ;; All vector integer and HF modes
 (define_mode_iterator VIHFBF
-  [(V16SI "TARGET_AVX512F") (V8DI "TARGET_AVX512F")
+  [(V16SI "TARGET_AVX512F && TARGET_EVEX512")
+   (V8DI "TARGET_AVX512F && TARGET_EVEX512")
    (V64QI "TARGET_AVX512BW") (V32QI "TARGET_AVX") V16QI
    (V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX") V8HI
    (V8SI "TARGET_AVX") V4SI
@@ -491,8 +506,8 @@ 
 (define_mode_iterator VI_AVX2
   [(V64QI "TARGET_AVX512BW") (V32QI "TARGET_AVX2") V16QI
    (V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX2") V8HI
-   (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX2") V4SI
-   (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX2") V2DI])
+   (V16SI "TARGET_AVX512F && TARGET_EVEX512") (V8SI "TARGET_AVX2") V4SI
+   (V8DI "TARGET_AVX512F && TARGET_EVEX512") (V4DI "TARGET_AVX2") V2DI])
 
 ;; All QImode vector integer modes
 (define_mode_iterator VI1
@@ -510,13 +525,13 @@ 
   (V8SI "TARGET_AVX") (V4DI "TARGET_AVX")])
 
 (define_mode_iterator VI8
-  [(V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX") V2DI])
+  [(V8DI "TARGET_AVX512F && TARGET_EVEX512") (V4DI "TARGET_AVX") V2DI])
 
 (define_mode_iterator VI8_FVL
   [(V8DI "TARGET_AVX512F") V4DI (V2DI "TARGET_AVX512VL")])
 
 (define_mode_iterator VI8_AVX512VL
-  [V8DI (V4DI "TARGET_AVX512VL") (V2DI "TARGET_AVX512VL")])
+  [(V8DI "TARGET_EVEX512") (V4DI "TARGET_AVX512VL") (V2DI "TARGET_AVX512VL")])
 
 (define_mode_iterator VI8_256_512
   [V8DI (V4DI "TARGET_AVX512VL")])
@@ -544,7 +559,7 @@ 
   [(V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX2") V8HI])
 
 (define_mode_iterator VI2_AVX512F
-  [(V32HI "TARGET_AVX512F") (V16HI "TARGET_AVX2") V8HI])
+  [(V32HI "TARGET_AVX512F && TARGET_EVEX512") (V16HI "TARGET_AVX2") V8HI])
 
 (define_mode_iterator VI2_AVX512VNNIBW
   [(V32HI "TARGET_AVX512BW || TARGET_AVX512VNNI")
@@ -557,14 +572,15 @@ 
   [(V8SI "TARGET_AVX2") V4SI])
 
 (define_mode_iterator VI4_AVX512F
-  [(V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX2") V4SI])
+  [(V16SI "TARGET_AVX512F && TARGET_EVEX512") (V8SI "TARGET_AVX2") V4SI])
 
 (define_mode_iterator VI4_AVX512VL
-  [V16SI (V8SI "TARGET_AVX512VL") (V4SI "TARGET_AVX512VL")])
+  [(V16SI "TARGET_EVEX512") (V8SI "TARGET_AVX512VL") (V4SI "TARGET_AVX512VL")])
 
 (define_mode_iterator VI48_AVX512F_AVX512VL
-  [V4SI V8SI (V16SI "TARGET_AVX512F")
-   (V2DI "TARGET_AVX512VL") (V4DI "TARGET_AVX512VL") (V8DI "TARGET_AVX512F")])
+  [V4SI V8SI (V16SI "TARGET_AVX512F && TARGET_EVEX512")
+   (V2DI "TARGET_AVX512VL") (V4DI "TARGET_AVX512VL")
+   (V8DI "TARGET_AVX512F && TARGET_EVEX512")])
 
 (define_mode_iterator VI2_AVX512VL
   [(V8HI "TARGET_AVX512VL") (V16HI "TARGET_AVX512VL") V32HI])
@@ -589,21 +605,21 @@ 
   [(V4DI "TARGET_AVX2") V2DI])
 
 (define_mode_iterator VI8_AVX2_AVX512F
-  [(V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX2") V2DI])
+  [(V8DI "TARGET_AVX512F && TARGET_EVEX512") (V4DI "TARGET_AVX2") V2DI])
 
 (define_mode_iterator VI8_AVX_AVX512F
-  [(V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX")])
+  [(V8DI "TARGET_AVX512F && TARGET_EVEX512") (V4DI "TARGET_AVX")])
 
 (define_mode_iterator VI4_128_8_256
   [V4SI V4DI])
 
 ;; All V8D* modes
 (define_mode_iterator V8FI
-  [V8DF V8DI])
+  [(V8DF "TARGET_EVEX512") (V8DI "TARGET_EVEX512")])
 
 ;; All V16S* modes
 (define_mode_iterator V16FI
-  [V16SF V16SI])
+  [(V16SF "TARGET_EVEX512") (V16SI "TARGET_EVEX512")])
 
 ;; ??? We should probably use TImode instead.
 (define_mode_iterator VIMAX_AVX2_AVX512BW
@@ -630,8 +646,8 @@ 
 
 (define_mode_iterator VI124_AVX2_24_AVX512F_1_AVX512BW
   [(V64QI "TARGET_AVX512BW") (V32QI "TARGET_AVX2") V16QI
-   (V32HI "TARGET_AVX512F") (V16HI "TARGET_AVX2") V8HI
-   (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX2") V4SI])
+   (V32HI "TARGET_AVX512F && TARGET_EVEX512") (V16HI "TARGET_AVX2") V8HI
+   (V16SI "TARGET_AVX512F && TARGET_EVEX512") (V8SI "TARGET_AVX2") V4SI])
 
 (define_mode_iterator VI124_AVX2
   [(V32QI "TARGET_AVX2") V16QI
@@ -648,8 +664,8 @@ 
   [(V32HI "TARGET_AVX512BW")
    (V16HI "TARGET_AVX512VL && TARGET_AVX512BW")
    (V8HI "TARGET_AVX512VL && TARGET_AVX512BW")
-   V16SI (V8SI "TARGET_AVX512VL") (V4SI "TARGET_AVX512VL")
-   V8DI (V4DI "TARGET_AVX512VL") (V2DI "TARGET_AVX512VL")])
+   (V16SI "TARGET_EVEX512") (V8SI "TARGET_AVX512VL") (V4SI "TARGET_AVX512VL")
+   (V8DI "TARGET_EVEX512") (V4DI "TARGET_AVX512VL") (V2DI "TARGET_AVX512VL")])
 
 (define_mode_iterator VI48_AVX2
   [(V8SI "TARGET_AVX2") V4SI
@@ -663,14 +679,15 @@ 
 (define_mode_iterator VI248_AVX2_8_AVX512F_24_AVX512BW
   [(V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX2") V8HI
    (V16SI "TARGET_AVX512BW") (V8SI "TARGET_AVX2") V4SI
-   (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX2") V2DI])
+   (V8DI "TARGET_AVX512F && TARGET_EVEX512") (V4DI "TARGET_AVX2") V2DI])
 
 (define_mode_iterator VI248_AVX512BW
-  [(V32HI "TARGET_AVX512BW") V16SI V8DI])
+  [(V32HI "TARGET_AVX512BW") (V16SI "TARGET_EVEX512")
+   (V8DI "TARGET_EVEX512")])
 
 (define_mode_iterator VI248_AVX512BW_AVX512VL
   [(V32HI "TARGET_AVX512BW") 
-   (V4DI "TARGET_AVX512VL") V16SI V8DI])
+   (V4DI "TARGET_AVX512VL") (V16SI "TARGET_EVEX512") (V8DI "TARGET_EVEX512")])
 
 ;; Suppose TARGET_AVX512VL as baseline
 (define_mode_iterator VI248_AVX512BW_1
@@ -684,16 +701,16 @@ 
   V4DI V2DI])
    
 (define_mode_iterator VI48_AVX512F
-  [(V16SI "TARGET_AVX512F") V8SI V4SI
-   (V8DI "TARGET_AVX512F") V4DI V2DI])
+  [(V16SI "TARGET_AVX512F && TARGET_EVEX512") V8SI V4SI
+   (V8DI "TARGET_AVX512F && TARGET_EVEX512") V4DI V2DI])
 
 (define_mode_iterator VI48_AVX_AVX512F
-  [(V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX") V4SI
-   (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX") V2DI])
+  [(V16SI "TARGET_AVX512F && TARGET_EVEX512") (V8SI "TARGET_AVX") V4SI
+   (V8DI "TARGET_AVX512F && TARGET_EVEX512") (V4DI "TARGET_AVX") V2DI])
 
 (define_mode_iterator VI12_AVX_AVX512F
-  [ (V64QI "TARGET_AVX512F") (V32QI "TARGET_AVX") V16QI
-    (V32HI "TARGET_AVX512F") (V16HI "TARGET_AVX") V8HI])
+  [(V64QI "TARGET_AVX512F && TARGET_EVEX512") (V32QI "TARGET_AVX") V16QI
+   (V32HI "TARGET_AVX512F && TARGET_EVEX512") (V16HI "TARGET_AVX") V8HI])
 
 (define_mode_iterator V48_128_256
   [V4SF V2DF
@@ -834,7 +851,8 @@ 
 (define_mode_iterator VI248_256 [V16HI V8SI V4DI])
 (define_mode_iterator VI248_512 [V32HI V16SI V8DI])
 (define_mode_iterator VI48_128 [V4SI V2DI])
-(define_mode_iterator VI148_512 [V64QI V16SI V8DI])
+(define_mode_iterator VI148_512
+  [(V64QI "TARGET_EVEX512") (V16SI "TARGET_EVEX512") (V8DI "TARGET_EVEX512")])
 (define_mode_iterator VI148_256 [V32QI V8SI V4DI])
 (define_mode_iterator VI148_128 [V16QI V4SI V2DI])
 
@@ -844,15 +862,18 @@ 
   [V32QI V16HI V8SI
    (V64QI "TARGET_AVX512BW")
    (V32HI "TARGET_AVX512BW")
-   (V16SI "TARGET_AVX512F")])
+   (V16SI "TARGET_AVX512F && TARGET_EVEX512")])
 (define_mode_iterator VI48_256 [V8SI V4DI])
-(define_mode_iterator VI48_512 [V16SI V8DI])
+(define_mode_iterator VI48_512
+  [(V16SI "TARGET_EVEX512") (V8DI "TARGET_EVEX512")])
 (define_mode_iterator VI4_256_8_512 [V8SI V8DI])
 (define_mode_iterator VI_AVX512BW
-  [V16SI V8DI (V32HI "TARGET_AVX512BW") (V64QI "TARGET_AVX512BW")])
+  [(V16SI "TARGET_EVEX512") (V8DI "TARGET_EVEX512")
+   (V32HI "TARGET_AVX512BW") (V64QI "TARGET_AVX512BW")])
 (define_mode_iterator VIHFBF_AVX512BW
-  [V16SI V8DI (V32HI "TARGET_AVX512BW") (V64QI "TARGET_AVX512BW")
-  (V32HF "TARGET_AVX512BW") (V32BF "TARGET_AVX512BW")])
+  [(V16SI "TARGET_EVEX512") (V8DI "TARGET_EVEX512")
+   (V32HI "TARGET_AVX512BW") (V64QI "TARGET_AVX512BW")
+   (V32HF "TARGET_AVX512BW") (V32BF "TARGET_AVX512BW")])
 
 ;; Int-float size matches
 (define_mode_iterator VI2F_256_512 [V16HI V32HI V16HF V32HF V16BF V32BF])
@@ -862,12 +883,15 @@ 
 (define_mode_iterator VI8F_256 [V4DI V4DF])
 (define_mode_iterator VI4F_256_512
   [V8SI V8SF
-   (V16SI "TARGET_AVX512F") (V16SF "TARGET_AVX512F")])
+   (V16SI "TARGET_AVX512F && TARGET_EVEX512")
+   (V16SF "TARGET_AVX512F && TARGET_EVEX512")])
 (define_mode_iterator VI48F_256_512
   [V8SI V8SF
-  (V16SI "TARGET_AVX512F") (V16SF "TARGET_AVX512F")
-  (V8DI  "TARGET_AVX512F") (V8DF  "TARGET_AVX512F")
-  (V4DI  "TARGET_AVX512VL") (V4DF  "TARGET_AVX512VL")])
+  (V16SI "TARGET_AVX512F && TARGET_EVEX512")
+  (V16SF "TARGET_AVX512F && TARGET_EVEX512")
+  (V8DI "TARGET_AVX512F && TARGET_EVEX512")
+  (V8DF "TARGET_AVX512F && TARGET_EVEX512")
+  (V4DI "TARGET_AVX512VL") (V4DF  "TARGET_AVX512VL")])
 (define_mode_iterator VF48_I1248
   [V16SI V16SF V8DI V8DF V32HI V64QI])
 (define_mode_iterator VF48H_AVX512VL
@@ -877,14 +901,17 @@ 
   [V2DF V4SF])
 
 (define_mode_iterator VI48F
-  [V16SI V16SF V8DI V8DF
+  [(V16SI "TARGET_EVEX512") (V16SF "TARGET_EVEX512")
+   (V8DI "TARGET_EVEX512") (V8DF "TARGET_EVEX512")
    (V8SI "TARGET_AVX512VL") (V8SF "TARGET_AVX512VL")
    (V4DI "TARGET_AVX512VL") (V4DF "TARGET_AVX512VL")
    (V4SI "TARGET_AVX512VL") (V4SF "TARGET_AVX512VL")
    (V2DI "TARGET_AVX512VL") (V2DF "TARGET_AVX512VL")])
 (define_mode_iterator VI12_VI48F_AVX512VL
-  [(V16SI "TARGET_AVX512F") (V16SF "TARGET_AVX512F")
-   (V8DI "TARGET_AVX512F") (V8DF "TARGET_AVX512F")
+  [(V16SI "TARGET_AVX512F && TARGET_EVEX512")
+   (V16SF "TARGET_AVX512F && TARGET_EVEX512")
+   (V8DI "TARGET_AVX512F && TARGET_EVEX512")
+   (V8DF "TARGET_AVX512F && TARGET_EVEX512")
    (V8SI "TARGET_AVX512VL") (V8SF "TARGET_AVX512VL")
    (V4DI "TARGET_AVX512VL") (V4DF "TARGET_AVX512VL")
    (V4SI "TARGET_AVX512VL") (V4SF "TARGET_AVX512VL")
@@ -901,7 +928,8 @@ 
 
 (define_mode_iterator V8_128 [V8HI V8HF V8BF])
 (define_mode_iterator V16_256 [V16HI V16HF V16BF])
-(define_mode_iterator V32_512 [V32HI V32HF V32BF])
+(define_mode_iterator V32_512
+ [(V32HI "TARGET_EVEX512") (V32HF "TARGET_EVEX512") (V32BF "TARGET_EVEX512")])
 
 ;; Mapping from float mode to required SSE level
 (define_mode_attr sse
@@ -1295,7 +1323,8 @@ 
 
 ;; Mix-n-match
 (define_mode_iterator AVX256MODE2P [V8SI V8SF V4DF])
-(define_mode_iterator AVX512MODE2P [V16SI V16SF V8DF])
+(define_mode_iterator AVX512MODE2P
+  [(V16SI "TARGET_EVEX512") (V16SF "TARGET_EVEX512") (V8DF "TARGET_EVEX512")])
 
 ;; Mapping for dbpsabbw modes
 (define_mode_attr dbpsadbwmode
@@ -1897,9 +1926,11 @@ 
 (define_mode_iterator STORENT_MODE
   [(DI "TARGET_SSE2 && TARGET_64BIT") (SI "TARGET_SSE2")
    (SF "TARGET_SSE4A") (DF "TARGET_SSE4A")
-   (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX") (V2DI "TARGET_SSE2")
-   (V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF
-   (V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")])
+   (V8DI "TARGET_AVX512F && TARGET_EVEX512")
+   (V4DI "TARGET_AVX") (V2DI "TARGET_SSE2")
+   (V16SF "TARGET_AVX512F && TARGET_EVEX512") (V8SF "TARGET_AVX") V4SF
+   (V8DF "TARGET_AVX512F && TARGET_EVEX512")
+   (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")])
 
 (define_expand "storent<mode>"
   [(set (match_operand:STORENT_MODE 0 "memory_operand")
@@ -3377,9 +3408,11 @@ 
 (define_mode_iterator REDUC_PLUS_MODE
  [(V4DF "TARGET_AVX") (V8SF "TARGET_AVX")
   (V16HF "TARGET_AVX512FP16 && TARGET_AVX512VL")
-  (V8DF "TARGET_AVX512F") (V16SF "TARGET_AVX512F")
+  (V8DF "TARGET_AVX512F && TARGET_EVEX512")
+  (V16SF "TARGET_AVX512F && TARGET_EVEX512")
   (V32HF "TARGET_AVX512FP16 && TARGET_AVX512VL")
-  (V32QI "TARGET_AVX") (V64QI "TARGET_AVX512F")])
+  (V32QI "TARGET_AVX")
+  (V64QI "TARGET_AVX512F && TARGET_EVEX512")])
 
 (define_expand "reduc_plus_scal_<mode>"
  [(plus:REDUC_PLUS_MODE
@@ -3423,9 +3456,11 @@ 
    (V8SF "TARGET_AVX") (V4DF "TARGET_AVX")
    (V64QI "TARGET_AVX512BW")
    (V32HF "TARGET_AVX512FP16 && TARGET_AVX512VL")
-   (V32HI "TARGET_AVX512BW") (V16SI "TARGET_AVX512F")
-   (V8DI "TARGET_AVX512F") (V16SF "TARGET_AVX512F")
-   (V8DF "TARGET_AVX512F")])
+   (V32HI "TARGET_AVX512BW")
+   (V16SI "TARGET_AVX512F && TARGET_EVEX512")
+   (V8DI "TARGET_AVX512F && TARGET_EVEX512")
+   (V16SF "TARGET_AVX512F && TARGET_EVEX512")
+   (V8DF "TARGET_AVX512F && TARGET_EVEX512")])
 
 (define_expand "reduc_<code>_scal_<mode>"
   [(smaxmin:REDUC_SMINMAX_MODE
@@ -5035,7 +5070,7 @@ 
   output_asm_insn (buf, operands);
   return "";
 }
-  [(set_attr "isa" "noavx,avx,avx512vl,avx512f")
+  [(set_attr "isa" "noavx,avx,avx512vl,avx512f_512")
    (set_attr "type" "sselog")
    (set_attr "prefix" "orig,vex,evex,evex")
    (set (attr "mode")
@@ -5092,7 +5127,7 @@ 
   output_asm_insn (buf, operands);
   return "";
 }
-  [(set_attr "isa" "noavx,avx,avx512vl,avx512f")
+  [(set_attr "isa" "noavx,avx,avx512vl,avx512f_512")
    (set_attr "type" "sselog")
    (set (attr "prefix_data16")
      (if_then_else
@@ -5161,7 +5196,7 @@ 
   output_asm_insn (buf, operands);
   return "";
 }
-  [(set_attr "isa" "noavx,avx,avx512vl,avx512f")
+  [(set_attr "isa" "noavx,avx,avx512vl,avx512f_512")
    (set_attr "type" "sselog")
    (set_attr "prefix" "orig,vex,evex,evex")
    (set (attr "mode")
@@ -5223,7 +5258,7 @@ 
   output_asm_insn (buf, operands);
   return "";
 }
-  [(set_attr "isa" "noavx,avx,avx512vl,avx512f")
+  [(set_attr "isa" "noavx,avx,avx512vl,avx512f_512")
    (set_attr "type" "sselog")
    (set (attr "prefix_data16")
      (if_then_else
@@ -5269,8 +5304,8 @@ 
    (V2DF "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512VL")
    (V8SF "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512VL")
    (V4DF "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512VL")
-   (V16SF "TARGET_AVX512F")
-   (V8DF "TARGET_AVX512F")
+   (V16SF "TARGET_AVX512F && TARGET_EVEX512")
+   (V8DF "TARGET_AVX512F && TARGET_EVEX512")
    (HF "TARGET_AVX512FP16")
    (V8HF "TARGET_AVX512FP16 && TARGET_AVX512VL")
    (V16HF "TARGET_AVX512FP16 && TARGET_AVX512VL")
@@ -5312,8 +5347,8 @@ 
   (V2DF "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512VL")
   (V8SF "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512VL")
   (V4DF "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512VL")
-  (V16SF "TARGET_AVX512F")
-  (V8DF "TARGET_AVX512F")])
+  (V16SF "TARGET_AVX512F && TARGET_EVEX512")
+  (V8DF "TARGET_AVX512F && TARGET_EVEX512")])
 
 (define_mode_iterator FMAMODE
   [SF DF V4SF V2DF V8SF V4DF])
@@ -5387,8 +5422,10 @@ 
    (V16HF "TARGET_AVX512FP16 && TARGET_AVX512VL")
    (V8HF "TARGET_AVX512FP16 && TARGET_AVX512VL")
    (HF "TARGET_AVX512FP16")
-   SF V16SF (V8SF "TARGET_AVX512VL") (V4SF "TARGET_AVX512VL")
-   DF V8DF (V4DF "TARGET_AVX512VL") (V2DF "TARGET_AVX512VL")])
+   SF (V16SF "TARGET_EVEX512")
+   (V8SF "TARGET_AVX512VL") (V4SF "TARGET_AVX512VL")
+   DF (V8DF "TARGET_EVEX512")
+   (V4DF "TARGET_AVX512VL") (V2DF "TARGET_AVX512VL")])
 
 (define_insn "<sd_mask_codefor>fma_fmadd_<mode><sd_maskz_name><round_name>"
   [(set (match_operand:VFH_SF_AVX512VL 0 "register_operand" "=v,v,v")
@@ -8028,7 +8065,7 @@ 
 	(unspec:V16SI
 	  [(match_operand:V16SF 1 "<round_nimm_predicate>" "<round_constraint>")]
 	  UNSPEC_FIX_NOTRUNC))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vcvtps2dq\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
   [(set_attr "type" "ssecvt")
    (set_attr "prefix" "evex")
@@ -8095,7 +8132,7 @@ 
   [(set (match_operand:V16SI 0 "register_operand" "=v")
 	(any_fix:V16SI
 	  (match_operand:V16SF 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vcvttps2<fixsuffix>dq\t{<round_saeonly_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_saeonly_mask_op2>}"
   [(set_attr "type" "ssecvt")
    (set_attr "prefix" "evex")
@@ -8595,7 +8632,7 @@ 
 		       (const_int 2) (const_int 3)
 		       (const_int 4) (const_int 5)
 		       (const_int 6) (const_int 7)]))))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vcvtdq2pd\t{%t1, %0|%0, %t1}"
   [(set_attr "type" "ssecvt")
    (set_attr "prefix" "evex")
@@ -8631,7 +8668,7 @@ 
 	(unspec:V8SI
 	  [(match_operand:V8DF 1 "<round_nimm_predicate>" "<round_constraint>")]
 	  UNSPEC_FIX_NOTRUNC))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vcvtpd2dq\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
   [(set_attr "type" "ssecvt")
    (set_attr "prefix" "evex")
@@ -8789,7 +8826,7 @@ 
   [(set (match_operand:V8SI 0 "register_operand" "=v")
 	(any_fix:V8SI
 	  (match_operand:V8DF 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vcvttpd2<fixsuffix>dq\t{<round_saeonly_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_saeonly_mask_op2>}"
   [(set_attr "type" "ssecvt")
    (set_attr "prefix" "evex")
@@ -9193,7 +9230,7 @@ 
   [(set (match_operand:V8SF 0 "register_operand" "=v")
 	(float_truncate:V8SF
 	  (match_operand:V8DF 1 "<round_nimm_predicate>" "<round_constraint>")))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vcvtpd2ps\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
   [(set_attr "type" "ssecvt")
    (set_attr "prefix" "evex")
@@ -9355,7 +9392,7 @@ 
 		       (const_int 2) (const_int 3)
 		       (const_int 4) (const_int 5)
 		       (const_int 6) (const_int 7)]))))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vcvtps2pd\t{%t1, %0|%0, %t1}"
   [(set_attr "type" "ssecvt")
    (set_attr "prefix" "evex")
@@ -9540,7 +9577,7 @@ 
    (set (match_operand:V8DF 0 "register_operand")
 	(float_extend:V8DF
 	  (match_dup 2)))]
-"TARGET_AVX512F"
+"TARGET_AVX512F && TARGET_EVEX512"
 "operands[2] = gen_reg_rtx (V8SFmode);")
 
 (define_expand "vec_unpacks_lo_v4sf"
@@ -9678,7 +9715,7 @@ 
    (set (match_operand:V8DF 0 "register_operand")
 	(float:V8DF
 	  (match_dup 2)))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "operands[2] = gen_reg_rtx (V8SImode);")
 
 (define_expand "vec_unpacks_float_lo_v16si"
@@ -9690,7 +9727,7 @@ 
 		       (const_int 2) (const_int 3)
 		       (const_int 4) (const_int 5)
 		       (const_int 6) (const_int 7)]))))]
-  "TARGET_AVX512F")
+  "TARGET_AVX512F && TARGET_EVEX512")
 
 (define_expand "vec_unpacku_float_hi_v4si"
   [(set (match_dup 5)
@@ -9786,7 +9823,7 @@ 
 (define_expand "vec_unpacku_float_hi_v16si"
   [(match_operand:V8DF 0 "register_operand")
    (match_operand:V16SI 1 "register_operand")]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
 {
   REAL_VALUE_TYPE TWO32r;
   rtx k, x, tmp[4];
@@ -9835,7 +9872,7 @@ 
 (define_expand "vec_unpacku_float_lo_v16si"
   [(match_operand:V8DF 0 "register_operand")
    (match_operand:V16SI 1 "nonimmediate_operand")]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
 {
   REAL_VALUE_TYPE TWO32r;
   rtx k, x, tmp[3];
@@ -9929,7 +9966,7 @@ 
   [(match_operand:V16SI 0 "register_operand")
    (match_operand:V8DF 1 "nonimmediate_operand")
    (match_operand:V8DF 2 "nonimmediate_operand")]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
 {
   rtx r1, r2;
 
@@ -10044,7 +10081,7 @@ 
   [(match_operand:V16SI 0 "register_operand")
    (match_operand:V8DF 1 "nonimmediate_operand")
    (match_operand:V8DF 2 "nonimmediate_operand")]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
 {
   rtx r1, r2;
 
@@ -10237,7 +10274,7 @@ 
 		     (const_int 11) (const_int 27)
 		     (const_int 14) (const_int 30)
 		     (const_int 15) (const_int 31)])))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vunpckhps\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
   [(set_attr "type" "sselog")
    (set_attr "prefix" "evex")
@@ -10325,7 +10362,7 @@ 
 		     (const_int 9) (const_int 25)
 		     (const_int 12) (const_int 28)
 		     (const_int 13) (const_int 29)])))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vunpcklps\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
   [(set_attr "type" "sselog")
    (set_attr "prefix" "evex")
@@ -10465,7 +10502,7 @@ 
 		     (const_int 11) (const_int 11)
 		     (const_int 13) (const_int 13)
 		     (const_int 15) (const_int 15)])))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vmovshdup\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
   [(set_attr "type" "sse")
    (set_attr "prefix" "evex")
@@ -10518,7 +10555,7 @@ 
 		     (const_int 10) (const_int 10)
 		     (const_int 12) (const_int 12)
 		     (const_int 14) (const_int 14)])))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vmovsldup\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
   [(set_attr "type" "sse")
    (set_attr "prefix" "evex")
@@ -11429,7 +11466,8 @@ 
    (V8SF "32x4") (V8SI "32x4") (V4DF "64x2") (V4DI "64x2")])
 
 (define_mode_iterator AVX512_VEC
-  [(V8DF "TARGET_AVX512DQ") (V8DI "TARGET_AVX512DQ") V16SF V16SI])
+  [(V8DF "TARGET_AVX512DQ") (V8DI "TARGET_AVX512DQ")
+   (V16SF "TARGET_EVEX512") (V16SI "TARGET_EVEX512")])
 
 (define_expand "<extract_type>_vextract<shuffletype><extract_suf>_mask"
   [(match_operand:<ssequartermode> 0 "nonimmediate_operand")
@@ -11598,7 +11636,8 @@ 
   [(V16SF "32x8") (V16SI "32x8") (V8DF "64x4") (V8DI "64x4")])
 
 (define_mode_iterator AVX512_VEC_2
-  [(V16SF "TARGET_AVX512DQ") (V16SI "TARGET_AVX512DQ") V8DF V8DI])
+  [(V16SF "TARGET_AVX512DQ") (V16SI "TARGET_AVX512DQ")
+   (V8DF "TARGET_EVEX512") (V8DI "TARGET_EVEX512")])
 
 (define_expand "<extract_type_2>_vextract<shuffletype><extract_suf_2>_mask"
   [(match_operand:<ssehalfvecmode> 0 "nonimmediate_operand")
@@ -12155,7 +12194,8 @@ 
 		     (const_int 26) (const_int 27)
 		     (const_int 28) (const_int 29)
 		     (const_int 30) (const_int 31)])))]
-  "TARGET_AVX512F && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
+  "TARGET_AVX512F && TARGET_EVEX512
+   && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
 {
   if (TARGET_AVX512VL
       || REG_P (operands[0])
@@ -12203,7 +12243,7 @@ 
 		     (const_int 58) (const_int 59)
 		     (const_int 60) (const_int 61)
 		     (const_int 62) (const_int 63)])))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vextracti64x4\t{$0x1, %1, %0|%0, %1, 0x1}"
   [(set_attr "type" "sselog1")
    (set_attr "length_immediate" "1")
@@ -12299,13 +12339,13 @@ 
 (define_mode_iterator VEC_EXTRACT_MODE
   [(V64QI "TARGET_AVX512BW") (V32QI "TARGET_AVX") V16QI
    (V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX") V8HI
-   (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX") V4SI
-   (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX") V2DI
+   (V16SI "TARGET_AVX512F && TARGET_EVEX512") (V8SI "TARGET_AVX") V4SI
+   (V8DI "TARGET_AVX512F && TARGET_EVEX512") (V4DI "TARGET_AVX") V2DI
    (V32HF "TARGET_AVX512BW") (V16HF "TARGET_AVX") V8HF
    (V32BF "TARGET_AVX512BW") (V16BF "TARGET_AVX") V8BF
-   (V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF
-   (V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX") V2DF
-   (V4TI "TARGET_AVX512F") (V2TI "TARGET_AVX")])
+   (V16SF "TARGET_AVX512F && TARGET_EVEX512") (V8SF "TARGET_AVX") V4SF
+   (V8DF "TARGET_AVX512F && TARGET_EVEX512") (V4DF "TARGET_AVX") V2DF
+   (V4TI "TARGET_AVX512F && TARGET_EVEX512") (V2TI "TARGET_AVX")])
 
 (define_expand "vec_extract<mode><ssescalarmodelower>"
   [(match_operand:<ssescalarmode> 0 "register_operand")
@@ -12347,7 +12387,7 @@ 
 		     (const_int 3) (const_int 11)
 		     (const_int 5) (const_int 13)
 		     (const_int 7) (const_int 15)])))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vunpckhpd\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
   [(set_attr "type" "sselog")
    (set_attr "prefix" "evex")
@@ -12461,7 +12501,7 @@ 
 		     (const_int 2) (const_int 10)
 		     (const_int 4) (const_int 12)
 		     (const_int 6) (const_int 14)])))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vmovddup\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
   [(set_attr "type" "sselog1")
    (set_attr "prefix" "evex")
@@ -12477,7 +12517,7 @@ 
 		     (const_int 2) (const_int 10)
 		     (const_int 4) (const_int 12)
 		     (const_int 6) (const_int 14)])))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vunpcklpd\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
   [(set_attr "type" "sselog")
    (set_attr "prefix" "evex")
@@ -12689,7 +12729,7 @@ 
 	   (match_operand:SI 4 "const_0_to_255_operand")]
 	  UNSPEC_VTERNLOG))]
   "(<MODE_SIZE> == 64 || TARGET_AVX512VL
-    || (TARGET_AVX512F && !TARGET_PREFER_AVX256))
+    || (TARGET_AVX512F && TARGET_EVEX512 && !TARGET_PREFER_AVX256))
 /* Disallow embeded broadcast for vector HFmode since
    it's not real AVX512FP16 instruction.  */
   && (GET_MODE_SIZE (GET_MODE_INNER (<MODE>mode)) >= 4
@@ -12781,7 +12821,7 @@ 
 	    (match_operand:V 3 "regmem_or_bitnot_regmem_operand")
 	    (match_operand:V 4 "regmem_or_bitnot_regmem_operand"))))]
   "(<MODE_SIZE> == 64 || TARGET_AVX512VL
-    || (TARGET_AVX512F && !TARGET_PREFER_AVX256))
+    || (TARGET_AVX512F && TARGET_EVEX512 && !TARGET_PREFER_AVX256))
    && ix86_pre_reload_split ()
    && (rtx_equal_p (STRIP_UNARY (operands[1]),
 		    STRIP_UNARY (operands[4]))
@@ -12866,7 +12906,7 @@ 
 	    (match_operand:V 3 "regmem_or_bitnot_regmem_operand"))
 	  (match_operand:V 4 "regmem_or_bitnot_regmem_operand")))]
   "(<MODE_SIZE> == 64 || TARGET_AVX512VL
-    || (TARGET_AVX512F && !TARGET_PREFER_AVX256))
+    || (TARGET_AVX512F && TARGET_EVEX512 && !TARGET_PREFER_AVX256))
    && ix86_pre_reload_split ()
    && (rtx_equal_p (STRIP_UNARY (operands[1]),
 		    STRIP_UNARY (operands[4]))
@@ -12950,7 +12990,7 @@ 
 	    (match_operand:V 2 "regmem_or_bitnot_regmem_operand"))
 	  (match_operand:V 3 "regmem_or_bitnot_regmem_operand")))]
   "(<MODE_SIZE> == 64 || TARGET_AVX512VL
-    || (TARGET_AVX512F && !TARGET_PREFER_AVX256))
+    || (TARGET_AVX512F && TARGET_EVEX512 && !TARGET_PREFER_AVX256))
    && ix86_pre_reload_split ()"
   "#"
   "&& 1"
@@ -13074,7 +13114,7 @@ 
    (match_operand:SI 3 "const_0_to_255_operand")
    (match_operand:V16SF 4 "register_operand")
    (match_operand:HI 5 "register_operand")]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
 {
   int mask = INTVAL (operands[3]);
   emit_insn (gen_avx512f_shufps512_1_mask (operands[0], operands[1], operands[2],
@@ -13261,7 +13301,7 @@ 
 		     (match_operand 16  "const_12_to_15_operand")
 		     (match_operand 17  "const_28_to_31_operand")
 		     (match_operand 18  "const_28_to_31_operand")])))]
-  "TARGET_AVX512F
+  "TARGET_AVX512F && TARGET_EVEX512
    && (INTVAL (operands[3]) == (INTVAL (operands[7]) - 4)
        && INTVAL (operands[4]) == (INTVAL (operands[8]) - 4)
        && INTVAL (operands[5]) == (INTVAL (operands[9]) - 4)
@@ -13296,7 +13336,7 @@ 
    (match_operand:SI 3 "const_0_to_255_operand")
    (match_operand:V8DF 4 "register_operand")
    (match_operand:QI 5 "register_operand")]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
 {
   int mask = INTVAL (operands[3]);
   emit_insn (gen_avx512f_shufpd512_1_mask (operands[0], operands[1], operands[2],
@@ -13326,7 +13366,7 @@ 
 		     (match_operand 8 "const_12_to_13_operand")
 		     (match_operand 9 "const_6_to_7_operand")
 		     (match_operand 10 "const_14_to_15_operand")])))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
 {
   int mask;
   mask = INTVAL (operands[3]);
@@ -13458,7 +13498,7 @@ 
 		     (const_int 3) (const_int 11)
 		     (const_int 5) (const_int 13)
 		     (const_int 7) (const_int 15)])))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vpunpckhqdq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
   [(set_attr "type" "sselog")
    (set_attr "prefix" "evex")
@@ -13508,7 +13548,7 @@ 
 		     (const_int 2) (const_int 10)
 		     (const_int 4) (const_int 12)
 		     (const_int 6) (const_int 14)])))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vpunpcklqdq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
   [(set_attr "type" "sselog")
    (set_attr "prefix" "evex")
@@ -13872,8 +13912,8 @@ 
    (set_attr "mode" "V2DF,DF,V8DF")
    (set (attr "enabled")
 	(cond [(eq_attr "alternative" "2")
-		 (symbol_ref "TARGET_AVX512F && !TARGET_AVX512VL
-			      && !TARGET_PREFER_AVX256")
+		 (symbol_ref "TARGET_AVX512F && TARGET_EVEX512
+			      && !TARGET_AVX512VL && !TARGET_PREFER_AVX256")
 	       (match_test "<mask_avx512vl_condition>")
 	         (const_string "*")
 	      ]
@@ -13957,13 +13997,13 @@ 
   [(set (match_operand:PMOV_DST_MODE_1 0 "nonimmediate_operand")
 	(truncate:PMOV_DST_MODE_1
 	  (match_operand:<pmov_src_mode> 1 "register_operand")))]
-  "TARGET_AVX512F")
+  "TARGET_AVX512F && TARGET_EVEX512")
 
 (define_insn "*avx512f_<code><pmov_src_lower><mode>2"
   [(set (match_operand:PMOV_DST_MODE_1 0 "nonimmediate_operand" "=v,m")
 	(any_truncate:PMOV_DST_MODE_1
 	  (match_operand:<pmov_src_mode> 1 "register_operand" "v,v")))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vpmov<trunsuffix><pmov_suff_1>\t{%1, %0|%0, %1}"
   [(set_attr "type" "ssemov")
    (set_attr "memory" "none,store")
@@ -14094,7 +14134,7 @@ 
 		     (const_int 2) (const_int 3)
 		     (const_int 4) (const_int 5)
 		     (const_int 6) (const_int 7)])))]
-  "TARGET_AVX512F && ix86_pre_reload_split ()"
+  "TARGET_AVX512F && TARGET_EVEX512 && ix86_pre_reload_split ()"
   "#"
   "&& 1"
   [(set (match_dup 0)
@@ -14110,7 +14150,7 @@ 
         (match_operand:<pmov_src_mode> 1 "register_operand" "v,v"))
       (match_operand:PMOV_DST_MODE_1 2 "nonimm_or_0_operand" "0C,0")
       (match_operand:<avx512fmaskmode> 3 "register_operand" "Yk,Yk")))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vpmov<trunsuffix><pmov_suff_1>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
   [(set_attr "type" "ssemov")
    (set_attr "memory" "none,store")
@@ -14124,7 +14164,7 @@ 
         (match_operand:<pmov_src_mode> 1 "register_operand"))
       (match_dup 0)
       (match_operand:<avx512fmaskmode> 2 "register_operand")))]
-  "TARGET_AVX512F")
+  "TARGET_AVX512F && TARGET_EVEX512")
 
 (define_expand "truncv32hiv32qi2"
   [(set (match_operand:V32QI 0 "nonimmediate_operand")
@@ -15072,7 +15112,7 @@ 
   [(set (match_operand:V8QI 0 "register_operand")
 	(truncate:V8QI
 	    (match_operand:V8DI 1 "register_operand")))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
 {
   rtx op0 = gen_reg_rtx (V16QImode);
 
@@ -15092,7 +15132,7 @@ 
 			      (const_int 0) (const_int 0)
 			      (const_int 0) (const_int 0)
 			      (const_int 0) (const_int 0)])))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vpmov<trunsuffix>qb\t{%1, %0|%0, %1}"
   [(set_attr "type" "ssemov")
    (set_attr "prefix" "evex")
@@ -15102,7 +15142,7 @@ 
   [(set (match_operand:V8QI 0 "memory_operand" "=m")
 	(any_truncate:V8QI
 	  (match_operand:V8DI 1 "register_operand" "v")))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vpmov<trunsuffix>qb\t{%1, %0|%0, %1}"
   [(set_attr "type" "ssemov")
    (set_attr "memory" "store")
@@ -15114,7 +15154,7 @@ 
 	(subreg:DI
 	  (any_truncate:V8QI
 	    (match_operand:V8DI 1 "register_operand")) 0))]
-  "TARGET_AVX512F && ix86_pre_reload_split ()"
+  "TARGET_AVX512F && TARGET_EVEX512 && ix86_pre_reload_split ()"
   "#"
   "&& 1"
   [(set (match_dup 0)
@@ -15138,7 +15178,7 @@ 
                           (const_int 0) (const_int 0)
                           (const_int 0) (const_int 0)
                           (const_int 0) (const_int 0)])))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vpmov<trunsuffix>qb\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
   [(set_attr "type" "ssemov")
    (set_attr "prefix" "evex")
@@ -15159,7 +15199,7 @@ 
 			  (const_int 0) (const_int 0)
 			  (const_int 0) (const_int 0)
 			  (const_int 0) (const_int 0)])))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vpmov<trunsuffix>qb\t{%1, %0%{%2%}%{z%}|%0%{%2%}%{z%}, %1}"
   [(set_attr "type" "ssemov")
    (set_attr "prefix" "evex")
@@ -15172,7 +15212,7 @@ 
 	    (match_operand:V8DI 1 "register_operand" "v"))
 	(match_dup 0)
 	(match_operand:QI 2 "register_operand" "Yk")))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vpmov<trunsuffix>qb\t{%1, %0%{%2%}|%0%{%2%}, %1}"
   [(set_attr "type" "ssemov")
    (set_attr "memory" "store")
@@ -15195,7 +15235,7 @@ 
 		       (const_int 4) (const_int 5)
 		       (const_int 6) (const_int 7)]))
 	  (match_operand:QI 2 "register_operand")) 0))]
-  "TARGET_AVX512F && ix86_pre_reload_split ()"
+  "TARGET_AVX512F && TARGET_EVEX512 && ix86_pre_reload_split ()"
   "#"
   "&& 1"
   [(set (match_dup 0)
@@ -15453,7 +15493,7 @@ 
                          (const_int 4) (const_int 6)
                          (const_int 8) (const_int 10)
                          (const_int 12) (const_int 14)])))))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "ix86_fixup_binary_operands_no_copy (MULT, V16SImode, operands);")
 
 (define_insn "*vec_widen_umult_even_v16si<mask_name>"
@@ -15473,7 +15513,8 @@ 
                          (const_int 4) (const_int 6)
                          (const_int 8) (const_int 10)
                          (const_int 12) (const_int 14)])))))]
-  "TARGET_AVX512F && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
+  "TARGET_AVX512F && TARGET_EVEX512
+   && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
   "vpmuludq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
   [(set_attr "type" "sseimul")
    (set_attr "prefix" "evex")
@@ -15568,7 +15609,7 @@ 
                          (const_int 4) (const_int 6)
                          (const_int 8) (const_int 10)
                          (const_int 12) (const_int 14)])))))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "ix86_fixup_binary_operands_no_copy (MULT, V16SImode, operands);")
 
 (define_insn "*vec_widen_smult_even_v16si<mask_name>"
@@ -15588,7 +15629,8 @@ 
                          (const_int 4) (const_int 6)
                          (const_int 8) (const_int 10)
                          (const_int 12) (const_int 14)])))))]
-  "TARGET_AVX512F && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
+  "TARGET_AVX512F && TARGET_EVEX512
+   && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
   "vpmuldq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
   [(set_attr "type" "sseimul")
    (set_attr "prefix" "evex")
@@ -17263,8 +17305,10 @@ 
    (V8SI "TARGET_AVX2") (V4DI "TARGET_AVX2")
    (V8SF "TARGET_AVX2") (V4DF "TARGET_AVX2")
    (V16HF "TARGET_AVX512FP16")
-   (V16SF "TARGET_AVX512F") (V8DF "TARGET_AVX512F")
-   (V16SI "TARGET_AVX512F") (V8DI "TARGET_AVX512F")
+   (V16SF "TARGET_AVX512F && TARGET_EVEX512")
+   (V8DF "TARGET_AVX512F && TARGET_EVEX512")
+   (V16SI "TARGET_AVX512F && TARGET_EVEX512")
+   (V8DI "TARGET_AVX512F && TARGET_EVEX512")
    (V32HI "TARGET_AVX512BW") (V64QI "TARGET_AVX512VBMI")
    (V32HF "TARGET_AVX512FP16")])
 
@@ -17293,7 +17337,7 @@ 
 {
   operands[2] = CONSTM1_RTX (<MODE>mode);
 
-  if (!TARGET_AVX512F)
+  if (!TARGET_AVX512F || (!TARGET_AVX512VL && !TARGET_EVEX512))
     operands[2] = force_reg (<MODE>mode, operands[2]);
 })
 
@@ -17302,6 +17346,7 @@ 
 	(xor:VI (match_operand:VI 1 "bcst_vector_operand"     " 0, m,Br")
 		(match_operand:VI 2 "vector_all_ones_operand" "BC,BC,BC")))]
   "TARGET_AVX512F
+   && (<MODE_SIZE> == 64 || TARGET_AVX512VL || TARGET_EVEX512)
    && (!<mask_applied>
        || <ssescalarmode>mode == SImode
        || <ssescalarmode>mode == DImode)"
@@ -17368,7 +17413,7 @@ 
 		(match_operand:VI 2 "vector_all_ones_operand" "BC,BC,BC")))
    (unspec [(match_operand:VI 3 "register_operand" "0,0,0")]
      UNSPEC_INSN_FALSE_DEP)]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && (<MODE_SIZE> == 64 || TARGET_AVX512VL || TARGET_EVEX512)"
 {
   if (TARGET_AVX512VL)
     return "vpternlog<ternlogsuffix>\t{$0x55, %1, %0, %0<mask_operand3>|%0<mask_operand3>, %0, %1, 0x55}";
@@ -17392,7 +17437,7 @@ 
 	  (not:<ssescalarmode>
 	    (match_operand:<ssescalarmode> 1 "nonimmediate_operand"))))]
   "<MODE_SIZE> == 64 || TARGET_AVX512VL
-   || (TARGET_AVX512F && !TARGET_PREFER_AVX256)"
+   || (TARGET_AVX512F && TARGET_EVEX512 && !TARGET_PREFER_AVX256)"
   [(set (match_dup 0)
 	(xor:VI48_AVX512F
 	  (vec_duplicate:VI48_AVX512F (match_dup 1))
@@ -17538,7 +17583,8 @@ 
 		 (symbol_ref "<MODE_SIZE> == 64 || TARGET_AVX512VL")
 	       (eq_attr "alternative" "4")
 		 (symbol_ref "<MODE_SIZE> == 64 || TARGET_AVX512VL
-			      || (TARGET_AVX512F && !TARGET_PREFER_AVX256)")
+			      || (TARGET_AVX512F && TARGET_EVEX512
+				  && !TARGET_PREFER_AVX256)")
 	      ]
 	      (const_string "*")))])
 
@@ -17582,7 +17628,7 @@ 
 	      (match_operand:<ssescalarmode> 1 "nonimmediate_operand")))
 	  (match_operand:VI 2 "vector_operand")))]
   "<MODE_SIZE> == 64 || TARGET_AVX512VL
-   || (TARGET_AVX512F && !TARGET_PREFER_AVX256)"
+   || (TARGET_AVX512F && TARGET_EVEX512 && !TARGET_PREFER_AVX256)"
   [(set (match_dup 3)
 	(vec_duplicate:VI (match_dup 1)))
    (set (match_dup 0)
@@ -17597,7 +17643,7 @@ 
 	      (match_operand:<ssescalarmode> 1 "nonimmediate_operand")))
 	  (match_operand:VI 2 "vector_operand")))]
   "<MODE_SIZE> == 64 || TARGET_AVX512VL
-   || (TARGET_AVX512F && !TARGET_PREFER_AVX256)"
+   || (TARGET_AVX512F && TARGET_EVEX512 && !TARGET_PREFER_AVX256)"
   [(set (match_dup 3)
 	(vec_duplicate:VI (match_dup 1)))
    (set (match_dup 0)
@@ -17883,7 +17929,7 @@ 
 	    (match_operand:VI 1 "bcst_vector_operand" "0,m,  0,vBr"))
 	  (match_operand:VI 2 "bcst_vector_operand"   "m,0,vBr,  0")))]
   "(<MODE_SIZE> == 64 || TARGET_AVX512VL
-    || (TARGET_AVX512F && !TARGET_PREFER_AVX256))
+    || (TARGET_AVX512F && TARGET_EVEX512 && !TARGET_PREFER_AVX256))
    && (register_operand (operands[1], <MODE>mode)
        || register_operand (operands[2], <MODE>mode))"
 {
@@ -17916,7 +17962,7 @@ 
 	    (match_operand:VI 1 "bcst_vector_operand" "%0, 0")
 	    (match_operand:VI 2 "bcst_vector_operand" " m,vBr"))))]
   "(<MODE_SIZE> == 64 || TARGET_AVX512VL
-    || (TARGET_AVX512F && !TARGET_PREFER_AVX256))
+    || (TARGET_AVX512F && TARGET_EVEX512 && !TARGET_PREFER_AVX256))
    && (register_operand (operands[1], <MODE>mode)
        || register_operand (operands[2], <MODE>mode))"
 {
@@ -17947,7 +17993,7 @@ 
 	  (not:VI (match_operand:VI 1 "bcst_vector_operand" "%0, 0"))
 	  (not:VI (match_operand:VI 2 "bcst_vector_operand" "m,vBr"))))]
   "(<MODE_SIZE> == 64 || TARGET_AVX512VL
-    || (TARGET_AVX512F && !TARGET_PREFER_AVX256))
+    || (TARGET_AVX512F && TARGET_EVEX512 && !TARGET_PREFER_AVX256))
    && (register_operand (operands[1], <MODE>mode)
        || register_operand (operands[2], <MODE>mode))"
 {
@@ -18669,7 +18715,7 @@ 
 		     (const_int 11) (const_int 27)
 		     (const_int 14) (const_int 30)
 		     (const_int 15) (const_int 31)])))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vpunpckhdq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
   [(set_attr "type" "sselog")
    (set_attr "prefix" "evex")
@@ -18724,7 +18770,7 @@ 
 		     (const_int 9) (const_int 25)
 		     (const_int 12) (const_int 28)
 		     (const_int 13) (const_int 29)])))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vpunpckldq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
   [(set_attr "type" "sselog")
    (set_attr "prefix" "evex")
@@ -19418,7 +19464,7 @@ 
    (match_operand:SI 2 "const_0_to_255_operand")
    (match_operand:V16SI 3 "register_operand")
    (match_operand:HI 4 "register_operand")]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
 {
   int mask = INTVAL (operands[2]);
   emit_insn (gen_avx512f_pshufd_1_mask (operands[0], operands[1],
@@ -19462,7 +19508,7 @@ 
 		     (match_operand 15 "const_12_to_15_operand")
 		     (match_operand 16 "const_12_to_15_operand")
 		     (match_operand 17 "const_12_to_15_operand")])))]
-  "TARGET_AVX512F
+  "TARGET_AVX512F && TARGET_EVEX512
    && INTVAL (operands[2]) + 4 == INTVAL (operands[6])
    && INTVAL (operands[3]) + 4 == INTVAL (operands[7])
    && INTVAL (operands[4]) + 4 == INTVAL (operands[8])
@@ -20315,7 +20361,7 @@ 
 	  (match_operand:V4TI 1 "register_operand" "v")
 	  (parallel
 	    [(match_operand:SI 2 "const_0_to_3_operand")])))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vextracti32x4\t{%2, %1, %0|%0, %1, %2}"
   [(set_attr "type" "sselog")
    (set_attr "length_immediate" "1")
@@ -20323,7 +20369,7 @@ 
    (set_attr "mode" "XI")])
 
 (define_mode_iterator VEXTRACTI128_MODE
-  [(V4TI "TARGET_AVX512F") V2TI])
+  [(V4TI "TARGET_AVX512F && TARGET_EVEX512") V2TI])
 
 (define_split
   [(set (match_operand:TI 0 "nonimmediate_operand")
@@ -20346,7 +20392,8 @@ 
    && VECTOR_MODE_P (GET_MODE (operands[1]))
    && ((TARGET_SSE && GET_MODE_SIZE (GET_MODE (operands[1])) == 16)
        || (TARGET_AVX && GET_MODE_SIZE (GET_MODE (operands[1])) == 32)
-       || (TARGET_AVX512F && GET_MODE_SIZE (GET_MODE (operands[1])) == 64))
+       || (TARGET_AVX512F && TARGET_EVEX512
+	   && GET_MODE_SIZE (GET_MODE (operands[1])) == 64))
    && (<MODE>mode == SImode || TARGET_64BIT || MEM_P (operands[0]))"
   [(set (match_dup 0) (vec_select:SWI48x (match_dup 1)
 					 (parallel [(const_int 0)])))]
@@ -21994,8 +22041,9 @@ 
 (define_mode_iterator VI1248_AVX512VL_AVX512BW
   [(V64QI "TARGET_AVX512BW") (V32QI "TARGET_AVX2") V16QI
    (V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX2") V8HI
-   (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX2") V4SI
-   (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX512VL") (V2DI "TARGET_AVX512VL")])
+   (V16SI "TARGET_AVX512F && TARGET_EVEX512") (V8SI "TARGET_AVX2") V4SI
+   (V8DI "TARGET_AVX512F && TARGET_EVEX512") (V4DI "TARGET_AVX512VL")
+   (V2DI "TARGET_AVX512VL")])
 
 (define_insn "*abs<mode>2"
   [(set (match_operand:VI1248_AVX512VL_AVX512BW 0 "register_operand" "=<v_Yw>")
@@ -22840,7 +22888,7 @@ 
   [(set (match_operand:V16SI 0 "register_operand" "=v")
 	(any_extend:V16SI
 	  (match_operand:V16QI 1 "nonimmediate_operand" "vm")))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vpmov<extsuffix>bd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
   [(set_attr "type" "ssemov")
    (set_attr "prefix" "evex")
@@ -22850,7 +22898,7 @@ 
   [(set (match_operand:V16SI 0 "register_operand")
 	(any_extend:V16SI
 	  (match_operand:V16QI 1 "nonimmediate_operand")))]
-  "TARGET_AVX512F")
+  "TARGET_AVX512F && TARGET_EVEX512")
 
 (define_insn "avx2_<code>v8qiv8si2<mask_name>"
   [(set (match_operand:V8SI 0 "register_operand" "=v")
@@ -22982,7 +23030,7 @@ 
   [(set (match_operand:V16SI 0 "register_operand" "=v")
 	(any_extend:V16SI
 	  (match_operand:V16HI 1 "nonimmediate_operand" "vm")))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vpmov<extsuffix>wd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
   [(set_attr "type" "ssemov")
    (set_attr "prefix" "evex")
@@ -22992,7 +23040,7 @@ 
   [(set (match_operand:V16SI 0 "register_operand")
 	(any_extend:V16SI
 	  (match_operand:V16HI 1 "nonimmediate_operand")))]
-  "TARGET_AVX512F")
+  "TARGET_AVX512F && TARGET_EVEX512")
 
 (define_insn_and_split "avx512f_zero_extendv16hiv16si2_1"
   [(set (match_operand:V32HI 0 "register_operand" "=v")
@@ -23002,7 +23050,7 @@ 
 	    (match_operand:V32HI 2 "const0_operand"))
 	  (match_parallel 3 "pmovzx_parallel"
 	    [(match_operand 4 "const_int_operand")])))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "#"
   "&& reload_completed"
   [(set (match_dup 0) (zero_extend:V16SI (match_dup 1)))]
@@ -23223,7 +23271,7 @@ 
 		       (const_int 2) (const_int 3)
 		       (const_int 4) (const_int 5)
 		       (const_int 6) (const_int 7)]))))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vpmov<extsuffix>bq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
   [(set_attr "type" "ssemov")
    (set_attr "prefix" "evex")
@@ -23233,7 +23281,7 @@ 
   [(set (match_operand:V8DI 0 "register_operand" "=v")
 	(any_extend:V8DI
 	  (match_operand:V8QI 1 "memory_operand" "m")))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vpmov<extsuffix>bq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
   [(set_attr "type" "ssemov")
    (set_attr "prefix" "evex")
@@ -23251,7 +23299,7 @@ 
 		       (const_int 2) (const_int 3)
 		       (const_int 4) (const_int 5)
 		       (const_int 6) (const_int 7)]))))]
-  "TARGET_AVX512F && ix86_pre_reload_split ()"
+  "TARGET_AVX512F && TARGET_EVEX512 && ix86_pre_reload_split ()"
   "#"
   "&& 1"
   [(set (match_dup 0)
@@ -23262,7 +23310,7 @@ 
   [(set (match_operand:V8DI 0 "register_operand")
 	(any_extend:V8DI
 	  (match_operand:V8QI 1 "nonimmediate_operand")))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
 {
   if (!MEM_P (operands[1]))
     {
@@ -23402,7 +23450,7 @@ 
   [(set (match_operand:V8DI 0 "register_operand" "=v")
 	(any_extend:V8DI
 	  (match_operand:V8HI 1 "nonimmediate_operand" "vm")))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vpmov<extsuffix>wq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
   [(set_attr "type" "ssemov")
    (set_attr "prefix" "evex")
@@ -23412,7 +23460,7 @@ 
   [(set (match_operand:V8DI 0 "register_operand")
 	(any_extend:V8DI
 	  (match_operand:V8HI 1 "nonimmediate_operand")))]
-  "TARGET_AVX512F")
+  "TARGET_AVX512F && TARGET_EVEX512")
 
 (define_insn "avx2_<code>v4hiv4di2<mask_name>"
   [(set (match_operand:V4DI 0 "register_operand" "=v")
@@ -23538,7 +23586,7 @@ 
   [(set (match_operand:V8DI 0 "register_operand" "=v")
 	(any_extend:V8DI
 	  (match_operand:V8SI 1 "nonimmediate_operand" "vm")))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vpmov<extsuffix>dq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
   [(set_attr "type" "ssemov")
    (set_attr "prefix" "evex")
@@ -23552,7 +23600,7 @@ 
 	    (match_operand:V16SI 2 "const0_operand"))
 	  (match_parallel 3 "pmovzx_parallel"
 	    [(match_operand 4 "const_int_operand")])))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "#"
   "&& reload_completed"
   [(set (match_dup 0) (zero_extend:V8DI (match_dup 1)))]
@@ -23571,7 +23619,7 @@ 
 	    (match_operand:V16SI 3 "const0_operand"))
 	  (match_parallel 4 "pmovzx_parallel"
 	    [(match_operand 5 "const_int_operand")])))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "#"
   "&& reload_completed"
   [(set (match_dup 0) (zero_extend:V8DI (match_dup 1)))]
@@ -23583,7 +23631,7 @@ 
   [(set (match_operand:V8DI 0 "register_operand" "=v")
 	(any_extend:V8DI
 	  (match_operand:V8SI 1 "nonimmediate_operand" "vm")))]
-  "TARGET_AVX512F")
+  "TARGET_AVX512F && TARGET_EVEX512")
 
 (define_insn "avx2_<code>v4siv4di2<mask_name>"
   [(set (match_operand:V4DI 0 "register_operand" "=v")
@@ -23977,7 +24025,7 @@ 
   [(match_operand:V16SI 0 "register_operand")
    (match_operand:V16SF 1 "nonimmediate_operand")
    (match_operand:SI 2 "const_0_to_15_operand")]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
 {
   rtx tmp = gen_reg_rtx (V16SFmode);
   emit_insn (gen_avx512f_rndscalev16sf (tmp, operands[1], operands[2]));
@@ -25394,7 +25442,7 @@ 
 	(ashiftrt:V8DI
 	  (match_operand:V8DI 1 "register_operand")
 	  (match_operand:V8DI 2 "nonimmediate_operand")))]
-  "TARGET_AVX512F")
+  "TARGET_AVX512F && TARGET_EVEX512")
 
 (define_expand "vashrv4di3"
   [(set (match_operand:V4DI 0 "register_operand")
@@ -25485,7 +25533,7 @@ 
   [(set (match_operand:V16SI 0 "register_operand")
 	(ashiftrt:V16SI (match_operand:V16SI 1 "register_operand")
 		        (match_operand:V16SI 2 "nonimmediate_operand")))]
-  "TARGET_AVX512F")
+  "TARGET_AVX512F && TARGET_EVEX512")
 
 (define_expand "vashrv8si3"
   [(set (match_operand:V8SI 0 "register_operand")
@@ -26058,8 +26106,8 @@ 
 (define_mode_attr pbroadcast_evex_isa
   [(V64QI "avx512bw") (V32QI "avx512bw") (V16QI "avx512bw")
    (V32HI "avx512bw") (V16HI "avx512bw") (V8HI "avx512bw")
-   (V16SI "avx512f") (V8SI "avx512f") (V4SI "avx512f")
-   (V8DI "avx512f") (V4DI "avx512f") (V2DI "avx512f")
+   (V16SI "avx512f_512") (V8SI "avx512f") (V4SI "avx512f")
+   (V8DI "avx512f_512") (V4DI "avx512f") (V2DI "avx512f")
    (V32HF "avx512bw") (V16HF "avx512bw") (V8HF "avx512bw")
    (V32BF "avx512bw") (V16BF "avx512bw") (V8BF "avx512bw")])
 
@@ -26602,7 +26650,7 @@ 
    (set (attr "enabled")
 	(if_then_else (eq_attr "alternative" "1")
 		      (symbol_ref "TARGET_AVX512F && !TARGET_AVX512VL
-				   && !TARGET_PREFER_AVX256")
+				   && TARGET_EVEX512 && !TARGET_PREFER_AVX256")
 		      (const_string "*")))])
 
 (define_insn "*vec_dupv4si"
@@ -26630,7 +26678,7 @@ 
    (set (attr "enabled")
 	(if_then_else (eq_attr "alternative" "1")
 		      (symbol_ref "TARGET_AVX512F && !TARGET_AVX512VL
-				   && !TARGET_PREFER_AVX256")
+				   && TARGET_EVEX512 && !TARGET_PREFER_AVX256")
 		      (const_string "*")))])
 
 (define_insn "*vec_dupv2di"
@@ -26661,7 +26709,8 @@ 
 	(if_then_else
 	  (eq_attr "alternative" "2")
 	  (symbol_ref "TARGET_AVX512VL
-		       || (TARGET_AVX512F && !TARGET_PREFER_AVX256)")
+		       || (TARGET_AVX512F && TARGET_EVEX512
+			   && !TARGET_PREFER_AVX256)")
 	  (const_string "*")))])
 
 (define_insn "avx2_vbroadcasti128_<mode>"
@@ -26741,7 +26790,7 @@ 
   [(set_attr "type" "ssemov")
    (set_attr "prefix_extra" "1")
    (set_attr "prefix" "maybe_evex")
-   (set_attr "isa" "avx2,noavx2,avx2,avx512f,noavx2")
+   (set_attr "isa" "avx2,noavx2,avx2,avx512f_512,noavx2")
    (set_attr "mode" "<sseinsnmode>,V8SF,<sseinsnmode>,<sseinsnmode>,V8SF")])
 
 (define_split
@@ -26908,7 +26957,8 @@ 
    (set_attr "mode" "<sseinsnmode>")])
 
 (define_mode_iterator VPERMI2
-  [V16SI V16SF V8DI V8DF
+  [(V16SI "TARGET_EVEX512") (V16SF "TARGET_EVEX512")
+   (V8DI "TARGET_EVEX512") (V8DF "TARGET_EVEX512")
    (V8SI "TARGET_AVX512VL") (V8SF "TARGET_AVX512VL")
    (V4DI "TARGET_AVX512VL") (V4DF "TARGET_AVX512VL")
    (V4SI "TARGET_AVX512VL") (V4SF "TARGET_AVX512VL")
@@ -26919,7 +26969,7 @@ 
    (V16QI "TARGET_AVX512VBMI && TARGET_AVX512VL")])
 
 (define_mode_iterator VPERMI2I
-  [V16SI V8DI
+  [(V16SI "TARGET_EVEX512") (V8DI "TARGET_EVEX512")
    (V8SI "TARGET_AVX512VL") (V4SI "TARGET_AVX512VL")
    (V4DI "TARGET_AVX512VL") (V2DI "TARGET_AVX512VL")
    (V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX512BW && TARGET_AVX512VL")
@@ -27543,28 +27593,29 @@ 
 
 ;; Modes handled by vec_init expanders.
 (define_mode_iterator VEC_INIT_MODE
-  [(V64QI "TARGET_AVX512F") (V32QI "TARGET_AVX") V16QI
-   (V32HI "TARGET_AVX512F") (V16HI "TARGET_AVX") V8HI
-   (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX") V4SI
-   (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX") V2DI
-   (V32HF "TARGET_AVX512F") (V16HF "TARGET_AVX") V8HF
-   (V32BF "TARGET_AVX512F") (V16BF "TARGET_AVX") V8BF
-   (V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF
-   (V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")
-   (V4TI "TARGET_AVX512F") (V2TI "TARGET_AVX")])
+  [(V64QI "TARGET_AVX512F && TARGET_EVEX512") (V32QI "TARGET_AVX") V16QI
+   (V32HI "TARGET_AVX512F && TARGET_EVEX512") (V16HI "TARGET_AVX") V8HI
+   (V16SI "TARGET_AVX512F && TARGET_EVEX512") (V8SI "TARGET_AVX") V4SI
+   (V8DI "TARGET_AVX512F && TARGET_EVEX512") (V4DI "TARGET_AVX") V2DI
+   (V32HF "TARGET_AVX512F && TARGET_EVEX512") (V16HF "TARGET_AVX") V8HF
+   (V32BF "TARGET_AVX512F && TARGET_EVEX512") (V16BF "TARGET_AVX") V8BF
+   (V16SF "TARGET_AVX512F && TARGET_EVEX512") (V8SF "TARGET_AVX") V4SF
+   (V8DF "TARGET_AVX512F && TARGET_EVEX512")
+   (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")
+   (V4TI "TARGET_AVX512F && TARGET_EVEX512") (V2TI "TARGET_AVX")])
 
 ;; Likewise, but for initialization from half sized vectors.
 ;; Thus, these are all VEC_INIT_MODE modes except V2??.
 (define_mode_iterator VEC_INIT_HALF_MODE
-  [(V64QI "TARGET_AVX512F") (V32QI "TARGET_AVX") V16QI
-   (V32HI "TARGET_AVX512F") (V16HI "TARGET_AVX") V8HI
-   (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX") V4SI
-   (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX")
-   (V32HF "TARGET_AVX512F") (V16HF "TARGET_AVX") V8HF
-   (V32BF "TARGET_AVX512F") (V16BF "TARGET_AVX") V8BF
-   (V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF
-   (V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX")
-   (V4TI "TARGET_AVX512F")])
+  [(V64QI "TARGET_AVX512F && TARGET_EVEX512") (V32QI "TARGET_AVX") V16QI
+   (V32HI "TARGET_AVX512F && TARGET_EVEX512") (V16HI "TARGET_AVX") V8HI
+   (V16SI "TARGET_AVX512F && TARGET_EVEX512") (V8SI "TARGET_AVX") V4SI
+   (V8DI "TARGET_AVX512F && TARGET_EVEX512") (V4DI "TARGET_AVX")
+   (V32HF "TARGET_AVX512F && TARGET_EVEX512") (V16HF "TARGET_AVX") V8HF
+   (V32BF "TARGET_AVX512F && TARGET_EVEX512") (V16BF "TARGET_AVX") V8BF
+   (V16SF "TARGET_AVX512F && TARGET_EVEX512") (V8SF "TARGET_AVX") V4SF
+   (V8DF "TARGET_AVX512F && TARGET_EVEX512") (V4DF "TARGET_AVX")
+   (V4TI "TARGET_AVX512F && TARGET_EVEX512")])
 
 (define_expand "vec_init<mode><ssescalarmodelower>"
   [(match_operand:VEC_INIT_MODE 0 "register_operand")
@@ -27817,7 +27868,7 @@ 
 	(unspec:V16SF
 	  [(match_operand:V16HI 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")]
 	  UNSPEC_VCVTPH2PS))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vcvtph2ps\t{<round_saeonly_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_saeonly_mask_op2>}"
   [(set_attr "type" "ssecvt")
    (set_attr "prefix" "evex")
@@ -27907,7 +27958,7 @@ 
 	   UNSPEC_VCVTPS2PH)
 	 (match_operand:V16HI 3 "nonimm_or_0_operand")
 	 (match_operand:HI 4 "register_operand")))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
 {
   int round = INTVAL (operands[2]);
   /* Separate {sae} from rounding control imm,
@@ -27926,7 +27977,7 @@ 
 	  [(match_operand:V16SF 1 "register_operand" "v")
 	   (match_operand:SI 2 "const_0_to_255_operand")]
 	  UNSPEC_VCVTPS2PH))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vcvtps2ph\t{%2, <round_saeonly_mask_op3>%1, %0<mask_operand3>|%0<mask_operand3>, %1<round_saeonly_mask_op3>, %2}"
   [(set_attr "type" "ssecvt")
    (set_attr "prefix" "evex")
@@ -27938,7 +27989,7 @@ 
 	  [(match_operand:V16SF 1 "register_operand" "v")
 	   (match_operand:SI 2 "const_0_to_255_operand")]
 	  UNSPEC_VCVTPS2PH))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && TARGET_EVEX512"
   "vcvtps2ph\t{%2, %1, %0<merge_mask_operand3>|%0<merge_mask_operand3>, %1, %2}"
   [(set_attr "type" "ssecvt")
    (set_attr "prefix" "evex")
@@ -30285,10 +30336,10 @@ 
 ;;	vinserti64x4	$0x1, %ymm15, %zmm15, %zmm15
 
 (define_mode_iterator INT_BROADCAST_MODE
-  [(V64QI "TARGET_AVX512F") (V32QI "TARGET_AVX") V16QI
-   (V32HI "TARGET_AVX512F") (V16HI "TARGET_AVX") V8HI
-   (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX") V4SI
-   (V8DI "TARGET_AVX512F && TARGET_64BIT")
+  [(V64QI "TARGET_AVX512F && TARGET_EVEX512") (V32QI "TARGET_AVX") V16QI
+   (V32HI "TARGET_AVX512F && TARGET_EVEX512") (V16HI "TARGET_AVX") V8HI
+   (V16SI "TARGET_AVX512F && TARGET_EVEX512") (V8SI "TARGET_AVX") V4SI
+   (V8DI "TARGET_AVX512F && TARGET_EVEX512 && TARGET_64BIT")
    (V4DI "TARGET_AVX && TARGET_64BIT") (V2DI "TARGET_64BIT")])
 
 ;; Broadcast from an integer.  NB: Enable broadcast only if we can move
diff --git a/gcc/testsuite/gcc.target/i386/pr89229-5b.c b/gcc/testsuite/gcc.target/i386/pr89229-5b.c
index 261f2e12e8d..8a81585e790 100644
--- a/gcc/testsuite/gcc.target/i386/pr89229-5b.c
+++ b/gcc/testsuite/gcc.target/i386/pr89229-5b.c
@@ -3,4 +3,4 @@ 
 
 #include "pr89229-5a.c"
 
-/* { dg-final { scan-assembler-times "vmovdqa32\[^\n\r]*zmm1\[67]\[^\n\r]*zmm1\[67]" 1 } } */
+/* { dg-final { scan-assembler-times "vmovsd\[^\n\r]*xmm1\[67]\[^\n\r]*xmm1\[67]\[^\n\r]*xmm1\[67]" 1 } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr89229-6b.c b/gcc/testsuite/gcc.target/i386/pr89229-6b.c
index a74f7169e6e..0c27daa4f74 100644
--- a/gcc/testsuite/gcc.target/i386/pr89229-6b.c
+++ b/gcc/testsuite/gcc.target/i386/pr89229-6b.c
@@ -3,4 +3,4 @@ 
 
 #include "pr89229-6a.c"
 
-/* { dg-final { scan-assembler-times "vmovaps\[^\n\r]*zmm1\[67]\[^\n\r]*zmm1\[67]" 1 } } */
+/* { dg-final { scan-assembler-times "vmovss\[^\n\r]*xmm1\[67]\[^\n\r]*xmm1\[67]\[^\n\r]*xmm1\[67]" 1 } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr89229-7b.c b/gcc/testsuite/gcc.target/i386/pr89229-7b.c
index d3a56e6e2b7..baba99ec775 100644
--- a/gcc/testsuite/gcc.target/i386/pr89229-7b.c
+++ b/gcc/testsuite/gcc.target/i386/pr89229-7b.c
@@ -3,4 +3,4 @@ 
 
 #include "pr89229-7a.c"
 
-/* { dg-final { scan-assembler-times "vmovdqa32\[^\n\r]*zmm1\[67]\[^\n\r]*zmm1\[67]" 1 } } */
+/* { dg-final { scan-assembler-times "vmovss\[^\n\r]*xmm1\[67]\[^\n\r]*xmm1\[67]\[^\n\r]*xmm1\[67]" 1 } } */