diff mbox series

[i386] Clear odata for aes(enc|dec)(wide)?kl intrinsics

Message ID 20210701075149.1619883-1-hongyu.wang@intel.com
State New
Headers show
Series [i386] Clear odata for aes(enc|dec)(wide)?kl intrinsics | expand

Commit Message

Hongyu Wang July 1, 2021, 7:51 a.m. UTC 
For Keylocker aesenc/aesdec intrinsics, current implementation
moves idata to odata unconditionally, which causes safety issue when
the instruction meets runtime error. So we add a branch to clear
odata when ZF is set after instruction exectution.

gcc/ChangeLog:

	* config/i386/i386-expand.c (ix86_expand_builtin):
	Add branch to clear odata when ZF is set for asedecenc_expand
	and wideaesdecenc_expand.

gcc/testsuite/ChangeLog:

	* gcc.target/i386/keylocker-aesdec128kl.c: Update test.
	* gcc.target/i386/keylocker-aesdec256kl.c: Likewise.
	* gcc.target/i386/keylocker-aesdecwide128kl.c: Likewise.
	* gcc.target/i386/keylocker-aesdecwide256kl.c: Likewise.
	* gcc.target/i386/keylocker-aesenc128kl.c: Likewise.
	* gcc.target/i386/keylocker-aesenc256kl.c: Likewise.
	* gcc.target/i386/keylocker-aesencwide128kl.c: Likewise.
	* gcc.target/i386/keylocker-aesencwide256kl.c: Likewise.
---
 gcc/config/i386/i386-expand.c                 | 33 ++++++++++++++++---
 .../gcc.target/i386/keylocker-aesdec128kl.c   |  2 ++
 .../gcc.target/i386/keylocker-aesdec256kl.c   |  2 ++
 .../i386/keylocker-aesdecwide128kl.c          |  9 +++++
 .../i386/keylocker-aesdecwide256kl.c          |  9 +++++
 .../gcc.target/i386/keylocker-aesenc128kl.c   |  2 ++
 .../gcc.target/i386/keylocker-aesenc256kl.c   |  2 ++
 .../i386/keylocker-aesencwide128kl.c          |  9 +++++
 .../i386/keylocker-aesencwide256kl.c          |  9 +++++
 9 files changed, 72 insertions(+), 5 deletions(-)

Comments

Hongtao Liu July 1, 2021, 8:11 a.m. UTC | #1 
On Thu, Jul 1, 2021 at 3:51 PM Hongyu Wang <hongyu.wang@intel.com> wrote:
>
> For Keylocker aesenc/aesdec intrinsics, current implementation
> moves idata to odata unconditionally, which causes safety issue when
> the instruction meets runtime error. So we add a branch to clear
> odata when ZF is set after instruction exectution.
>
> gcc/ChangeLog:
>
>         * config/i386/i386-expand.c (ix86_expand_builtin):
>         Add branch to clear odata when ZF is set for asedecenc_expand
>         and wideaesdecenc_expand.
>
> gcc/testsuite/ChangeLog:
>
>         * gcc.target/i386/keylocker-aesdec128kl.c: Update test.
>         * gcc.target/i386/keylocker-aesdec256kl.c: Likewise.
>         * gcc.target/i386/keylocker-aesdecwide128kl.c: Likewise.
>         * gcc.target/i386/keylocker-aesdecwide256kl.c: Likewise.
>         * gcc.target/i386/keylocker-aesenc128kl.c: Likewise.
>         * gcc.target/i386/keylocker-aesenc256kl.c: Likewise.
>         * gcc.target/i386/keylocker-aesencwide128kl.c: Likewise.
>         * gcc.target/i386/keylocker-aesencwide256kl.c: Likewise.
> ---
>  gcc/config/i386/i386-expand.c                 | 33 ++++++++++++++++---
>  .../gcc.target/i386/keylocker-aesdec128kl.c   |  2 ++
>  .../gcc.target/i386/keylocker-aesdec256kl.c   |  2 ++
>  .../i386/keylocker-aesdecwide128kl.c          |  9 +++++
>  .../i386/keylocker-aesdecwide256kl.c          |  9 +++++
>  .../gcc.target/i386/keylocker-aesenc128kl.c   |  2 ++
>  .../gcc.target/i386/keylocker-aesenc256kl.c   |  2 ++
>  .../i386/keylocker-aesencwide128kl.c          |  9 +++++
>  .../i386/keylocker-aesencwide256kl.c          |  9 +++++
>  9 files changed, 72 insertions(+), 5 deletions(-)
>
> diff --git a/gcc/config/i386/i386-expand.c b/gcc/config/i386/i386-expand.c
> index e9763eb5b3e..de85f256fee 100644
> --- a/gcc/config/i386/i386-expand.c
> +++ b/gcc/config/i386/i386-expand.c
> @@ -11556,6 +11556,9 @@ ix86_expand_builtin (tree exp, rtx target, rtx subtarget,
>
>      aesdecenc_expand:
>
> +      rtx_code_label *ok_label;
> +      rtx tmp;
> +
>        arg0 = CALL_EXPR_ARG (exp, 0); // __m128i *odata
>        arg1 = CALL_EXPR_ARG (exp, 1); // __m128i idata
>        arg2 = CALL_EXPR_ARG (exp, 2); // const void *p
> @@ -11586,10 +11589,21 @@ ix86_expand_builtin (tree exp, rtx target, rtx subtarget,
>        if (target == 0)
>         target = gen_reg_rtx (QImode);
>
> -      pat = gen_rtx_EQ (QImode, gen_rtx_REG (CCZmode, FLAGS_REG),
> -                       const0_rtx);
> -      emit_insn (gen_rtx_SET (target, pat));
> +      /* NB: For some keylocker insn, ZF will be set when runtime
> +        error occurs. Then the output should be cleared for safety. */
Change some keylocker insn to Keylocker aesenc/aesdec in comments.
others LGTM.
> +      tmp = gen_rtx_REG (CCZmode, FLAGS_REG);
> +      pat = gen_rtx_EQ (QImode, tmp, const0_rtx);
> +      ok_label = gen_label_rtx ();
> +      emit_cmp_and_jump_insns (tmp, const0_rtx, NE, 0, GET_MODE (tmp),
> +                              true, ok_label);
> +      /* Usually the runtime error seldom occur, so predict OK path as
> +        hotspot to optimize it as fallthrough block. */
> +      predict_jump (REG_BR_PROB_BASE * 90 / 100);
> +
> +      emit_insn (gen_rtx_SET (op1, const0_rtx));
>
> +      emit_label (ok_label);
> +      emit_insn (gen_rtx_SET (target, pat));
>        emit_insn (gen_rtx_SET (op0, op1));
>
>        return target;
> @@ -11644,8 +11658,17 @@ ix86_expand_builtin (tree exp, rtx target, rtx subtarget,
>        if (target == 0)
>         target = gen_reg_rtx (QImode);
>
> -      pat = gen_rtx_EQ (QImode, gen_rtx_REG (CCZmode, FLAGS_REG),
> -                       const0_rtx);
> +      tmp = gen_rtx_REG (CCZmode, FLAGS_REG);
> +      pat = gen_rtx_EQ (QImode, tmp, const0_rtx);
> +      ok_label = gen_label_rtx ();
> +      emit_cmp_and_jump_insns (tmp, const0_rtx, NE, 0, GET_MODE (tmp),
> +                              true, ok_label);
> +      predict_jump (REG_BR_PROB_BASE * 90 / 100);
> +
> +      for (i = 0; i < 8; i++)
> +       emit_insn (gen_rtx_SET (xmm_regs[i], const0_rtx));
> +
> +      emit_label (ok_label);
>        emit_insn (gen_rtx_SET (target, pat));
>
>        for (i = 0; i < 8; i++)
> diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesdec128kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesdec128kl.c
> index d134612beea..71111c3206c 100644
> --- a/gcc/testsuite/gcc.target/i386/keylocker-aesdec128kl.c
> +++ b/gcc/testsuite/gcc.target/i386/keylocker-aesdec128kl.c
> @@ -2,8 +2,10 @@
>  /* { dg-options "-mkl -O2" } */
>  /* { dg-final { scan-assembler "movdqa\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
>  /* { dg-final { scan-assembler "aesdec128kl\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
> +/* { dg-final { scan-assembler "j\[ez\]" } } */
>  /* { dg-final { scan-assembler "sete" } } */
>  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
>
>  #include <immintrin.h>
>
> diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesdec256kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesdec256kl.c
> index 34736d2d61a..30189d6ae06 100644
> --- a/gcc/testsuite/gcc.target/i386/keylocker-aesdec256kl.c
> +++ b/gcc/testsuite/gcc.target/i386/keylocker-aesdec256kl.c
> @@ -2,8 +2,10 @@
>  /* { dg-options "-mkl -O2" } */
>  /* { dg-final { scan-assembler "movdqa\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
>  /* { dg-final { scan-assembler "aesdec256kl\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
> +/* { dg-final { scan-assembler "j\[ez\]" } } */
>  /* { dg-final { scan-assembler "sete" } } */
>  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
>
>  #include <immintrin.h>
>
> diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide128kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide128kl.c
> index d23cf4b6517..93806e51508 100644
> --- a/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide128kl.c
> +++ b/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide128kl.c
> @@ -9,6 +9,7 @@
>  /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*96\[^\\n\\r\]*, %xmm6" } } */
>  /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*112\[^\\n\\r\]*, %xmm7" } } */
>  /* { dg-final { scan-assembler "aesdecwide128kl\[ \\t\]+\[^\\n\\r\]*" } } */
> +/* { dg-final { scan-assembler "j\[ez\]" } } */
>  /* { dg-final { scan-assembler "sete" } } */
>  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
>  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm1,\[^\\n\\r\]*16\[^\\n\\r\]*" } } */
> @@ -18,6 +19,14 @@
>  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm5,\[^\\n\\r\]*80\[^\\n\\r\]*" } } */
>  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm6,\[^\\n\\r\]*96\[^\\n\\r\]*" } } */
>  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm7,\[^\\n\\r\]*112\[^\\n\\r\]*" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm1, %xmm1" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm2, %xmm2" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm3, %xmm3" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm4, %xmm4" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm5, %xmm5" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm6, %xmm6" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm7, %xmm7" } } */
>
>  #include <immintrin.h>
>
> diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide256kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide256kl.c
> index 44c3252ab47..f9ccc82c7ca 100644
> --- a/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide256kl.c
> +++ b/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide256kl.c
> @@ -9,6 +9,7 @@
>  /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*96\[^\\n\\r\]*, %xmm6" } } */
>  /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*112\[^\\n\\r\]*, %xmm7" } } */
>  /* { dg-final { scan-assembler "aesdecwide256kl\[ \\t\]+\[^\\n\\r\]*" } } */
> +/* { dg-final { scan-assembler "j\[ez\]" } } */
>  /* { dg-final { scan-assembler "sete" } } */
>  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
>  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm1,\[^\\n\\r\]*16\[^\\n\\r\]*" } } */
> @@ -18,6 +19,14 @@
>  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm5,\[^\\n\\r\]*80\[^\\n\\r\]*" } } */
>  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm6,\[^\\n\\r\]*96\[^\\n\\r\]*" } } */
>  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm7,\[^\\n\\r\]*112\[^\\n\\r\]*" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm1, %xmm1" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm2, %xmm2" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm3, %xmm3" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm4, %xmm4" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm5, %xmm5" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm6, %xmm6" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm7, %xmm7" } } */
>
>  #include <immintrin.h>
>
> diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesenc128kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesenc128kl.c
> index 9ff483656fd..61a9cc279fc 100644
> --- a/gcc/testsuite/gcc.target/i386/keylocker-aesenc128kl.c
> +++ b/gcc/testsuite/gcc.target/i386/keylocker-aesenc128kl.c
> @@ -2,8 +2,10 @@
>  /* { dg-options "-mkl -O2" } */
>  /* { dg-final { scan-assembler "movdqa\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
>  /* { dg-final { scan-assembler "aesenc128kl\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
> +/* { dg-final { scan-assembler "j\[ez\]" } } */
>  /* { dg-final { scan-assembler "sete" } } */
>  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
>
>  #include <immintrin.h>
>
> diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesenc256kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesenc256kl.c
> index 1c5e0765c07..f8e6bb7321f 100644
> --- a/gcc/testsuite/gcc.target/i386/keylocker-aesenc256kl.c
> +++ b/gcc/testsuite/gcc.target/i386/keylocker-aesenc256kl.c
> @@ -2,8 +2,10 @@
>  /* { dg-options "-mkl -O2" } */
>  /* { dg-final { scan-assembler "movdqa\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
>  /* { dg-final { scan-assembler "aesenc256kl\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
> +/* { dg-final { scan-assembler "j\[ez\]" } } */
>  /* { dg-final { scan-assembler "sete" } } */
>  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
>
>  #include <immintrin.h>
>
> diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesencwide128kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesencwide128kl.c
> index 9fb9c49314f..c0fcd28fb07 100644
> --- a/gcc/testsuite/gcc.target/i386/keylocker-aesencwide128kl.c
> +++ b/gcc/testsuite/gcc.target/i386/keylocker-aesencwide128kl.c
> @@ -9,6 +9,7 @@
>  /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*96\[^\\n\\r\]*, %xmm6" } } */
>  /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*112\[^\\n\\r\]*, %xmm7" } } */
>  /* { dg-final { scan-assembler "aesencwide128kl\[ \\t\]+\[^\\n\\r\]*" } } */
> +/* { dg-final { scan-assembler "j\[ez\]" } } */
>  /* { dg-final { scan-assembler "sete" } } */
>  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
>  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm1,\[^\\n\\r\]*16\[^\\n\\r\]*" } } */
> @@ -18,6 +19,14 @@
>  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm5,\[^\\n\\r\]*80\[^\\n\\r\]*" } } */
>  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm6,\[^\\n\\r\]*96\[^\\n\\r\]*" } } */
>  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm7,\[^\\n\\r\]*112\[^\\n\\r\]*" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm1, %xmm1" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm2, %xmm2" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm3, %xmm3" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm4, %xmm4" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm5, %xmm5" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm6, %xmm6" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm7, %xmm7" } } */
>
>  #include <immintrin.h>
>
> diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesencwide256kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesencwide256kl.c
> index 125a787dcd9..31463a8b2da 100644
> --- a/gcc/testsuite/gcc.target/i386/keylocker-aesencwide256kl.c
> +++ b/gcc/testsuite/gcc.target/i386/keylocker-aesencwide256kl.c
> @@ -9,6 +9,7 @@
>  /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*96\[^\\n\\r\]*, %xmm6" } } */
>  /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*112\[^\\n\\r\]*, %xmm7" } } */
>  /* { dg-final { scan-assembler "aesencwide256kl\[ \\t\]+\[^\\n\\r\]*" } } */
> +/* { dg-final { scan-assembler "j\[ez\]" } } */
>  /* { dg-final { scan-assembler "sete" } } */
>  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
>  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm1,\[^\\n\\r\]*16\[^\\n\\r\]*" } } */
> @@ -18,6 +19,14 @@
>  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm5,\[^\\n\\r\]*80\[^\\n\\r\]*" } } */
>  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm6,\[^\\n\\r\]*96\[^\\n\\r\]*" } } */
>  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm7,\[^\\n\\r\]*112\[^\\n\\r\]*" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm1, %xmm1" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm2, %xmm2" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm3, %xmm3" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm4, %xmm4" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm5, %xmm5" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm6, %xmm6" } } */
> +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm7, %xmm7" } } */
>
>  #include <immintrin.h>
>
> --
> 2.18.2
>


--
BR,
Hongtao
Hongyu Wang July 1, 2021, 8:16 a.m. UTC | #2 
> Change some keylocker insn to Keylocker aesenc/aesdec in comments.
> others LGTM.

Changed.

Forgot to mention bootstrapped and regression tested on
x86_64-linux-gnu {,-m32}.

Attached file is the patch i'm going to check-in.

Hongtao Liu via Gcc-patches <gcc-patches@gcc.gnu.org> 于2021年7月1日周四 下午4:07写道：
>
> On Thu, Jul 1, 2021 at 3:51 PM Hongyu Wang <hongyu.wang@intel.com> wrote:
> >
> > For Keylocker aesenc/aesdec intrinsics, current implementation
> > moves idata to odata unconditionally, which causes safety issue when
> > the instruction meets runtime error. So we add a branch to clear
> > odata when ZF is set after instruction exectution.
> >
> > gcc/ChangeLog:
> >
> >         * config/i386/i386-expand.c (ix86_expand_builtin):
> >         Add branch to clear odata when ZF is set for asedecenc_expand
> >         and wideaesdecenc_expand.
> >
> > gcc/testsuite/ChangeLog:
> >
> >         * gcc.target/i386/keylocker-aesdec128kl.c: Update test.
> >         * gcc.target/i386/keylocker-aesdec256kl.c: Likewise.
> >         * gcc.target/i386/keylocker-aesdecwide128kl.c: Likewise.
> >         * gcc.target/i386/keylocker-aesdecwide256kl.c: Likewise.
> >         * gcc.target/i386/keylocker-aesenc128kl.c: Likewise.
> >         * gcc.target/i386/keylocker-aesenc256kl.c: Likewise.
> >         * gcc.target/i386/keylocker-aesencwide128kl.c: Likewise.
> >         * gcc.target/i386/keylocker-aesencwide256kl.c: Likewise.
> > ---
> >  gcc/config/i386/i386-expand.c                 | 33 ++++++++++++++++---
> >  .../gcc.target/i386/keylocker-aesdec128kl.c   |  2 ++
> >  .../gcc.target/i386/keylocker-aesdec256kl.c   |  2 ++
> >  .../i386/keylocker-aesdecwide128kl.c          |  9 +++++
> >  .../i386/keylocker-aesdecwide256kl.c          |  9 +++++
> >  .../gcc.target/i386/keylocker-aesenc128kl.c   |  2 ++
> >  .../gcc.target/i386/keylocker-aesenc256kl.c   |  2 ++
> >  .../i386/keylocker-aesencwide128kl.c          |  9 +++++
> >  .../i386/keylocker-aesencwide256kl.c          |  9 +++++
> >  9 files changed, 72 insertions(+), 5 deletions(-)
> >
> > diff --git a/gcc/config/i386/i386-expand.c b/gcc/config/i386/i386-expand.c
> > index e9763eb5b3e..de85f256fee 100644
> > --- a/gcc/config/i386/i386-expand.c
> > +++ b/gcc/config/i386/i386-expand.c
> > @@ -11556,6 +11556,9 @@ ix86_expand_builtin (tree exp, rtx target, rtx subtarget,
> >
> >      aesdecenc_expand:
> >
> > +      rtx_code_label *ok_label;
> > +      rtx tmp;
> > +
> >        arg0 = CALL_EXPR_ARG (exp, 0); // __m128i *odata
> >        arg1 = CALL_EXPR_ARG (exp, 1); // __m128i idata
> >        arg2 = CALL_EXPR_ARG (exp, 2); // const void *p
> > @@ -11586,10 +11589,21 @@ ix86_expand_builtin (tree exp, rtx target, rtx subtarget,
> >        if (target == 0)
> >         target = gen_reg_rtx (QImode);
> >
> > -      pat = gen_rtx_EQ (QImode, gen_rtx_REG (CCZmode, FLAGS_REG),
> > -                       const0_rtx);
> > -      emit_insn (gen_rtx_SET (target, pat));
> > +      /* NB: For some keylocker insn, ZF will be set when runtime
> > +        error occurs. Then the output should be cleared for safety. */
> Change some keylocker insn to Keylocker aesenc/aesdec in comments.
> others LGTM.
> > +      tmp = gen_rtx_REG (CCZmode, FLAGS_REG);
> > +      pat = gen_rtx_EQ (QImode, tmp, const0_rtx);
> > +      ok_label = gen_label_rtx ();
> > +      emit_cmp_and_jump_insns (tmp, const0_rtx, NE, 0, GET_MODE (tmp),
> > +                              true, ok_label);
> > +      /* Usually the runtime error seldom occur, so predict OK path as
> > +        hotspot to optimize it as fallthrough block. */
> > +      predict_jump (REG_BR_PROB_BASE * 90 / 100);
> > +
> > +      emit_insn (gen_rtx_SET (op1, const0_rtx));
> >
> > +      emit_label (ok_label);
> > +      emit_insn (gen_rtx_SET (target, pat));
> >        emit_insn (gen_rtx_SET (op0, op1));
> >
> >        return target;
> > @@ -11644,8 +11658,17 @@ ix86_expand_builtin (tree exp, rtx target, rtx subtarget,
> >        if (target == 0)
> >         target = gen_reg_rtx (QImode);
> >
> > -      pat = gen_rtx_EQ (QImode, gen_rtx_REG (CCZmode, FLAGS_REG),
> > -                       const0_rtx);
> > +      tmp = gen_rtx_REG (CCZmode, FLAGS_REG);
> > +      pat = gen_rtx_EQ (QImode, tmp, const0_rtx);
> > +      ok_label = gen_label_rtx ();
> > +      emit_cmp_and_jump_insns (tmp, const0_rtx, NE, 0, GET_MODE (tmp),
> > +                              true, ok_label);
> > +      predict_jump (REG_BR_PROB_BASE * 90 / 100);
> > +
> > +      for (i = 0; i < 8; i++)
> > +       emit_insn (gen_rtx_SET (xmm_regs[i], const0_rtx));
> > +
> > +      emit_label (ok_label);
> >        emit_insn (gen_rtx_SET (target, pat));
> >
> >        for (i = 0; i < 8; i++)
> > diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesdec128kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesdec128kl.c
> > index d134612beea..71111c3206c 100644
> > --- a/gcc/testsuite/gcc.target/i386/keylocker-aesdec128kl.c
> > +++ b/gcc/testsuite/gcc.target/i386/keylocker-aesdec128kl.c
> > @@ -2,8 +2,10 @@
> >  /* { dg-options "-mkl -O2" } */
> >  /* { dg-final { scan-assembler "movdqa\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
> >  /* { dg-final { scan-assembler "aesdec128kl\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
> > +/* { dg-final { scan-assembler "j\[ez\]" } } */
> >  /* { dg-final { scan-assembler "sete" } } */
> >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
> >
> >  #include <immintrin.h>
> >
> > diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesdec256kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesdec256kl.c
> > index 34736d2d61a..30189d6ae06 100644
> > --- a/gcc/testsuite/gcc.target/i386/keylocker-aesdec256kl.c
> > +++ b/gcc/testsuite/gcc.target/i386/keylocker-aesdec256kl.c
> > @@ -2,8 +2,10 @@
> >  /* { dg-options "-mkl -O2" } */
> >  /* { dg-final { scan-assembler "movdqa\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
> >  /* { dg-final { scan-assembler "aesdec256kl\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
> > +/* { dg-final { scan-assembler "j\[ez\]" } } */
> >  /* { dg-final { scan-assembler "sete" } } */
> >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
> >
> >  #include <immintrin.h>
> >
> > diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide128kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide128kl.c
> > index d23cf4b6517..93806e51508 100644
> > --- a/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide128kl.c
> > +++ b/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide128kl.c
> > @@ -9,6 +9,7 @@
> >  /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*96\[^\\n\\r\]*, %xmm6" } } */
> >  /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*112\[^\\n\\r\]*, %xmm7" } } */
> >  /* { dg-final { scan-assembler "aesdecwide128kl\[ \\t\]+\[^\\n\\r\]*" } } */
> > +/* { dg-final { scan-assembler "j\[ez\]" } } */
> >  /* { dg-final { scan-assembler "sete" } } */
> >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
> >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm1,\[^\\n\\r\]*16\[^\\n\\r\]*" } } */
> > @@ -18,6 +19,14 @@
> >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm5,\[^\\n\\r\]*80\[^\\n\\r\]*" } } */
> >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm6,\[^\\n\\r\]*96\[^\\n\\r\]*" } } */
> >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm7,\[^\\n\\r\]*112\[^\\n\\r\]*" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm1, %xmm1" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm2, %xmm2" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm3, %xmm3" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm4, %xmm4" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm5, %xmm5" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm6, %xmm6" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm7, %xmm7" } } */
> >
> >  #include <immintrin.h>
> >
> > diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide256kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide256kl.c
> > index 44c3252ab47..f9ccc82c7ca 100644
> > --- a/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide256kl.c
> > +++ b/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide256kl.c
> > @@ -9,6 +9,7 @@
> >  /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*96\[^\\n\\r\]*, %xmm6" } } */
> >  /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*112\[^\\n\\r\]*, %xmm7" } } */
> >  /* { dg-final { scan-assembler "aesdecwide256kl\[ \\t\]+\[^\\n\\r\]*" } } */
> > +/* { dg-final { scan-assembler "j\[ez\]" } } */
> >  /* { dg-final { scan-assembler "sete" } } */
> >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
> >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm1,\[^\\n\\r\]*16\[^\\n\\r\]*" } } */
> > @@ -18,6 +19,14 @@
> >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm5,\[^\\n\\r\]*80\[^\\n\\r\]*" } } */
> >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm6,\[^\\n\\r\]*96\[^\\n\\r\]*" } } */
> >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm7,\[^\\n\\r\]*112\[^\\n\\r\]*" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm1, %xmm1" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm2, %xmm2" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm3, %xmm3" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm4, %xmm4" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm5, %xmm5" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm6, %xmm6" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm7, %xmm7" } } */
> >
> >  #include <immintrin.h>
> >
> > diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesenc128kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesenc128kl.c
> > index 9ff483656fd..61a9cc279fc 100644
> > --- a/gcc/testsuite/gcc.target/i386/keylocker-aesenc128kl.c
> > +++ b/gcc/testsuite/gcc.target/i386/keylocker-aesenc128kl.c
> > @@ -2,8 +2,10 @@
> >  /* { dg-options "-mkl -O2" } */
> >  /* { dg-final { scan-assembler "movdqa\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
> >  /* { dg-final { scan-assembler "aesenc128kl\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
> > +/* { dg-final { scan-assembler "j\[ez\]" } } */
> >  /* { dg-final { scan-assembler "sete" } } */
> >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
> >
> >  #include <immintrin.h>
> >
> > diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesenc256kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesenc256kl.c
> > index 1c5e0765c07..f8e6bb7321f 100644
> > --- a/gcc/testsuite/gcc.target/i386/keylocker-aesenc256kl.c
> > +++ b/gcc/testsuite/gcc.target/i386/keylocker-aesenc256kl.c
> > @@ -2,8 +2,10 @@
> >  /* { dg-options "-mkl -O2" } */
> >  /* { dg-final { scan-assembler "movdqa\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
> >  /* { dg-final { scan-assembler "aesenc256kl\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
> > +/* { dg-final { scan-assembler "j\[ez\]" } } */
> >  /* { dg-final { scan-assembler "sete" } } */
> >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
> >
> >  #include <immintrin.h>
> >
> > diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesencwide128kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesencwide128kl.c
> > index 9fb9c49314f..c0fcd28fb07 100644
> > --- a/gcc/testsuite/gcc.target/i386/keylocker-aesencwide128kl.c
> > +++ b/gcc/testsuite/gcc.target/i386/keylocker-aesencwide128kl.c
> > @@ -9,6 +9,7 @@
> >  /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*96\[^\\n\\r\]*, %xmm6" } } */
> >  /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*112\[^\\n\\r\]*, %xmm7" } } */
> >  /* { dg-final { scan-assembler "aesencwide128kl\[ \\t\]+\[^\\n\\r\]*" } } */
> > +/* { dg-final { scan-assembler "j\[ez\]" } } */
> >  /* { dg-final { scan-assembler "sete" } } */
> >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
> >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm1,\[^\\n\\r\]*16\[^\\n\\r\]*" } } */
> > @@ -18,6 +19,14 @@
> >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm5,\[^\\n\\r\]*80\[^\\n\\r\]*" } } */
> >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm6,\[^\\n\\r\]*96\[^\\n\\r\]*" } } */
> >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm7,\[^\\n\\r\]*112\[^\\n\\r\]*" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm1, %xmm1" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm2, %xmm2" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm3, %xmm3" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm4, %xmm4" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm5, %xmm5" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm6, %xmm6" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm7, %xmm7" } } */
> >
> >  #include <immintrin.h>
> >
> > diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesencwide256kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesencwide256kl.c
> > index 125a787dcd9..31463a8b2da 100644
> > --- a/gcc/testsuite/gcc.target/i386/keylocker-aesencwide256kl.c
> > +++ b/gcc/testsuite/gcc.target/i386/keylocker-aesencwide256kl.c
> > @@ -9,6 +9,7 @@
> >  /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*96\[^\\n\\r\]*, %xmm6" } } */
> >  /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*112\[^\\n\\r\]*, %xmm7" } } */
> >  /* { dg-final { scan-assembler "aesencwide256kl\[ \\t\]+\[^\\n\\r\]*" } } */
> > +/* { dg-final { scan-assembler "j\[ez\]" } } */
> >  /* { dg-final { scan-assembler "sete" } } */
> >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
> >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm1,\[^\\n\\r\]*16\[^\\n\\r\]*" } } */
> > @@ -18,6 +19,14 @@
> >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm5,\[^\\n\\r\]*80\[^\\n\\r\]*" } } */
> >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm6,\[^\\n\\r\]*96\[^\\n\\r\]*" } } */
> >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm7,\[^\\n\\r\]*112\[^\\n\\r\]*" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm1, %xmm1" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm2, %xmm2" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm3, %xmm3" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm4, %xmm4" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm5, %xmm5" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm6, %xmm6" } } */
> > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm7, %xmm7" } } */
> >
> >  #include <immintrin.h>
> >
> > --
> > 2.18.2
> >
>
>
> --
> BR,
> Hongtao
Hongyu Wang July 2, 2021, 3:12 a.m. UTC | #3 
Updated patch with minor change to move the variable declaration after comment.

Hongtao, could you help check-in the patch?

Hongyu Wang <wwwhhhyyy333@gmail.com> 于2021年7月1日周四 下午4:16写道：
>
> > Change some keylocker insn to Keylocker aesenc/aesdec in comments.
> > others LGTM.
>
> Changed.
>
> Forgot to mention bootstrapped and regression tested on
> x86_64-linux-gnu {,-m32}.
>
> Attached file is the patch i'm going to check-in.
>
> Hongtao Liu via Gcc-patches <gcc-patches@gcc.gnu.org> 于2021年7月1日周四 下午4:07写道：
> >
> > On Thu, Jul 1, 2021 at 3:51 PM Hongyu Wang <hongyu.wang@intel.com> wrote:
> > >
> > > For Keylocker aesenc/aesdec intrinsics, current implementation
> > > moves idata to odata unconditionally, which causes safety issue when
> > > the instruction meets runtime error. So we add a branch to clear
> > > odata when ZF is set after instruction exectution.
> > >
> > > gcc/ChangeLog:
> > >
> > >         * config/i386/i386-expand.c (ix86_expand_builtin):
> > >         Add branch to clear odata when ZF is set for asedecenc_expand
> > >         and wideaesdecenc_expand.
> > >
> > > gcc/testsuite/ChangeLog:
> > >
> > >         * gcc.target/i386/keylocker-aesdec128kl.c: Update test.
> > >         * gcc.target/i386/keylocker-aesdec256kl.c: Likewise.
> > >         * gcc.target/i386/keylocker-aesdecwide128kl.c: Likewise.
> > >         * gcc.target/i386/keylocker-aesdecwide256kl.c: Likewise.
> > >         * gcc.target/i386/keylocker-aesenc128kl.c: Likewise.
> > >         * gcc.target/i386/keylocker-aesenc256kl.c: Likewise.
> > >         * gcc.target/i386/keylocker-aesencwide128kl.c: Likewise.
> > >         * gcc.target/i386/keylocker-aesencwide256kl.c: Likewise.
> > > ---
> > >  gcc/config/i386/i386-expand.c                 | 33 ++++++++++++++++---
> > >  .../gcc.target/i386/keylocker-aesdec128kl.c   |  2 ++
> > >  .../gcc.target/i386/keylocker-aesdec256kl.c   |  2 ++
> > >  .../i386/keylocker-aesdecwide128kl.c          |  9 +++++
> > >  .../i386/keylocker-aesdecwide256kl.c          |  9 +++++
> > >  .../gcc.target/i386/keylocker-aesenc128kl.c   |  2 ++
> > >  .../gcc.target/i386/keylocker-aesenc256kl.c   |  2 ++
> > >  .../i386/keylocker-aesencwide128kl.c          |  9 +++++
> > >  .../i386/keylocker-aesencwide256kl.c          |  9 +++++
> > >  9 files changed, 72 insertions(+), 5 deletions(-)
> > >
> > > diff --git a/gcc/config/i386/i386-expand.c b/gcc/config/i386/i386-expand.c
> > > index e9763eb5b3e..de85f256fee 100644
> > > --- a/gcc/config/i386/i386-expand.c
> > > +++ b/gcc/config/i386/i386-expand.c
> > > @@ -11556,6 +11556,9 @@ ix86_expand_builtin (tree exp, rtx target, rtx subtarget,
> > >
> > >      aesdecenc_expand:
> > >
> > > +      rtx_code_label *ok_label;
> > > +      rtx tmp;
> > > +
> > >        arg0 = CALL_EXPR_ARG (exp, 0); // __m128i *odata
> > >        arg1 = CALL_EXPR_ARG (exp, 1); // __m128i idata
> > >        arg2 = CALL_EXPR_ARG (exp, 2); // const void *p
> > > @@ -11586,10 +11589,21 @@ ix86_expand_builtin (tree exp, rtx target, rtx subtarget,
> > >        if (target == 0)
> > >         target = gen_reg_rtx (QImode);
> > >
> > > -      pat = gen_rtx_EQ (QImode, gen_rtx_REG (CCZmode, FLAGS_REG),
> > > -                       const0_rtx);
> > > -      emit_insn (gen_rtx_SET (target, pat));
> > > +      /* NB: For some keylocker insn, ZF will be set when runtime
> > > +        error occurs. Then the output should be cleared for safety. */
> > Change some keylocker insn to Keylocker aesenc/aesdec in comments.
> > others LGTM.
> > > +      tmp = gen_rtx_REG (CCZmode, FLAGS_REG);
> > > +      pat = gen_rtx_EQ (QImode, tmp, const0_rtx);
> > > +      ok_label = gen_label_rtx ();
> > > +      emit_cmp_and_jump_insns (tmp, const0_rtx, NE, 0, GET_MODE (tmp),
> > > +                              true, ok_label);
> > > +      /* Usually the runtime error seldom occur, so predict OK path as
> > > +        hotspot to optimize it as fallthrough block. */
> > > +      predict_jump (REG_BR_PROB_BASE * 90 / 100);
> > > +
> > > +      emit_insn (gen_rtx_SET (op1, const0_rtx));
> > >
> > > +      emit_label (ok_label);
> > > +      emit_insn (gen_rtx_SET (target, pat));
> > >        emit_insn (gen_rtx_SET (op0, op1));
> > >
> > >        return target;
> > > @@ -11644,8 +11658,17 @@ ix86_expand_builtin (tree exp, rtx target, rtx subtarget,
> > >        if (target == 0)
> > >         target = gen_reg_rtx (QImode);
> > >
> > > -      pat = gen_rtx_EQ (QImode, gen_rtx_REG (CCZmode, FLAGS_REG),
> > > -                       const0_rtx);
> > > +      tmp = gen_rtx_REG (CCZmode, FLAGS_REG);
> > > +      pat = gen_rtx_EQ (QImode, tmp, const0_rtx);
> > > +      ok_label = gen_label_rtx ();
> > > +      emit_cmp_and_jump_insns (tmp, const0_rtx, NE, 0, GET_MODE (tmp),
> > > +                              true, ok_label);
> > > +      predict_jump (REG_BR_PROB_BASE * 90 / 100);
> > > +
> > > +      for (i = 0; i < 8; i++)
> > > +       emit_insn (gen_rtx_SET (xmm_regs[i], const0_rtx));
> > > +
> > > +      emit_label (ok_label);
> > >        emit_insn (gen_rtx_SET (target, pat));
> > >
> > >        for (i = 0; i < 8; i++)
> > > diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesdec128kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesdec128kl.c
> > > index d134612beea..71111c3206c 100644
> > > --- a/gcc/testsuite/gcc.target/i386/keylocker-aesdec128kl.c
> > > +++ b/gcc/testsuite/gcc.target/i386/keylocker-aesdec128kl.c
> > > @@ -2,8 +2,10 @@
> > >  /* { dg-options "-mkl -O2" } */
> > >  /* { dg-final { scan-assembler "movdqa\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
> > >  /* { dg-final { scan-assembler "aesdec128kl\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
> > > +/* { dg-final { scan-assembler "j\[ez\]" } } */
> > >  /* { dg-final { scan-assembler "sete" } } */
> > >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
> > >
> > >  #include <immintrin.h>
> > >
> > > diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesdec256kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesdec256kl.c
> > > index 34736d2d61a..30189d6ae06 100644
> > > --- a/gcc/testsuite/gcc.target/i386/keylocker-aesdec256kl.c
> > > +++ b/gcc/testsuite/gcc.target/i386/keylocker-aesdec256kl.c
> > > @@ -2,8 +2,10 @@
> > >  /* { dg-options "-mkl -O2" } */
> > >  /* { dg-final { scan-assembler "movdqa\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
> > >  /* { dg-final { scan-assembler "aesdec256kl\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
> > > +/* { dg-final { scan-assembler "j\[ez\]" } } */
> > >  /* { dg-final { scan-assembler "sete" } } */
> > >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
> > >
> > >  #include <immintrin.h>
> > >
> > > diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide128kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide128kl.c
> > > index d23cf4b6517..93806e51508 100644
> > > --- a/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide128kl.c
> > > +++ b/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide128kl.c
> > > @@ -9,6 +9,7 @@
> > >  /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*96\[^\\n\\r\]*, %xmm6" } } */
> > >  /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*112\[^\\n\\r\]*, %xmm7" } } */
> > >  /* { dg-final { scan-assembler "aesdecwide128kl\[ \\t\]+\[^\\n\\r\]*" } } */
> > > +/* { dg-final { scan-assembler "j\[ez\]" } } */
> > >  /* { dg-final { scan-assembler "sete" } } */
> > >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
> > >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm1,\[^\\n\\r\]*16\[^\\n\\r\]*" } } */
> > > @@ -18,6 +19,14 @@
> > >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm5,\[^\\n\\r\]*80\[^\\n\\r\]*" } } */
> > >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm6,\[^\\n\\r\]*96\[^\\n\\r\]*" } } */
> > >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm7,\[^\\n\\r\]*112\[^\\n\\r\]*" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm1, %xmm1" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm2, %xmm2" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm3, %xmm3" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm4, %xmm4" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm5, %xmm5" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm6, %xmm6" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm7, %xmm7" } } */
> > >
> > >  #include <immintrin.h>
> > >
> > > diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide256kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide256kl.c
> > > index 44c3252ab47..f9ccc82c7ca 100644
> > > --- a/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide256kl.c
> > > +++ b/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide256kl.c
> > > @@ -9,6 +9,7 @@
> > >  /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*96\[^\\n\\r\]*, %xmm6" } } */
> > >  /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*112\[^\\n\\r\]*, %xmm7" } } */
> > >  /* { dg-final { scan-assembler "aesdecwide256kl\[ \\t\]+\[^\\n\\r\]*" } } */
> > > +/* { dg-final { scan-assembler "j\[ez\]" } } */
> > >  /* { dg-final { scan-assembler "sete" } } */
> > >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
> > >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm1,\[^\\n\\r\]*16\[^\\n\\r\]*" } } */
> > > @@ -18,6 +19,14 @@
> > >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm5,\[^\\n\\r\]*80\[^\\n\\r\]*" } } */
> > >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm6,\[^\\n\\r\]*96\[^\\n\\r\]*" } } */
> > >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm7,\[^\\n\\r\]*112\[^\\n\\r\]*" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm1, %xmm1" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm2, %xmm2" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm3, %xmm3" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm4, %xmm4" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm5, %xmm5" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm6, %xmm6" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm7, %xmm7" } } */
> > >
> > >  #include <immintrin.h>
> > >
> > > diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesenc128kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesenc128kl.c
> > > index 9ff483656fd..61a9cc279fc 100644
> > > --- a/gcc/testsuite/gcc.target/i386/keylocker-aesenc128kl.c
> > > +++ b/gcc/testsuite/gcc.target/i386/keylocker-aesenc128kl.c
> > > @@ -2,8 +2,10 @@
> > >  /* { dg-options "-mkl -O2" } */
> > >  /* { dg-final { scan-assembler "movdqa\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
> > >  /* { dg-final { scan-assembler "aesenc128kl\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
> > > +/* { dg-final { scan-assembler "j\[ez\]" } } */
> > >  /* { dg-final { scan-assembler "sete" } } */
> > >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
> > >
> > >  #include <immintrin.h>
> > >
> > > diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesenc256kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesenc256kl.c
> > > index 1c5e0765c07..f8e6bb7321f 100644
> > > --- a/gcc/testsuite/gcc.target/i386/keylocker-aesenc256kl.c
> > > +++ b/gcc/testsuite/gcc.target/i386/keylocker-aesenc256kl.c
> > > @@ -2,8 +2,10 @@
> > >  /* { dg-options "-mkl -O2" } */
> > >  /* { dg-final { scan-assembler "movdqa\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
> > >  /* { dg-final { scan-assembler "aesenc256kl\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
> > > +/* { dg-final { scan-assembler "j\[ez\]" } } */
> > >  /* { dg-final { scan-assembler "sete" } } */
> > >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
> > >
> > >  #include <immintrin.h>
> > >
> > > diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesencwide128kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesencwide128kl.c
> > > index 9fb9c49314f..c0fcd28fb07 100644
> > > --- a/gcc/testsuite/gcc.target/i386/keylocker-aesencwide128kl.c
> > > +++ b/gcc/testsuite/gcc.target/i386/keylocker-aesencwide128kl.c
> > > @@ -9,6 +9,7 @@
> > >  /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*96\[^\\n\\r\]*, %xmm6" } } */
> > >  /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*112\[^\\n\\r\]*, %xmm7" } } */
> > >  /* { dg-final { scan-assembler "aesencwide128kl\[ \\t\]+\[^\\n\\r\]*" } } */
> > > +/* { dg-final { scan-assembler "j\[ez\]" } } */
> > >  /* { dg-final { scan-assembler "sete" } } */
> > >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
> > >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm1,\[^\\n\\r\]*16\[^\\n\\r\]*" } } */
> > > @@ -18,6 +19,14 @@
> > >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm5,\[^\\n\\r\]*80\[^\\n\\r\]*" } } */
> > >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm6,\[^\\n\\r\]*96\[^\\n\\r\]*" } } */
> > >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm7,\[^\\n\\r\]*112\[^\\n\\r\]*" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm1, %xmm1" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm2, %xmm2" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm3, %xmm3" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm4, %xmm4" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm5, %xmm5" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm6, %xmm6" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm7, %xmm7" } } */
> > >
> > >  #include <immintrin.h>
> > >
> > > diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesencwide256kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesencwide256kl.c
> > > index 125a787dcd9..31463a8b2da 100644
> > > --- a/gcc/testsuite/gcc.target/i386/keylocker-aesencwide256kl.c
> > > +++ b/gcc/testsuite/gcc.target/i386/keylocker-aesencwide256kl.c
> > > @@ -9,6 +9,7 @@
> > >  /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*96\[^\\n\\r\]*, %xmm6" } } */
> > >  /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*112\[^\\n\\r\]*, %xmm7" } } */
> > >  /* { dg-final { scan-assembler "aesencwide256kl\[ \\t\]+\[^\\n\\r\]*" } } */
> > > +/* { dg-final { scan-assembler "j\[ez\]" } } */
> > >  /* { dg-final { scan-assembler "sete" } } */
> > >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
> > >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm1,\[^\\n\\r\]*16\[^\\n\\r\]*" } } */
> > > @@ -18,6 +19,14 @@
> > >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm5,\[^\\n\\r\]*80\[^\\n\\r\]*" } } */
> > >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm6,\[^\\n\\r\]*96\[^\\n\\r\]*" } } */
> > >  /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm7,\[^\\n\\r\]*112\[^\\n\\r\]*" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm1, %xmm1" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm2, %xmm2" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm3, %xmm3" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm4, %xmm4" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm5, %xmm5" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm6, %xmm6" } } */
> > > +/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm7, %xmm7" } } */
> > >
> > >  #include <immintrin.h>
> > >
> > > --
> > > 2.18.2
> > >
> >
> >
> > --
> > BR,
> > Hongtao
diff mbox series

Patch

diff --git a/gcc/config/i386/i386-expand.c b/gcc/config/i386/i386-expand.c
index e9763eb5b3e..de85f256fee 100644
--- a/gcc/config/i386/i386-expand.c
+++ b/gcc/config/i386/i386-expand.c
@@ -11556,6 +11556,9 @@  ix86_expand_builtin (tree exp, rtx target, rtx subtarget,
 
     aesdecenc_expand:
 
+      rtx_code_label *ok_label;
+      rtx tmp;
+
       arg0 = CALL_EXPR_ARG (exp, 0); // __m128i *odata
       arg1 = CALL_EXPR_ARG (exp, 1); // __m128i idata
       arg2 = CALL_EXPR_ARG (exp, 2); // const void *p
@@ -11586,10 +11589,21 @@  ix86_expand_builtin (tree exp, rtx target, rtx subtarget,
       if (target == 0)
 	target = gen_reg_rtx (QImode);
 
-      pat = gen_rtx_EQ (QImode, gen_rtx_REG (CCZmode, FLAGS_REG),
-			const0_rtx);
-      emit_insn (gen_rtx_SET (target, pat));
+      /* NB: For some keylocker insn, ZF will be set when runtime
+	 error occurs. Then the output should be cleared for safety. */
+      tmp = gen_rtx_REG (CCZmode, FLAGS_REG);
+      pat = gen_rtx_EQ (QImode, tmp, const0_rtx);
+      ok_label = gen_label_rtx ();
+      emit_cmp_and_jump_insns (tmp, const0_rtx, NE, 0, GET_MODE (tmp),
+			       true, ok_label);
+      /* Usually the runtime error seldom occur, so predict OK path as
+	 hotspot to optimize it as fallthrough block. */
+      predict_jump (REG_BR_PROB_BASE * 90 / 100);
+
+      emit_insn (gen_rtx_SET (op1, const0_rtx));
 
+      emit_label (ok_label);
+      emit_insn (gen_rtx_SET (target, pat));
       emit_insn (gen_rtx_SET (op0, op1));
 
       return target;
@@ -11644,8 +11658,17 @@  ix86_expand_builtin (tree exp, rtx target, rtx subtarget,
       if (target == 0)
 	target = gen_reg_rtx (QImode);
 
-      pat = gen_rtx_EQ (QImode, gen_rtx_REG (CCZmode, FLAGS_REG),
-			const0_rtx);
+      tmp = gen_rtx_REG (CCZmode, FLAGS_REG);
+      pat = gen_rtx_EQ (QImode, tmp, const0_rtx);
+      ok_label = gen_label_rtx ();
+      emit_cmp_and_jump_insns (tmp, const0_rtx, NE, 0, GET_MODE (tmp),
+			       true, ok_label);
+      predict_jump (REG_BR_PROB_BASE * 90 / 100);
+
+      for (i = 0; i < 8; i++)
+	emit_insn (gen_rtx_SET (xmm_regs[i], const0_rtx));
+
+      emit_label (ok_label);
       emit_insn (gen_rtx_SET (target, pat));
 
       for (i = 0; i < 8; i++)
diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesdec128kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesdec128kl.c
index d134612beea..71111c3206c 100644
--- a/gcc/testsuite/gcc.target/i386/keylocker-aesdec128kl.c
+++ b/gcc/testsuite/gcc.target/i386/keylocker-aesdec128kl.c
@@ -2,8 +2,10 @@ 
 /* { dg-options "-mkl -O2" } */
 /* { dg-final { scan-assembler "movdqa\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
 /* { dg-final { scan-assembler "aesdec128kl\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
+/* { dg-final { scan-assembler "j\[ez\]" } } */
 /* { dg-final { scan-assembler "sete" } } */
 /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
 
 #include <immintrin.h>
 
diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesdec256kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesdec256kl.c
index 34736d2d61a..30189d6ae06 100644
--- a/gcc/testsuite/gcc.target/i386/keylocker-aesdec256kl.c
+++ b/gcc/testsuite/gcc.target/i386/keylocker-aesdec256kl.c
@@ -2,8 +2,10 @@ 
 /* { dg-options "-mkl -O2" } */
 /* { dg-final { scan-assembler "movdqa\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
 /* { dg-final { scan-assembler "aesdec256kl\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
+/* { dg-final { scan-assembler "j\[ez\]" } } */
 /* { dg-final { scan-assembler "sete" } } */
 /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
 
 #include <immintrin.h>
 
diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide128kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide128kl.c
index d23cf4b6517..93806e51508 100644
--- a/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide128kl.c
+++ b/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide128kl.c
@@ -9,6 +9,7 @@ 
 /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*96\[^\\n\\r\]*, %xmm6" } } */
 /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*112\[^\\n\\r\]*, %xmm7" } } */
 /* { dg-final { scan-assembler "aesdecwide128kl\[ \\t\]+\[^\\n\\r\]*" } } */
+/* { dg-final { scan-assembler "j\[ez\]" } } */
 /* { dg-final { scan-assembler "sete" } } */
 /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
 /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm1,\[^\\n\\r\]*16\[^\\n\\r\]*" } } */
@@ -18,6 +19,14 @@ 
 /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm5,\[^\\n\\r\]*80\[^\\n\\r\]*" } } */
 /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm6,\[^\\n\\r\]*96\[^\\n\\r\]*" } } */
 /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm7,\[^\\n\\r\]*112\[^\\n\\r\]*" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm1, %xmm1" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm2, %xmm2" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm3, %xmm3" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm4, %xmm4" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm5, %xmm5" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm6, %xmm6" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm7, %xmm7" } } */
 
 #include <immintrin.h>
 
diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide256kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide256kl.c
index 44c3252ab47..f9ccc82c7ca 100644
--- a/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide256kl.c
+++ b/gcc/testsuite/gcc.target/i386/keylocker-aesdecwide256kl.c
@@ -9,6 +9,7 @@ 
 /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*96\[^\\n\\r\]*, %xmm6" } } */
 /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*112\[^\\n\\r\]*, %xmm7" } } */
 /* { dg-final { scan-assembler "aesdecwide256kl\[ \\t\]+\[^\\n\\r\]*" } } */
+/* { dg-final { scan-assembler "j\[ez\]" } } */
 /* { dg-final { scan-assembler "sete" } } */
 /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
 /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm1,\[^\\n\\r\]*16\[^\\n\\r\]*" } } */
@@ -18,6 +19,14 @@ 
 /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm5,\[^\\n\\r\]*80\[^\\n\\r\]*" } } */
 /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm6,\[^\\n\\r\]*96\[^\\n\\r\]*" } } */
 /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm7,\[^\\n\\r\]*112\[^\\n\\r\]*" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm1, %xmm1" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm2, %xmm2" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm3, %xmm3" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm4, %xmm4" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm5, %xmm5" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm6, %xmm6" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm7, %xmm7" } } */
 
 #include <immintrin.h>
 
diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesenc128kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesenc128kl.c
index 9ff483656fd..61a9cc279fc 100644
--- a/gcc/testsuite/gcc.target/i386/keylocker-aesenc128kl.c
+++ b/gcc/testsuite/gcc.target/i386/keylocker-aesenc128kl.c
@@ -2,8 +2,10 @@ 
 /* { dg-options "-mkl -O2" } */
 /* { dg-final { scan-assembler "movdqa\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
 /* { dg-final { scan-assembler "aesenc128kl\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
+/* { dg-final { scan-assembler "j\[ez\]" } } */
 /* { dg-final { scan-assembler "sete" } } */
 /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
 
 #include <immintrin.h>
 
diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesenc256kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesenc256kl.c
index 1c5e0765c07..f8e6bb7321f 100644
--- a/gcc/testsuite/gcc.target/i386/keylocker-aesenc256kl.c
+++ b/gcc/testsuite/gcc.target/i386/keylocker-aesenc256kl.c
@@ -2,8 +2,10 @@ 
 /* { dg-options "-mkl -O2" } */
 /* { dg-final { scan-assembler "movdqa\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
 /* { dg-final { scan-assembler "aesenc256kl\[ \\t\]+\[^\\n\\r\]*, %xmm0" } } */
+/* { dg-final { scan-assembler "j\[ez\]" } } */
 /* { dg-final { scan-assembler "sete" } } */
 /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
 
 #include <immintrin.h>
 
diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesencwide128kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesencwide128kl.c
index 9fb9c49314f..c0fcd28fb07 100644
--- a/gcc/testsuite/gcc.target/i386/keylocker-aesencwide128kl.c
+++ b/gcc/testsuite/gcc.target/i386/keylocker-aesencwide128kl.c
@@ -9,6 +9,7 @@ 
 /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*96\[^\\n\\r\]*, %xmm6" } } */
 /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*112\[^\\n\\r\]*, %xmm7" } } */
 /* { dg-final { scan-assembler "aesencwide128kl\[ \\t\]+\[^\\n\\r\]*" } } */
+/* { dg-final { scan-assembler "j\[ez\]" } } */
 /* { dg-final { scan-assembler "sete" } } */
 /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
 /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm1,\[^\\n\\r\]*16\[^\\n\\r\]*" } } */
@@ -18,6 +19,14 @@ 
 /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm5,\[^\\n\\r\]*80\[^\\n\\r\]*" } } */
 /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm6,\[^\\n\\r\]*96\[^\\n\\r\]*" } } */
 /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm7,\[^\\n\\r\]*112\[^\\n\\r\]*" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm1, %xmm1" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm2, %xmm2" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm3, %xmm3" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm4, %xmm4" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm5, %xmm5" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm6, %xmm6" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm7, %xmm7" } } */
 
 #include <immintrin.h>
 
diff --git a/gcc/testsuite/gcc.target/i386/keylocker-aesencwide256kl.c b/gcc/testsuite/gcc.target/i386/keylocker-aesencwide256kl.c
index 125a787dcd9..31463a8b2da 100644
--- a/gcc/testsuite/gcc.target/i386/keylocker-aesencwide256kl.c
+++ b/gcc/testsuite/gcc.target/i386/keylocker-aesencwide256kl.c
@@ -9,6 +9,7 @@ 
 /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*96\[^\\n\\r\]*, %xmm6" } } */
 /* { dg-final { scan-assembler "movdqu\[ \\t\]+\[^\\n\\r\]*112\[^\\n\\r\]*, %xmm7" } } */
 /* { dg-final { scan-assembler "aesencwide256kl\[ \\t\]+\[^\\n\\r\]*" } } */
+/* { dg-final { scan-assembler "j\[ez\]" } } */
 /* { dg-final { scan-assembler "sete" } } */
 /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm0,\[^\\n\\r\]*" } } */
 /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm1,\[^\\n\\r\]*16\[^\\n\\r\]*" } } */
@@ -18,6 +19,14 @@ 
 /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm5,\[^\\n\\r\]*80\[^\\n\\r\]*" } } */
 /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm6,\[^\\n\\r\]*96\[^\\n\\r\]*" } } */
 /* { dg-final { scan-assembler "(?:movdqu|movups)\[ \\t\]+\[^\\n\\r\]*%xmm7,\[^\\n\\r\]*112\[^\\n\\r\]*" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm0, %xmm0" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm1, %xmm1" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm2, %xmm2" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm3, %xmm3" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm4, %xmm4" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm5, %xmm5" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm6, %xmm6" } } */
+/* { dg-final { scan-assembler "pxor\[ \t\]+%xmm7, %xmm7" } } */
 
 #include <immintrin.h>