| Message ID | patch-18097-tamar@arm.com |
|---|---|
| State | New |
| Headers | show |
| Series | middle-end: Handle hybrid SLP induction vectorization with early breaks. | expand |
On Tue, 19 Dec 2023, Tamar Christina wrote: > Hi All, > > While we don't support SLP for early break vectorization, we > can land in the situation where the induction was vectorized > through hybrid SLP. This means when vectorizing the early > break live operation we need to get the results of the SLP > operation. > > > Bootstrapped Regtested on aarch64-none-linux-gnu, > x86_64-pc-linux-gnu and no issues. > > Ok for master? OK. > Thanks, > Tamar > > gcc/ChangeLog: > > * tree-vect-loop.cc (vectorizable_live_operation): Handle SLP. > > gcc/testsuite/ChangeLog: > > * gcc.dg/vect/vect-early-break_82.c: New test. > > --- inline copy of patch -- > diff --git a/gcc/testsuite/gcc.dg/vect/vect-early-break_82.c b/gcc/testsuite/gcc.dg/vect/vect-early-break_82.c > new file mode 100644 > index 0000000000000000000000000000000000000000..f2a6d640f9c0c381cc2af09bd824e272bcfee0b8 > --- /dev/null > +++ b/gcc/testsuite/gcc.dg/vect/vect-early-break_82.c > @@ -0,0 +1,27 @@ > +/* { dg-do compile } */ > +/* { dg-require-effective-target vect_early_break } */ > +/* { dg-require-effective-target vect_int } */ > + > +/* { dg-additional-options "-Ofast" } */ > + > +/* { dg-final { scan-tree-dump "LOOP VECTORIZED" "vect" } } */ > + > +#include <complex.h> > + > +#define N 1024 > +complex double vect_a[N]; > +complex double vect_b[N]; > + > +complex double test4(complex double x, complex double t) > +{ > + complex double ret = 0; > + for (int i = 0; i < N; i++) > + { > + vect_a[i] = t + i; > + if (vect_a[i] == x) > + return i; > + vect_a[i] += x * vect_a[i]; > + > + } > + return ret; > +} > diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc > index 85b81d30c5ab869cb1f7323caabd9fe4648bdc50..0993d184afe068784474ac225768d9f38d76c040 100644 > --- a/gcc/tree-vect-loop.cc > +++ b/gcc/tree-vect-loop.cc > @@ -10856,8 +10856,8 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info, > bitsize = vector_element_bits_tree (vectype); > > /* Get the vectorized lhs of STMT and the lane to use (counted in bits). */ > - tree vec_lhs, bitstart; > - gimple *vec_stmt; > + tree vec_lhs, vec_lhs0, bitstart; > + gimple *vec_stmt, *vec_stmt0; > if (slp_node) > { > gcc_assert (!loop_vinfo > @@ -10868,6 +10868,10 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info, > vec_lhs = SLP_TREE_VEC_DEFS (slp_node)[vec_entry]; > vec_stmt = SSA_NAME_DEF_STMT (vec_lhs); > > + /* In case we need to early break vectorize also get the first stmt. */ > + vec_lhs0 = SLP_TREE_VEC_DEFS (slp_node)[0]; > + vec_stmt0 = SSA_NAME_DEF_STMT (vec_lhs0); > + > /* Get entry to use. */ > bitstart = bitsize_int (vec_index); > bitstart = int_const_binop (MULT_EXPR, bitsize, bitstart); > @@ -10878,6 +10882,10 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info, > vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info).last (); > vec_lhs = gimple_get_lhs (vec_stmt); > > + /* In case we need to early break vectorize also get the first stmt. */ > + vec_stmt0 = STMT_VINFO_VEC_STMTS (stmt_info)[0]; > + vec_lhs0 = gimple_get_lhs (vec_stmt0); > + > /* Get the last lane in the vector. */ > bitstart = int_const_binop (MULT_EXPR, bitsize, bitsize_int (nunits - 1)); > } > @@ -10917,7 +10925,6 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info, > so use ->src. For main exit the merge block is the > destination. */ > basic_block dest = main_exit_edge ? main_e->dest : e->src; > - gimple *tmp_vec_stmt = vec_stmt; > tree tmp_vec_lhs = vec_lhs; > tree tmp_bitstart = bitstart; > > @@ -10928,8 +10935,7 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info, > if (restart_loop > && STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def) > { > - tmp_vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; > - tmp_vec_lhs = gimple_get_lhs (tmp_vec_stmt); > + tmp_vec_lhs = vec_lhs0; > tmp_bitstart = build_zero_cst (TREE_TYPE (bitstart)); > } > > > > > >
--- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/vect-early-break_82.c @@ -0,0 +1,27 @@ +/* { dg-do compile } */ +/* { dg-require-effective-target vect_early_break } */ +/* { dg-require-effective-target vect_int } */ + +/* { dg-additional-options "-Ofast" } */ + +/* { dg-final { scan-tree-dump "LOOP VECTORIZED" "vect" } } */ + +#include <complex.h> + +#define N 1024 +complex double vect_a[N]; +complex double vect_b[N]; + +complex double test4(complex double x, complex double t) +{ + complex double ret = 0; + for (int i = 0; i < N; i++) + { + vect_a[i] = t + i; + if (vect_a[i] == x) + return i; + vect_a[i] += x * vect_a[i]; + + } + return ret; +} diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc index 85b81d30c5ab869cb1f7323caabd9fe4648bdc50..0993d184afe068784474ac225768d9f38d76c040 100644 --- a/gcc/tree-vect-loop.cc +++ b/gcc/tree-vect-loop.cc @@ -10856,8 +10856,8 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info, bitsize = vector_element_bits_tree (vectype); /* Get the vectorized lhs of STMT and the lane to use (counted in bits). */ - tree vec_lhs, bitstart; - gimple *vec_stmt; + tree vec_lhs, vec_lhs0, bitstart; + gimple *vec_stmt, *vec_stmt0; if (slp_node) { gcc_assert (!loop_vinfo @@ -10868,6 +10868,10 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info, vec_lhs = SLP_TREE_VEC_DEFS (slp_node)[vec_entry]; vec_stmt = SSA_NAME_DEF_STMT (vec_lhs); + /* In case we need to early break vectorize also get the first stmt. */ + vec_lhs0 = SLP_TREE_VEC_DEFS (slp_node)[0]; + vec_stmt0 = SSA_NAME_DEF_STMT (vec_lhs0); + /* Get entry to use. */ bitstart = bitsize_int (vec_index); bitstart = int_const_binop (MULT_EXPR, bitsize, bitstart); @@ -10878,6 +10882,10 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info, vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info).last (); vec_lhs = gimple_get_lhs (vec_stmt); + /* In case we need to early break vectorize also get the first stmt. */ + vec_stmt0 = STMT_VINFO_VEC_STMTS (stmt_info)[0]; + vec_lhs0 = gimple_get_lhs (vec_stmt0); + /* Get the last lane in the vector. */ bitstart = int_const_binop (MULT_EXPR, bitsize, bitsize_int (nunits - 1)); } @@ -10917,7 +10925,6 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info, so use ->src. For main exit the merge block is the destination. */ basic_block dest = main_exit_edge ? main_e->dest : e->src; - gimple *tmp_vec_stmt = vec_stmt; tree tmp_vec_lhs = vec_lhs; tree tmp_bitstart = bitstart; @@ -10928,8 +10935,7 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info, if (restart_loop && STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def) { - tmp_vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; - tmp_vec_lhs = gimple_get_lhs (tmp_vec_stmt); + tmp_vec_lhs = vec_lhs0; tmp_bitstart = build_zero_cst (TREE_TYPE (bitstart)); }
Hi All, While we don't support SLP for early break vectorization, we can land in the situation where the induction was vectorized through hybrid SLP. This means when vectorizing the early break live operation we need to get the results of the SLP operation. Bootstrapped Regtested on aarch64-none-linux-gnu, x86_64-pc-linux-gnu and no issues. Ok for master? Thanks, Tamar gcc/ChangeLog: * tree-vect-loop.cc (vectorizable_live_operation): Handle SLP. gcc/testsuite/ChangeLog: * gcc.dg/vect/vect-early-break_82.c: New test. --- inline copy of patch -- diff --git a/gcc/testsuite/gcc.dg/vect/vect-early-break_82.c b/gcc/testsuite/gcc.dg/vect/vect-early-break_82.c new file mode 100644 index 0000000000000000000000000000000000000000..f2a6d640f9c0c381cc2af09bd824e272bcfee0b8 -- diff --git a/gcc/testsuite/gcc.dg/vect/vect-early-break_82.c b/gcc/testsuite/gcc.dg/vect/vect-early-break_82.c new file mode 100644 index 0000000000000000000000000000000000000000..f2a6d640f9c0c381cc2af09bd824e272bcfee0b8 --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/vect-early-break_82.c @@ -0,0 +1,27 @@ +/* { dg-do compile } */ +/* { dg-require-effective-target vect_early_break } */ +/* { dg-require-effective-target vect_int } */ + +/* { dg-additional-options "-Ofast" } */ + +/* { dg-final { scan-tree-dump "LOOP VECTORIZED" "vect" } } */ + +#include <complex.h> + +#define N 1024 +complex double vect_a[N]; +complex double vect_b[N]; + +complex double test4(complex double x, complex double t) +{ + complex double ret = 0; + for (int i = 0; i < N; i++) + { + vect_a[i] = t + i; + if (vect_a[i] == x) + return i; + vect_a[i] += x * vect_a[i]; + + } + return ret; +} diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc index 85b81d30c5ab869cb1f7323caabd9fe4648bdc50..0993d184afe068784474ac225768d9f38d76c040 100644 --- a/gcc/tree-vect-loop.cc +++ b/gcc/tree-vect-loop.cc @@ -10856,8 +10856,8 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info, bitsize = vector_element_bits_tree (vectype); /* Get the vectorized lhs of STMT and the lane to use (counted in bits). */ - tree vec_lhs, bitstart; - gimple *vec_stmt; + tree vec_lhs, vec_lhs0, bitstart; + gimple *vec_stmt, *vec_stmt0; if (slp_node) { gcc_assert (!loop_vinfo @@ -10868,6 +10868,10 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info, vec_lhs = SLP_TREE_VEC_DEFS (slp_node)[vec_entry]; vec_stmt = SSA_NAME_DEF_STMT (vec_lhs); + /* In case we need to early break vectorize also get the first stmt. */ + vec_lhs0 = SLP_TREE_VEC_DEFS (slp_node)[0]; + vec_stmt0 = SSA_NAME_DEF_STMT (vec_lhs0); + /* Get entry to use. */ bitstart = bitsize_int (vec_index); bitstart = int_const_binop (MULT_EXPR, bitsize, bitstart); @@ -10878,6 +10882,10 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info, vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info).last (); vec_lhs = gimple_get_lhs (vec_stmt); + /* In case we need to early break vectorize also get the first stmt. */ + vec_stmt0 = STMT_VINFO_VEC_STMTS (stmt_info)[0]; + vec_lhs0 = gimple_get_lhs (vec_stmt0); + /* Get the last lane in the vector. */ bitstart = int_const_binop (MULT_EXPR, bitsize, bitsize_int (nunits - 1)); } @@ -10917,7 +10925,6 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info, so use ->src. For main exit the merge block is the destination. */ basic_block dest = main_exit_edge ? main_e->dest : e->src; - gimple *tmp_vec_stmt = vec_stmt; tree tmp_vec_lhs = vec_lhs; tree tmp_bitstart = bitstart; @@ -10928,8 +10935,7 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info, if (restart_loop && STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def) { - tmp_vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; - tmp_vec_lhs = gimple_get_lhs (tmp_vec_stmt); + tmp_vec_lhs = vec_lhs0; tmp_bitstart = build_zero_cst (TREE_TYPE (bitstart)); }