Message ID | patch-18034-tamar@arm.com |
---|---|
State | New |
Headers | show |
Series | middle-end: refactor vectorizable_live_operation into helper method for codegen | expand |
On Mon, 27 Nov 2023, Tamar Christina wrote: > Hi All, > > To make code review of the updates to add multiple exit supports to > vectorizable_live_operation easier I've extracted the refactoring part to > its own patch. > > This patch is a straight extract of the function with no functional changes. > > Bootstrapped Regtested on aarch64-none-linux-gnu and no issues. > > Ok for master? OK. Richard. > Thanks, > Tamar > > gcc/ChangeLog: > > * tree-vect-loop.cc (vectorizable_live_operation_1): New. > (vectorizable_live_operation): Extract code to vectorizable_live_operation_1. > > --- inline copy of patch -- > diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc > index 8a50380de49bc12105be47ea1d8ee3cf1f2bdab4..df5e1d28fac2ce35e71decdec0d8e31fb75557f5 100644 > --- a/gcc/tree-vect-loop.cc > +++ b/gcc/tree-vect-loop.cc > @@ -10481,6 +10481,95 @@ vectorizable_induction (loop_vec_info loop_vinfo, > return true; > } > > + > +/* Function vectorizable_live_operation_1. > + helper function for vectorizable_live_operation. */ > +tree > +vectorizable_live_operation_1 (loop_vec_info loop_vinfo, > + stmt_vec_info stmt_info, edge exit_e, > + tree vectype, int ncopies, slp_tree slp_node, > + tree bitsize, tree bitstart, tree vec_lhs, > + tree lhs_type, gimple_stmt_iterator *exit_gsi) > +{ > + basic_block exit_bb = exit_e->dest; > + gcc_assert (single_pred_p (exit_bb) || LOOP_VINFO_EARLY_BREAKS (loop_vinfo)); > + > + tree vec_lhs_phi = copy_ssa_name (vec_lhs); > + gimple *phi = create_phi_node (vec_lhs_phi, exit_bb); > + for (unsigned i = 0; i < gimple_phi_num_args (phi); i++) > + SET_PHI_ARG_DEF (phi, i, vec_lhs); > + > + gimple_seq stmts = NULL; > + tree new_tree; > + if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)) > + { > + /* Emit: > + SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1> > + where VEC_LHS is the vectorized live-out result and MASK is > + the loop mask for the final iteration. */ > + gcc_assert (ncopies == 1 && !slp_node); > + gimple_seq tem = NULL; > + gimple_stmt_iterator gsi = gsi_last (tem); > + tree len = vect_get_loop_len (loop_vinfo, &gsi, > + &LOOP_VINFO_LENS (loop_vinfo), > + 1, vectype, 0, 0); > + /* BIAS - 1. */ > + signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo); > + tree bias_minus_one > + = int_const_binop (MINUS_EXPR, > + build_int_cst (TREE_TYPE (len), biasval), > + build_one_cst (TREE_TYPE (len))); > + /* LAST_INDEX = LEN + (BIAS - 1). */ > + tree last_index = gimple_build (&stmts, PLUS_EXPR, TREE_TYPE (len), > + len, bias_minus_one); > + /* This needs to implement extraction of the first index, but not sure > + how the LEN stuff works. At the moment we shouldn't get here since > + there's no LEN support for early breaks. But guard this so there's > + no incorrect codegen. */ > + gcc_assert (!LOOP_VINFO_EARLY_BREAKS (loop_vinfo)); > + > + /* SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>. */ > + tree scalar_res > + = gimple_build (&stmts, CFN_VEC_EXTRACT, TREE_TYPE (vectype), > + vec_lhs_phi, last_index); > + /* Convert the extracted vector element to the scalar type. */ > + new_tree = gimple_convert (&stmts, lhs_type, scalar_res); > + } > + else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)) > + { > + /* Emit: > + SCALAR_RES = EXTRACT_LAST <VEC_LHS, MASK> > + where VEC_LHS is the vectorized live-out result and MASK is > + the loop mask for the final iteration. */ > + gcc_assert (!slp_node); > + tree scalar_type = TREE_TYPE (STMT_VINFO_VECTYPE (stmt_info)); > + gimple_seq tem = NULL; > + gimple_stmt_iterator gsi = gsi_last (tem); > + tree mask = vect_get_loop_mask (loop_vinfo, &gsi, > + &LOOP_VINFO_MASKS (loop_vinfo), > + 1, vectype, 0); > + > + gimple_seq_add_seq (&stmts, tem); > + tree scalar_res = gimple_build (&stmts, CFN_EXTRACT_LAST, scalar_type, > + mask, vec_lhs_phi); > + /* Convert the extracted vector element to the scalar type. */ > + new_tree = gimple_convert (&stmts, lhs_type, scalar_res); > + } > + else > + { > + tree bftype = TREE_TYPE (vectype); > + if (VECTOR_BOOLEAN_TYPE_P (vectype)) > + bftype = build_nonstandard_integer_type (tree_to_uhwi (bitsize), 1); > + new_tree = build3 (BIT_FIELD_REF, bftype, vec_lhs_phi, bitsize, bitstart); > + new_tree = force_gimple_operand (fold_convert (lhs_type, new_tree), > + &stmts, true, NULL_TREE); > + } > + *exit_gsi = gsi_after_labels (exit_bb); > + if (stmts) > + gsi_insert_seq_before (exit_gsi, stmts, GSI_SAME_STMT); > + return new_tree; > +} > + > /* Function vectorizable_live_operation. > > STMT_INFO computes a value that is used outside the loop. Check if > @@ -10690,79 +10779,13 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info, > gimple *phi = create_phi_node (vec_lhs_phi, exit_bb); > SET_PHI_ARG_DEF (phi, LOOP_VINFO_IV_EXIT (loop_vinfo)->dest_idx, vec_lhs); > > - gimple_seq stmts = NULL; > - tree new_tree; > - if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)) > - { > - /* Emit: > - > - SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1> > - > - where VEC_LHS is the vectorized live-out result and MASK is > - the loop mask for the final iteration. */ > - gcc_assert (ncopies == 1 && !slp_node); > - gimple_seq tem = NULL; > - gimple_stmt_iterator gsi = gsi_last (tem); > - tree len > - = vect_get_loop_len (loop_vinfo, &gsi, > - &LOOP_VINFO_LENS (loop_vinfo), > - 1, vectype, 0, 0); > - > - /* BIAS - 1. */ > - signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo); > - tree bias_minus_one > - = int_const_binop (MINUS_EXPR, > - build_int_cst (TREE_TYPE (len), biasval), > - build_one_cst (TREE_TYPE (len))); > - > - /* LAST_INDEX = LEN + (BIAS - 1). */ > - tree last_index = gimple_build (&stmts, PLUS_EXPR, TREE_TYPE (len), > - len, bias_minus_one); > - > - /* SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>. */ > - tree scalar_res > - = gimple_build (&stmts, CFN_VEC_EXTRACT, TREE_TYPE (vectype), > - vec_lhs_phi, last_index); > - > - /* Convert the extracted vector element to the scalar type. */ > - new_tree = gimple_convert (&stmts, lhs_type, scalar_res); > - } > - else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)) > - { > - /* Emit: > - > - SCALAR_RES = EXTRACT_LAST <VEC_LHS, MASK> > - > - where VEC_LHS is the vectorized live-out result and MASK is > - the loop mask for the final iteration. */ > - gcc_assert (ncopies == 1 && !slp_node); > - tree scalar_type = TREE_TYPE (STMT_VINFO_VECTYPE (stmt_info)); > - gimple_seq tem = NULL; > - gimple_stmt_iterator gsi = gsi_last (tem); > - tree mask = vect_get_loop_mask (loop_vinfo, &gsi, > - &LOOP_VINFO_MASKS (loop_vinfo), > - 1, vectype, 0); > - gimple_seq_add_seq (&stmts, tem); > - tree scalar_res = gimple_build (&stmts, CFN_EXTRACT_LAST, scalar_type, > - mask, vec_lhs_phi); > - > - /* Convert the extracted vector element to the scalar type. */ > - new_tree = gimple_convert (&stmts, lhs_type, scalar_res); > - } > - else > - { > - tree bftype = TREE_TYPE (vectype); > - if (VECTOR_BOOLEAN_TYPE_P (vectype)) > - bftype = build_nonstandard_integer_type (tree_to_uhwi (bitsize), 1); > - new_tree = build3 (BIT_FIELD_REF, bftype, > - vec_lhs_phi, bitsize, bitstart); > - new_tree = force_gimple_operand (fold_convert (lhs_type, new_tree), > - &stmts, true, NULL_TREE); > - } > - > - gimple_stmt_iterator exit_gsi = gsi_after_labels (exit_bb); > - if (stmts) > - gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT); > + gimple_stmt_iterator exit_gsi; > + tree new_tree > + = vectorizable_live_operation_1 (loop_vinfo, stmt_info, > + LOOP_VINFO_IV_EXIT (loop_vinfo), > + vectype, ncopies, slp_node, bitsize, > + bitstart, vec_lhs, lhs_type, > + &exit_gsi); > > /* Remove existing phis that copy from lhs and create copies > from new_tree. */ > > > > >
--- a/gcc/tree-vect-loop.cc +++ b/gcc/tree-vect-loop.cc @@ -10481,6 +10481,95 @@ vectorizable_induction (loop_vec_info loop_vinfo, return true; } + +/* Function vectorizable_live_operation_1. + helper function for vectorizable_live_operation. */ +tree +vectorizable_live_operation_1 (loop_vec_info loop_vinfo, + stmt_vec_info stmt_info, edge exit_e, + tree vectype, int ncopies, slp_tree slp_node, + tree bitsize, tree bitstart, tree vec_lhs, + tree lhs_type, gimple_stmt_iterator *exit_gsi) +{ + basic_block exit_bb = exit_e->dest; + gcc_assert (single_pred_p (exit_bb) || LOOP_VINFO_EARLY_BREAKS (loop_vinfo)); + + tree vec_lhs_phi = copy_ssa_name (vec_lhs); + gimple *phi = create_phi_node (vec_lhs_phi, exit_bb); + for (unsigned i = 0; i < gimple_phi_num_args (phi); i++) + SET_PHI_ARG_DEF (phi, i, vec_lhs); + + gimple_seq stmts = NULL; + tree new_tree; + if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)) + { + /* Emit: + SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1> + where VEC_LHS is the vectorized live-out result and MASK is + the loop mask for the final iteration. */ + gcc_assert (ncopies == 1 && !slp_node); + gimple_seq tem = NULL; + gimple_stmt_iterator gsi = gsi_last (tem); + tree len = vect_get_loop_len (loop_vinfo, &gsi, + &LOOP_VINFO_LENS (loop_vinfo), + 1, vectype, 0, 0); + /* BIAS - 1. */ + signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo); + tree bias_minus_one + = int_const_binop (MINUS_EXPR, + build_int_cst (TREE_TYPE (len), biasval), + build_one_cst (TREE_TYPE (len))); + /* LAST_INDEX = LEN + (BIAS - 1). */ + tree last_index = gimple_build (&stmts, PLUS_EXPR, TREE_TYPE (len), + len, bias_minus_one); + /* This needs to implement extraction of the first index, but not sure + how the LEN stuff works. At the moment we shouldn't get here since + there's no LEN support for early breaks. But guard this so there's + no incorrect codegen. */ + gcc_assert (!LOOP_VINFO_EARLY_BREAKS (loop_vinfo)); + + /* SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>. */ + tree scalar_res + = gimple_build (&stmts, CFN_VEC_EXTRACT, TREE_TYPE (vectype), + vec_lhs_phi, last_index); + /* Convert the extracted vector element to the scalar type. */ + new_tree = gimple_convert (&stmts, lhs_type, scalar_res); + } + else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)) + { + /* Emit: + SCALAR_RES = EXTRACT_LAST <VEC_LHS, MASK> + where VEC_LHS is the vectorized live-out result and MASK is + the loop mask for the final iteration. */ + gcc_assert (!slp_node); + tree scalar_type = TREE_TYPE (STMT_VINFO_VECTYPE (stmt_info)); + gimple_seq tem = NULL; + gimple_stmt_iterator gsi = gsi_last (tem); + tree mask = vect_get_loop_mask (loop_vinfo, &gsi, + &LOOP_VINFO_MASKS (loop_vinfo), + 1, vectype, 0); + + gimple_seq_add_seq (&stmts, tem); + tree scalar_res = gimple_build (&stmts, CFN_EXTRACT_LAST, scalar_type, + mask, vec_lhs_phi); + /* Convert the extracted vector element to the scalar type. */ + new_tree = gimple_convert (&stmts, lhs_type, scalar_res); + } + else + { + tree bftype = TREE_TYPE (vectype); + if (VECTOR_BOOLEAN_TYPE_P (vectype)) + bftype = build_nonstandard_integer_type (tree_to_uhwi (bitsize), 1); + new_tree = build3 (BIT_FIELD_REF, bftype, vec_lhs_phi, bitsize, bitstart); + new_tree = force_gimple_operand (fold_convert (lhs_type, new_tree), + &stmts, true, NULL_TREE); + } + *exit_gsi = gsi_after_labels (exit_bb); + if (stmts) + gsi_insert_seq_before (exit_gsi, stmts, GSI_SAME_STMT); + return new_tree; +} + /* Function vectorizable_live_operation. STMT_INFO computes a value that is used outside the loop. Check if @@ -10690,79 +10779,13 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info, gimple *phi = create_phi_node (vec_lhs_phi, exit_bb); SET_PHI_ARG_DEF (phi, LOOP_VINFO_IV_EXIT (loop_vinfo)->dest_idx, vec_lhs); - gimple_seq stmts = NULL; - tree new_tree; - if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)) - { - /* Emit: - - SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1> - - where VEC_LHS is the vectorized live-out result and MASK is - the loop mask for the final iteration. */ - gcc_assert (ncopies == 1 && !slp_node); - gimple_seq tem = NULL; - gimple_stmt_iterator gsi = gsi_last (tem); - tree len - = vect_get_loop_len (loop_vinfo, &gsi, - &LOOP_VINFO_LENS (loop_vinfo), - 1, vectype, 0, 0); - - /* BIAS - 1. */ - signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo); - tree bias_minus_one - = int_const_binop (MINUS_EXPR, - build_int_cst (TREE_TYPE (len), biasval), - build_one_cst (TREE_TYPE (len))); - - /* LAST_INDEX = LEN + (BIAS - 1). */ - tree last_index = gimple_build (&stmts, PLUS_EXPR, TREE_TYPE (len), - len, bias_minus_one); - - /* SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>. */ - tree scalar_res - = gimple_build (&stmts, CFN_VEC_EXTRACT, TREE_TYPE (vectype), - vec_lhs_phi, last_index); - - /* Convert the extracted vector element to the scalar type. */ - new_tree = gimple_convert (&stmts, lhs_type, scalar_res); - } - else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)) - { - /* Emit: - - SCALAR_RES = EXTRACT_LAST <VEC_LHS, MASK> - - where VEC_LHS is the vectorized live-out result and MASK is - the loop mask for the final iteration. */ - gcc_assert (ncopies == 1 && !slp_node); - tree scalar_type = TREE_TYPE (STMT_VINFO_VECTYPE (stmt_info)); - gimple_seq tem = NULL; - gimple_stmt_iterator gsi = gsi_last (tem); - tree mask = vect_get_loop_mask (loop_vinfo, &gsi, - &LOOP_VINFO_MASKS (loop_vinfo), - 1, vectype, 0); - gimple_seq_add_seq (&stmts, tem); - tree scalar_res = gimple_build (&stmts, CFN_EXTRACT_LAST, scalar_type, - mask, vec_lhs_phi); - - /* Convert the extracted vector element to the scalar type. */ - new_tree = gimple_convert (&stmts, lhs_type, scalar_res); - } - else - { - tree bftype = TREE_TYPE (vectype); - if (VECTOR_BOOLEAN_TYPE_P (vectype)) - bftype = build_nonstandard_integer_type (tree_to_uhwi (bitsize), 1); - new_tree = build3 (BIT_FIELD_REF, bftype, - vec_lhs_phi, bitsize, bitstart); - new_tree = force_gimple_operand (fold_convert (lhs_type, new_tree), - &stmts, true, NULL_TREE); - } - - gimple_stmt_iterator exit_gsi = gsi_after_labels (exit_bb); - if (stmts) - gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT); + gimple_stmt_iterator exit_gsi; + tree new_tree + = vectorizable_live_operation_1 (loop_vinfo, stmt_info, + LOOP_VINFO_IV_EXIT (loop_vinfo), + vectype, ncopies, slp_node, bitsize, + bitstart, vec_lhs, lhs_type, + &exit_gsi); /* Remove existing phis that copy from lhs and create copies from new_tree. */