| Message ID | 20151008155030.GJ63757@msticlxl57.ims.intel.com |
|---|---|
| State | New |
| Headers | show |
On Thu, Oct 8, 2015 at 5:50 PM, Ilya Enkovich <enkovich.gnu@gmail.com> wrote: > Hi, > > This patch allows COND_EXPR with no embedded comparison to be vectorized. It's applied on top of vectorized comparison support series. New optab vcond_mask_optab is introduced for such statements. Bool patterns now avoid comparison in COND_EXPR in case vector comparison is supported by target. Ok. Thanks, Richard. > Thanks, > Ilya > -- > gcc/ > > 2015-10-08 Ilya Enkovich <enkovich.gnu@gmail.com> > > * optabs-query.h (get_vcond_mask_icode): New. > * optabs-tree.c (expand_vec_cond_expr_p): Use > get_vcond_mask_icode for VEC_COND_EXPR with mask. > * optabs.c (expand_vec_cond_mask_expr): New. > (expand_vec_cond_expr): Use get_vcond_mask_icode > when possible. > * optabs.def (vcond_mask_optab): New. > * tree-vect-patterns.c (vect_recog_bool_pattern): Don't > generate redundant comparison for COND_EXPR. > * tree-vect-stmts.c (vect_is_simple_cond): Allow SSA_NAME > as a condition. > (vectorizable_condition): Likewise. > > > diff --git a/gcc/optabs-query.h b/gcc/optabs-query.h > index 162d2e9..48bcf7c 100644 > --- a/gcc/optabs-query.h > +++ b/gcc/optabs-query.h > @@ -98,6 +98,15 @@ get_vcond_icode (machine_mode vmode, machine_mode cmode, bool uns) > return icode; > } > > +/* Return insn code for a conditional operator with a mask mode > + MMODE resulting in a value of mode VMODE. */ > + > +static inline enum insn_code > +get_vcond_mask_icode (machine_mode vmode, machine_mode mmode) > +{ > + return convert_optab_handler (vcond_mask_optab, vmode, mmode); > +} > + > /* Enumerates the possible extraction_insn operations. */ > enum extraction_pattern { EP_insv, EP_extv, EP_extzv }; > > diff --git a/gcc/optabs-tree.c b/gcc/optabs-tree.c > index aa863cf..d887619 100644 > --- a/gcc/optabs-tree.c > +++ b/gcc/optabs-tree.c > @@ -342,6 +342,9 @@ expand_vec_cond_expr_p (tree value_type, tree cmp_op_type) > { > machine_mode value_mode = TYPE_MODE (value_type); > machine_mode cmp_op_mode = TYPE_MODE (cmp_op_type); > + if (VECTOR_BOOLEAN_TYPE_P (cmp_op_type)) > + return get_vcond_mask_icode (TYPE_MODE (value_type), > + TYPE_MODE (cmp_op_type)) != CODE_FOR_nothing; > if (GET_MODE_SIZE (value_mode) != GET_MODE_SIZE (cmp_op_mode) > || GET_MODE_NUNITS (value_mode) != GET_MODE_NUNITS (cmp_op_mode) > || get_vcond_icode (TYPE_MODE (value_type), TYPE_MODE (cmp_op_type), > diff --git a/gcc/optabs.c b/gcc/optabs.c > index ca1a6e7..d26b8f8 100644 > --- a/gcc/optabs.c > +++ b/gcc/optabs.c > @@ -5346,6 +5346,38 @@ expand_vec_perm (machine_mode mode, rtx v0, rtx v1, rtx sel, rtx target) > return tmp; > } > > +/* Generate insns for a VEC_COND_EXPR with mask, given its TYPE and its > + three operands. */ > + > +rtx > +expand_vec_cond_mask_expr (tree vec_cond_type, tree op0, tree op1, tree op2, > + rtx target) > +{ > + struct expand_operand ops[4]; > + machine_mode mode = TYPE_MODE (vec_cond_type); > + machine_mode mask_mode = TYPE_MODE (TREE_TYPE (op0)); > + enum insn_code icode = get_vcond_mask_icode (mode, mask_mode); > + rtx mask, rtx_op1, rtx_op2; > + > + if (icode == CODE_FOR_nothing) > + return 0; > + > + mask = expand_normal (op0); > + rtx_op1 = expand_normal (op1); > + rtx_op2 = expand_normal (op2); > + > + mask = force_reg (GET_MODE (mask), mask); > + rtx_op1 = force_reg (GET_MODE (rtx_op1), rtx_op1); > + > + create_output_operand (&ops[0], target, mode); > + create_input_operand (&ops[1], rtx_op1, mode); > + create_input_operand (&ops[2], rtx_op2, mode); > + create_input_operand (&ops[3], mask, mask_mode); > + expand_insn (icode, 4, ops); > + > + return ops[0].value; > +} > + > /* Generate insns for a VEC_COND_EXPR, given its TYPE and its > three operands. */ > > @@ -5371,12 +5403,21 @@ expand_vec_cond_expr (tree vec_cond_type, tree op0, tree op1, tree op2, > } > else > { > - /* Fake op0 < 0. */ > gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0))); > - op0a = op0; > - op0b = build_zero_cst (TREE_TYPE (op0)); > - tcode = LT_EXPR; > - unsignedp = false; > + if (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0))) > + != CODE_FOR_nothing) > + return expand_vec_cond_mask_expr (vec_cond_type, op0, op1, > + op2, target); > + /* Fake op0 < 0. */ > + else > + { > + gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0))) > + == MODE_VECTOR_INT); > + op0a = op0; > + op0b = build_zero_cst (TREE_TYPE (op0)); > + tcode = LT_EXPR; > + unsignedp = false; > + } > } > cmp_op_mode = TYPE_MODE (TREE_TYPE (op0a)); > > diff --git a/gcc/optabs.def b/gcc/optabs.def > index 9804378..70530a6 100644 > --- a/gcc/optabs.def > +++ b/gcc/optabs.def > @@ -61,6 +61,7 @@ OPTAB_CD(vec_load_lanes_optab, "vec_load_lanes$a$b") > OPTAB_CD(vec_store_lanes_optab, "vec_store_lanes$a$b") > OPTAB_CD(vcond_optab, "vcond$a$b") > OPTAB_CD(vcondu_optab, "vcondu$a$b") > +OPTAB_CD(vcond_mask_optab, "vcond_mask_$a$b") > OPTAB_CD(vec_cmp_optab, "vec_cmp$a$b") > OPTAB_CD(vec_cmpu_optab, "vec_cmpu$a$b") > OPTAB_CD(maskload_optab, "maskload$a$b") > diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c > index e3be3d1..6269b69 100644 > --- a/gcc/tree-vect-patterns.c > +++ b/gcc/tree-vect-patterns.c > @@ -3356,7 +3356,7 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in, > else > { > tree type = search_type_for_mask (var, loop_vinfo, bb_vinfo); > - tree cst0, cst1, cmp, tmp; > + tree cst0, cst1, tmp; > > if (!type) > return NULL; > @@ -3372,9 +3372,7 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in, > cst0 = build_int_cst (type, 0); > cst1 = build_int_cst (type, 1); > tmp = vect_recog_temp_ssa_var (type, NULL); > - cmp = build2 (NE_EXPR, boolean_type_node, > - var, build_int_cst (TREE_TYPE (var), 0)); > - pattern_stmt = gimple_build_assign (tmp, COND_EXPR, cmp, cst1, cst0); > + pattern_stmt = gimple_build_assign (tmp, COND_EXPR, var, cst1, cst0); > > if (!useless_type_conversion_p (type, TREE_TYPE (lhs))) > { > @@ -3418,19 +3416,16 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in, > if (get_vectype_for_scalar_type (type) == NULL_TREE) > return NULL; > > - if (check_bool_pattern (var, loop_vinfo, bb_vinfo)) > - { > - rhs = adjust_bool_pattern (var, type, NULL_TREE, stmts); > - rhs = build2 (NE_EXPR, boolean_type_node, > - rhs, build_int_cst (type, 0)); > - } > - else > - rhs = build2 (NE_EXPR, boolean_type_node, > - var, build_int_cst (TREE_TYPE (var), 0)), > + if (!check_bool_pattern (var, loop_vinfo, bb_vinfo)) > + return NULL; > + > + rhs = adjust_bool_pattern (var, type, NULL_TREE, stmts); > > lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL); > pattern_stmt > - = gimple_build_assign (lhs, COND_EXPR, rhs, > + = gimple_build_assign (lhs, COND_EXPR, > + build2 (NE_EXPR, boolean_type_node, > + rhs, build_int_cst (type, 0)), > gimple_assign_rhs2 (last_stmt), > gimple_assign_rhs3 (last_stmt)); > *type_out = vectype; > @@ -3457,7 +3452,7 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in, > else > { > tree type = search_type_for_mask (var, loop_vinfo, bb_vinfo); > - tree cst0, cst1, cmp, new_vectype; > + tree cst0, cst1, new_vectype; > > if (!type) > return NULL; > @@ -3470,10 +3465,7 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in, > new_vectype = get_vectype_for_scalar_type (type); > > rhs = vect_recog_temp_ssa_var (type, NULL); > - cmp = build2 (NE_EXPR, boolean_type_node, > - var, build_int_cst (TREE_TYPE (var), 0)); > - pattern_stmt = gimple_build_assign (rhs, COND_EXPR, > - cmp, cst1, cst0); > + pattern_stmt = gimple_build_assign (rhs, COND_EXPR, var, cst1, cst0); > > pattern_stmt_info = new_stmt_vec_info (pattern_stmt, loop_vinfo, > bb_vinfo); > diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c > index d6819ec..b139601 100644 > --- a/gcc/tree-vect-stmts.c > +++ b/gcc/tree-vect-stmts.c > @@ -7296,6 +7296,19 @@ vect_is_simple_cond (tree cond, gimple *stmt, loop_vec_info loop_vinfo, > enum vect_def_type dt; > tree vectype1 = NULL_TREE, vectype2 = NULL_TREE; > > + /* Mask case. */ > + if (TREE_CODE (cond) == SSA_NAME > + && TREE_CODE (TREE_TYPE (cond)) == BOOLEAN_TYPE) > + { > + gimple *lhs_def_stmt = SSA_NAME_DEF_STMT (cond); > + if (!vect_is_simple_use_1 (cond, stmt, loop_vinfo, bb_vinfo, > + &lhs_def_stmt, &def, &dt, comp_vectype) > + || !*comp_vectype > + || !VECTOR_BOOLEAN_TYPE_P (*comp_vectype)) > + return false; > + return true; > + } > + > if (!COMPARISON_CLASS_P (cond)) > return false; > > @@ -7368,6 +7381,7 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, > vec<tree> vec_oprnds2 = vNULL; > vec<tree> vec_oprnds3 = vNULL; > tree vec_cmp_type; > + bool masked = false; > > if (reduc_index && STMT_SLP_TYPE (stmt_info)) > return false; > @@ -7443,7 +7457,13 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, > && TREE_CODE (else_clause) != FIXED_CST) > return false; > > - vec_cmp_type = build_same_sized_truth_vector_type (comp_vectype); > + if (VECTOR_BOOLEAN_TYPE_P (comp_vectype)) > + { > + vec_cmp_type = comp_vectype; > + masked = true; > + } > + else > + vec_cmp_type = build_same_sized_truth_vector_type (comp_vectype); > if (vec_cmp_type == NULL_TREE) > return false; > > @@ -7478,14 +7498,20 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, > auto_vec<tree, 4> ops; > auto_vec<vec<tree>, 4> vec_defs; > > - ops.safe_push (TREE_OPERAND (cond_expr, 0)); > - ops.safe_push (TREE_OPERAND (cond_expr, 1)); > + if (masked) > + ops.safe_push (cond_expr); > + else > + { > + ops.safe_push (TREE_OPERAND (cond_expr, 0)); > + ops.safe_push (TREE_OPERAND (cond_expr, 1)); > + } > ops.safe_push (then_clause); > ops.safe_push (else_clause); > vect_get_slp_defs (ops, slp_node, &vec_defs, -1); > vec_oprnds3 = vec_defs.pop (); > vec_oprnds2 = vec_defs.pop (); > - vec_oprnds1 = vec_defs.pop (); > + if (!masked) > + vec_oprnds1 = vec_defs.pop (); > vec_oprnds0 = vec_defs.pop (); > > ops.release (); > @@ -7494,17 +7520,28 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, > else > { > gimple *gtemp; > - vec_cond_lhs = > - vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0), > - stmt, NULL, comp_vectype); > - vect_is_simple_use (TREE_OPERAND (cond_expr, 0), stmt, > - loop_vinfo, NULL, >emp, &def, &dts[0]); > - > - vec_cond_rhs = > - vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1), > - stmt, NULL, comp_vectype); > - vect_is_simple_use (TREE_OPERAND (cond_expr, 1), stmt, > - loop_vinfo, NULL, >emp, &def, &dts[1]); > + if (masked) > + { > + vec_cond_lhs > + = vect_get_vec_def_for_operand (cond_expr, stmt, > + NULL, comp_vectype); > + vect_is_simple_use (cond_expr, stmt, loop_vinfo, NULL, > + >emp, &def, &dts[0]); > + } > + else > + { > + vec_cond_lhs = > + vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0), > + stmt, NULL, comp_vectype); > + vect_is_simple_use (TREE_OPERAND (cond_expr, 0), stmt, > + loop_vinfo, NULL, >emp, &def, &dts[0]); > + > + vec_cond_rhs = > + vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1), > + stmt, NULL, comp_vectype); > + vect_is_simple_use (TREE_OPERAND (cond_expr, 1), stmt, > + loop_vinfo, NULL, >emp, &def, &dts[1]); > + } > if (reduc_index == 1) > vec_then_clause = reduc_def; > else > @@ -7527,10 +7564,14 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, > } > else > { > - vec_cond_lhs = vect_get_vec_def_for_stmt_copy (dts[0], > - vec_oprnds0.pop ()); > - vec_cond_rhs = vect_get_vec_def_for_stmt_copy (dts[1], > - vec_oprnds1.pop ()); > + vec_cond_lhs > + = vect_get_vec_def_for_stmt_copy (dts[0], > + vec_oprnds0.pop ()); > + if (!masked) > + vec_cond_rhs > + = vect_get_vec_def_for_stmt_copy (dts[1], > + vec_oprnds1.pop ()); > + > vec_then_clause = vect_get_vec_def_for_stmt_copy (dts[2], > vec_oprnds2.pop ()); > vec_else_clause = vect_get_vec_def_for_stmt_copy (dts[3], > @@ -7540,7 +7581,8 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, > if (!slp_node) > { > vec_oprnds0.quick_push (vec_cond_lhs); > - vec_oprnds1.quick_push (vec_cond_rhs); > + if (!masked) > + vec_oprnds1.quick_push (vec_cond_rhs); > vec_oprnds2.quick_push (vec_then_clause); > vec_oprnds3.quick_push (vec_else_clause); > } > @@ -7548,12 +7590,17 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, > /* Arguments are ready. Create the new vector stmt. */ > FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_cond_lhs) > { > - vec_cond_rhs = vec_oprnds1[i]; > vec_then_clause = vec_oprnds2[i]; > vec_else_clause = vec_oprnds3[i]; > > - vec_compare = build2 (TREE_CODE (cond_expr), vec_cmp_type, > - vec_cond_lhs, vec_cond_rhs); > + if (masked) > + vec_compare = vec_cond_lhs; > + else > + { > + vec_cond_rhs = vec_oprnds1[i]; > + vec_compare = build2 (TREE_CODE (cond_expr), vec_cmp_type, > + vec_cond_lhs, vec_cond_rhs); > + } > vec_cond_expr = build3 (VEC_COND_EXPR, vectype, > vec_compare, vec_then_clause, vec_else_clause); >
On Thu, Oct 8, 2015 at 4:50 PM, Ilya Enkovich <enkovich.gnu@gmail.com> wrote: > Hi, > > This patch allows COND_EXPR with no embedded comparison to be vectorized. > It's applied on top of vectorized comparison support series. New optab vcond_mask_optab > is introduced for such statements. Bool patterns now avoid comparison in COND_EXPR in case vector comparison is supported by target. New standard pattern names are documented in the internals manual. This patch does not do so neither do I see any patches to do so. regards Ramana > > Thanks, > Ilya > -- > gcc/ > > 2015-10-08 Ilya Enkovich <enkovich.gnu@gmail.com> > > * optabs-query.h (get_vcond_mask_icode): New. > * optabs-tree.c (expand_vec_cond_expr_p): Use > get_vcond_mask_icode for VEC_COND_EXPR with mask. > * optabs.c (expand_vec_cond_mask_expr): New. > (expand_vec_cond_expr): Use get_vcond_mask_icode > when possible. > * optabs.def (vcond_mask_optab): New. > * tree-vect-patterns.c (vect_recog_bool_pattern): Don't > generate redundant comparison for COND_EXPR. > * tree-vect-stmts.c (vect_is_simple_cond): Allow SSA_NAME > as a condition. > (vectorizable_condition): Likewise. > > > diff --git a/gcc/optabs-query.h b/gcc/optabs-query.h > index 162d2e9..48bcf7c 100644 > --- a/gcc/optabs-query.h > +++ b/gcc/optabs-query.h > @@ -98,6 +98,15 @@ get_vcond_icode (machine_mode vmode, machine_mode cmode, bool uns) > return icode; > } > > +/* Return insn code for a conditional operator with a mask mode > + MMODE resulting in a value of mode VMODE. */ > + > +static inline enum insn_code > +get_vcond_mask_icode (machine_mode vmode, machine_mode mmode) > +{ > + return convert_optab_handler (vcond_mask_optab, vmode, mmode); > +} > + > /* Enumerates the possible extraction_insn operations. */ > enum extraction_pattern { EP_insv, EP_extv, EP_extzv }; > > diff --git a/gcc/optabs-tree.c b/gcc/optabs-tree.c > index aa863cf..d887619 100644 > --- a/gcc/optabs-tree.c > +++ b/gcc/optabs-tree.c > @@ -342,6 +342,9 @@ expand_vec_cond_expr_p (tree value_type, tree cmp_op_type) > { > machine_mode value_mode = TYPE_MODE (value_type); > machine_mode cmp_op_mode = TYPE_MODE (cmp_op_type); > + if (VECTOR_BOOLEAN_TYPE_P (cmp_op_type)) > + return get_vcond_mask_icode (TYPE_MODE (value_type), > + TYPE_MODE (cmp_op_type)) != CODE_FOR_nothing; > if (GET_MODE_SIZE (value_mode) != GET_MODE_SIZE (cmp_op_mode) > || GET_MODE_NUNITS (value_mode) != GET_MODE_NUNITS (cmp_op_mode) > || get_vcond_icode (TYPE_MODE (value_type), TYPE_MODE (cmp_op_type), > diff --git a/gcc/optabs.c b/gcc/optabs.c > index ca1a6e7..d26b8f8 100644 > --- a/gcc/optabs.c > +++ b/gcc/optabs.c > @@ -5346,6 +5346,38 @@ expand_vec_perm (machine_mode mode, rtx v0, rtx v1, rtx sel, rtx target) > return tmp; > } > > +/* Generate insns for a VEC_COND_EXPR with mask, given its TYPE and its > + three operands. */ > + > +rtx > +expand_vec_cond_mask_expr (tree vec_cond_type, tree op0, tree op1, tree op2, > + rtx target) > +{ > + struct expand_operand ops[4]; > + machine_mode mode = TYPE_MODE (vec_cond_type); > + machine_mode mask_mode = TYPE_MODE (TREE_TYPE (op0)); > + enum insn_code icode = get_vcond_mask_icode (mode, mask_mode); > + rtx mask, rtx_op1, rtx_op2; > + > + if (icode == CODE_FOR_nothing) > + return 0; > + > + mask = expand_normal (op0); > + rtx_op1 = expand_normal (op1); > + rtx_op2 = expand_normal (op2); > + > + mask = force_reg (GET_MODE (mask), mask); > + rtx_op1 = force_reg (GET_MODE (rtx_op1), rtx_op1); > + > + create_output_operand (&ops[0], target, mode); > + create_input_operand (&ops[1], rtx_op1, mode); > + create_input_operand (&ops[2], rtx_op2, mode); > + create_input_operand (&ops[3], mask, mask_mode); > + expand_insn (icode, 4, ops); > + > + return ops[0].value; > +} > + > /* Generate insns for a VEC_COND_EXPR, given its TYPE and its > three operands. */ > > @@ -5371,12 +5403,21 @@ expand_vec_cond_expr (tree vec_cond_type, tree op0, tree op1, tree op2, > } > else > { > - /* Fake op0 < 0. */ > gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0))); > - op0a = op0; > - op0b = build_zero_cst (TREE_TYPE (op0)); > - tcode = LT_EXPR; > - unsignedp = false; > + if (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0))) > + != CODE_FOR_nothing) > + return expand_vec_cond_mask_expr (vec_cond_type, op0, op1, > + op2, target); > + /* Fake op0 < 0. */ > + else > + { > + gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0))) > + == MODE_VECTOR_INT); > + op0a = op0; > + op0b = build_zero_cst (TREE_TYPE (op0)); > + tcode = LT_EXPR; > + unsignedp = false; > + } > } > cmp_op_mode = TYPE_MODE (TREE_TYPE (op0a)); > > diff --git a/gcc/optabs.def b/gcc/optabs.def > index 9804378..70530a6 100644 > --- a/gcc/optabs.def > +++ b/gcc/optabs.def > @@ -61,6 +61,7 @@ OPTAB_CD(vec_load_lanes_optab, "vec_load_lanes$a$b") > OPTAB_CD(vec_store_lanes_optab, "vec_store_lanes$a$b") > OPTAB_CD(vcond_optab, "vcond$a$b") > OPTAB_CD(vcondu_optab, "vcondu$a$b") > +OPTAB_CD(vcond_mask_optab, "vcond_mask_$a$b") > OPTAB_CD(vec_cmp_optab, "vec_cmp$a$b") > OPTAB_CD(vec_cmpu_optab, "vec_cmpu$a$b") > OPTAB_CD(maskload_optab, "maskload$a$b") > diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c > index e3be3d1..6269b69 100644 > --- a/gcc/tree-vect-patterns.c > +++ b/gcc/tree-vect-patterns.c > @@ -3356,7 +3356,7 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in, > else > { > tree type = search_type_for_mask (var, loop_vinfo, bb_vinfo); > - tree cst0, cst1, cmp, tmp; > + tree cst0, cst1, tmp; > > if (!type) > return NULL; > @@ -3372,9 +3372,7 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in, > cst0 = build_int_cst (type, 0); > cst1 = build_int_cst (type, 1); > tmp = vect_recog_temp_ssa_var (type, NULL); > - cmp = build2 (NE_EXPR, boolean_type_node, > - var, build_int_cst (TREE_TYPE (var), 0)); > - pattern_stmt = gimple_build_assign (tmp, COND_EXPR, cmp, cst1, cst0); > + pattern_stmt = gimple_build_assign (tmp, COND_EXPR, var, cst1, cst0); > > if (!useless_type_conversion_p (type, TREE_TYPE (lhs))) > { > @@ -3418,19 +3416,16 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in, > if (get_vectype_for_scalar_type (type) == NULL_TREE) > return NULL; > > - if (check_bool_pattern (var, loop_vinfo, bb_vinfo)) > - { > - rhs = adjust_bool_pattern (var, type, NULL_TREE, stmts); > - rhs = build2 (NE_EXPR, boolean_type_node, > - rhs, build_int_cst (type, 0)); > - } > - else > - rhs = build2 (NE_EXPR, boolean_type_node, > - var, build_int_cst (TREE_TYPE (var), 0)), > + if (!check_bool_pattern (var, loop_vinfo, bb_vinfo)) > + return NULL; > + > + rhs = adjust_bool_pattern (var, type, NULL_TREE, stmts); > > lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL); > pattern_stmt > - = gimple_build_assign (lhs, COND_EXPR, rhs, > + = gimple_build_assign (lhs, COND_EXPR, > + build2 (NE_EXPR, boolean_type_node, > + rhs, build_int_cst (type, 0)), > gimple_assign_rhs2 (last_stmt), > gimple_assign_rhs3 (last_stmt)); > *type_out = vectype; > @@ -3457,7 +3452,7 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in, > else > { > tree type = search_type_for_mask (var, loop_vinfo, bb_vinfo); > - tree cst0, cst1, cmp, new_vectype; > + tree cst0, cst1, new_vectype; > > if (!type) > return NULL; > @@ -3470,10 +3465,7 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in, > new_vectype = get_vectype_for_scalar_type (type); > > rhs = vect_recog_temp_ssa_var (type, NULL); > - cmp = build2 (NE_EXPR, boolean_type_node, > - var, build_int_cst (TREE_TYPE (var), 0)); > - pattern_stmt = gimple_build_assign (rhs, COND_EXPR, > - cmp, cst1, cst0); > + pattern_stmt = gimple_build_assign (rhs, COND_EXPR, var, cst1, cst0); > > pattern_stmt_info = new_stmt_vec_info (pattern_stmt, loop_vinfo, > bb_vinfo); > diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c > index d6819ec..b139601 100644 > --- a/gcc/tree-vect-stmts.c > +++ b/gcc/tree-vect-stmts.c > @@ -7296,6 +7296,19 @@ vect_is_simple_cond (tree cond, gimple *stmt, loop_vec_info loop_vinfo, > enum vect_def_type dt; > tree vectype1 = NULL_TREE, vectype2 = NULL_TREE; > > + /* Mask case. */ > + if (TREE_CODE (cond) == SSA_NAME > + && TREE_CODE (TREE_TYPE (cond)) == BOOLEAN_TYPE) > + { > + gimple *lhs_def_stmt = SSA_NAME_DEF_STMT (cond); > + if (!vect_is_simple_use_1 (cond, stmt, loop_vinfo, bb_vinfo, > + &lhs_def_stmt, &def, &dt, comp_vectype) > + || !*comp_vectype > + || !VECTOR_BOOLEAN_TYPE_P (*comp_vectype)) > + return false; > + return true; > + } > + > if (!COMPARISON_CLASS_P (cond)) > return false; > > @@ -7368,6 +7381,7 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, > vec<tree> vec_oprnds2 = vNULL; > vec<tree> vec_oprnds3 = vNULL; > tree vec_cmp_type; > + bool masked = false; > > if (reduc_index && STMT_SLP_TYPE (stmt_info)) > return false; > @@ -7443,7 +7457,13 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, > && TREE_CODE (else_clause) != FIXED_CST) > return false; > > - vec_cmp_type = build_same_sized_truth_vector_type (comp_vectype); > + if (VECTOR_BOOLEAN_TYPE_P (comp_vectype)) > + { > + vec_cmp_type = comp_vectype; > + masked = true; > + } > + else > + vec_cmp_type = build_same_sized_truth_vector_type (comp_vectype); > if (vec_cmp_type == NULL_TREE) > return false; > > @@ -7478,14 +7498,20 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, > auto_vec<tree, 4> ops; > auto_vec<vec<tree>, 4> vec_defs; > > - ops.safe_push (TREE_OPERAND (cond_expr, 0)); > - ops.safe_push (TREE_OPERAND (cond_expr, 1)); > + if (masked) > + ops.safe_push (cond_expr); > + else > + { > + ops.safe_push (TREE_OPERAND (cond_expr, 0)); > + ops.safe_push (TREE_OPERAND (cond_expr, 1)); > + } > ops.safe_push (then_clause); > ops.safe_push (else_clause); > vect_get_slp_defs (ops, slp_node, &vec_defs, -1); > vec_oprnds3 = vec_defs.pop (); > vec_oprnds2 = vec_defs.pop (); > - vec_oprnds1 = vec_defs.pop (); > + if (!masked) > + vec_oprnds1 = vec_defs.pop (); > vec_oprnds0 = vec_defs.pop (); > > ops.release (); > @@ -7494,17 +7520,28 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, > else > { > gimple *gtemp; > - vec_cond_lhs = > - vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0), > - stmt, NULL, comp_vectype); > - vect_is_simple_use (TREE_OPERAND (cond_expr, 0), stmt, > - loop_vinfo, NULL, >emp, &def, &dts[0]); > - > - vec_cond_rhs = > - vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1), > - stmt, NULL, comp_vectype); > - vect_is_simple_use (TREE_OPERAND (cond_expr, 1), stmt, > - loop_vinfo, NULL, >emp, &def, &dts[1]); > + if (masked) > + { > + vec_cond_lhs > + = vect_get_vec_def_for_operand (cond_expr, stmt, > + NULL, comp_vectype); > + vect_is_simple_use (cond_expr, stmt, loop_vinfo, NULL, > + >emp, &def, &dts[0]); > + } > + else > + { > + vec_cond_lhs = > + vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0), > + stmt, NULL, comp_vectype); > + vect_is_simple_use (TREE_OPERAND (cond_expr, 0), stmt, > + loop_vinfo, NULL, >emp, &def, &dts[0]); > + > + vec_cond_rhs = > + vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1), > + stmt, NULL, comp_vectype); > + vect_is_simple_use (TREE_OPERAND (cond_expr, 1), stmt, > + loop_vinfo, NULL, >emp, &def, &dts[1]); > + } > if (reduc_index == 1) > vec_then_clause = reduc_def; > else > @@ -7527,10 +7564,14 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, > } > else > { > - vec_cond_lhs = vect_get_vec_def_for_stmt_copy (dts[0], > - vec_oprnds0.pop ()); > - vec_cond_rhs = vect_get_vec_def_for_stmt_copy (dts[1], > - vec_oprnds1.pop ()); > + vec_cond_lhs > + = vect_get_vec_def_for_stmt_copy (dts[0], > + vec_oprnds0.pop ()); > + if (!masked) > + vec_cond_rhs > + = vect_get_vec_def_for_stmt_copy (dts[1], > + vec_oprnds1.pop ()); > + > vec_then_clause = vect_get_vec_def_for_stmt_copy (dts[2], > vec_oprnds2.pop ()); > vec_else_clause = vect_get_vec_def_for_stmt_copy (dts[3], > @@ -7540,7 +7581,8 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, > if (!slp_node) > { > vec_oprnds0.quick_push (vec_cond_lhs); > - vec_oprnds1.quick_push (vec_cond_rhs); > + if (!masked) > + vec_oprnds1.quick_push (vec_cond_rhs); > vec_oprnds2.quick_push (vec_then_clause); > vec_oprnds3.quick_push (vec_else_clause); > } > @@ -7548,12 +7590,17 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, > /* Arguments are ready. Create the new vector stmt. */ > FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_cond_lhs) > { > - vec_cond_rhs = vec_oprnds1[i]; > vec_then_clause = vec_oprnds2[i]; > vec_else_clause = vec_oprnds3[i]; > > - vec_compare = build2 (TREE_CODE (cond_expr), vec_cmp_type, > - vec_cond_lhs, vec_cond_rhs); > + if (masked) > + vec_compare = vec_cond_lhs; > + else > + { > + vec_cond_rhs = vec_oprnds1[i]; > + vec_compare = build2 (TREE_CODE (cond_expr), vec_cmp_type, > + vec_cond_lhs, vec_cond_rhs); > + } > vec_cond_expr = build3 (VEC_COND_EXPR, vectype, > vec_compare, vec_then_clause, vec_else_clause); >
2015-11-12 13:03 GMT+03:00 Ramana Radhakrishnan <ramana.gcc@googlemail.com>: > On Thu, Oct 8, 2015 at 4:50 PM, Ilya Enkovich <enkovich.gnu@gmail.com> wrote: >> Hi, >> >> This patch allows COND_EXPR with no embedded comparison to be vectorized. >> It's applied on top of vectorized comparison support series. New optab vcond_mask_optab >> is introduced for such statements. Bool patterns now avoid comparison in COND_EXPR in case vector comparison is supported by target. > > New standard pattern names are documented in the internals manual. > This patch does not do so neither do I see any patches to do so. > > > regards > Ramana Thanks for the point. I see we also miss description for some other patterns (e.g. maskload). Will add it. Ilya
diff --git a/gcc/optabs-query.h b/gcc/optabs-query.h index 162d2e9..48bcf7c 100644 --- a/gcc/optabs-query.h +++ b/gcc/optabs-query.h @@ -98,6 +98,15 @@ get_vcond_icode (machine_mode vmode, machine_mode cmode, bool uns) return icode; } +/* Return insn code for a conditional operator with a mask mode + MMODE resulting in a value of mode VMODE. */ + +static inline enum insn_code +get_vcond_mask_icode (machine_mode vmode, machine_mode mmode) +{ + return convert_optab_handler (vcond_mask_optab, vmode, mmode); +} + /* Enumerates the possible extraction_insn operations. */ enum extraction_pattern { EP_insv, EP_extv, EP_extzv }; diff --git a/gcc/optabs-tree.c b/gcc/optabs-tree.c index aa863cf..d887619 100644 --- a/gcc/optabs-tree.c +++ b/gcc/optabs-tree.c @@ -342,6 +342,9 @@ expand_vec_cond_expr_p (tree value_type, tree cmp_op_type) { machine_mode value_mode = TYPE_MODE (value_type); machine_mode cmp_op_mode = TYPE_MODE (cmp_op_type); + if (VECTOR_BOOLEAN_TYPE_P (cmp_op_type)) + return get_vcond_mask_icode (TYPE_MODE (value_type), + TYPE_MODE (cmp_op_type)) != CODE_FOR_nothing; if (GET_MODE_SIZE (value_mode) != GET_MODE_SIZE (cmp_op_mode) || GET_MODE_NUNITS (value_mode) != GET_MODE_NUNITS (cmp_op_mode) || get_vcond_icode (TYPE_MODE (value_type), TYPE_MODE (cmp_op_type), diff --git a/gcc/optabs.c b/gcc/optabs.c index ca1a6e7..d26b8f8 100644 --- a/gcc/optabs.c +++ b/gcc/optabs.c @@ -5346,6 +5346,38 @@ expand_vec_perm (machine_mode mode, rtx v0, rtx v1, rtx sel, rtx target) return tmp; } +/* Generate insns for a VEC_COND_EXPR with mask, given its TYPE and its + three operands. */ + +rtx +expand_vec_cond_mask_expr (tree vec_cond_type, tree op0, tree op1, tree op2, + rtx target) +{ + struct expand_operand ops[4]; + machine_mode mode = TYPE_MODE (vec_cond_type); + machine_mode mask_mode = TYPE_MODE (TREE_TYPE (op0)); + enum insn_code icode = get_vcond_mask_icode (mode, mask_mode); + rtx mask, rtx_op1, rtx_op2; + + if (icode == CODE_FOR_nothing) + return 0; + + mask = expand_normal (op0); + rtx_op1 = expand_normal (op1); + rtx_op2 = expand_normal (op2); + + mask = force_reg (GET_MODE (mask), mask); + rtx_op1 = force_reg (GET_MODE (rtx_op1), rtx_op1); + + create_output_operand (&ops[0], target, mode); + create_input_operand (&ops[1], rtx_op1, mode); + create_input_operand (&ops[2], rtx_op2, mode); + create_input_operand (&ops[3], mask, mask_mode); + expand_insn (icode, 4, ops); + + return ops[0].value; +} + /* Generate insns for a VEC_COND_EXPR, given its TYPE and its three operands. */ @@ -5371,12 +5403,21 @@ expand_vec_cond_expr (tree vec_cond_type, tree op0, tree op1, tree op2, } else { - /* Fake op0 < 0. */ gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0))); - op0a = op0; - op0b = build_zero_cst (TREE_TYPE (op0)); - tcode = LT_EXPR; - unsignedp = false; + if (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0))) + != CODE_FOR_nothing) + return expand_vec_cond_mask_expr (vec_cond_type, op0, op1, + op2, target); + /* Fake op0 < 0. */ + else + { + gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0))) + == MODE_VECTOR_INT); + op0a = op0; + op0b = build_zero_cst (TREE_TYPE (op0)); + tcode = LT_EXPR; + unsignedp = false; + } } cmp_op_mode = TYPE_MODE (TREE_TYPE (op0a)); diff --git a/gcc/optabs.def b/gcc/optabs.def index 9804378..70530a6 100644 --- a/gcc/optabs.def +++ b/gcc/optabs.def @@ -61,6 +61,7 @@ OPTAB_CD(vec_load_lanes_optab, "vec_load_lanes$a$b") OPTAB_CD(vec_store_lanes_optab, "vec_store_lanes$a$b") OPTAB_CD(vcond_optab, "vcond$a$b") OPTAB_CD(vcondu_optab, "vcondu$a$b") +OPTAB_CD(vcond_mask_optab, "vcond_mask_$a$b") OPTAB_CD(vec_cmp_optab, "vec_cmp$a$b") OPTAB_CD(vec_cmpu_optab, "vec_cmpu$a$b") OPTAB_CD(maskload_optab, "maskload$a$b") diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c index e3be3d1..6269b69 100644 --- a/gcc/tree-vect-patterns.c +++ b/gcc/tree-vect-patterns.c @@ -3356,7 +3356,7 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in, else { tree type = search_type_for_mask (var, loop_vinfo, bb_vinfo); - tree cst0, cst1, cmp, tmp; + tree cst0, cst1, tmp; if (!type) return NULL; @@ -3372,9 +3372,7 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in, cst0 = build_int_cst (type, 0); cst1 = build_int_cst (type, 1); tmp = vect_recog_temp_ssa_var (type, NULL); - cmp = build2 (NE_EXPR, boolean_type_node, - var, build_int_cst (TREE_TYPE (var), 0)); - pattern_stmt = gimple_build_assign (tmp, COND_EXPR, cmp, cst1, cst0); + pattern_stmt = gimple_build_assign (tmp, COND_EXPR, var, cst1, cst0); if (!useless_type_conversion_p (type, TREE_TYPE (lhs))) { @@ -3418,19 +3416,16 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in, if (get_vectype_for_scalar_type (type) == NULL_TREE) return NULL; - if (check_bool_pattern (var, loop_vinfo, bb_vinfo)) - { - rhs = adjust_bool_pattern (var, type, NULL_TREE, stmts); - rhs = build2 (NE_EXPR, boolean_type_node, - rhs, build_int_cst (type, 0)); - } - else - rhs = build2 (NE_EXPR, boolean_type_node, - var, build_int_cst (TREE_TYPE (var), 0)), + if (!check_bool_pattern (var, loop_vinfo, bb_vinfo)) + return NULL; + + rhs = adjust_bool_pattern (var, type, NULL_TREE, stmts); lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL); pattern_stmt - = gimple_build_assign (lhs, COND_EXPR, rhs, + = gimple_build_assign (lhs, COND_EXPR, + build2 (NE_EXPR, boolean_type_node, + rhs, build_int_cst (type, 0)), gimple_assign_rhs2 (last_stmt), gimple_assign_rhs3 (last_stmt)); *type_out = vectype; @@ -3457,7 +3452,7 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in, else { tree type = search_type_for_mask (var, loop_vinfo, bb_vinfo); - tree cst0, cst1, cmp, new_vectype; + tree cst0, cst1, new_vectype; if (!type) return NULL; @@ -3470,10 +3465,7 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in, new_vectype = get_vectype_for_scalar_type (type); rhs = vect_recog_temp_ssa_var (type, NULL); - cmp = build2 (NE_EXPR, boolean_type_node, - var, build_int_cst (TREE_TYPE (var), 0)); - pattern_stmt = gimple_build_assign (rhs, COND_EXPR, - cmp, cst1, cst0); + pattern_stmt = gimple_build_assign (rhs, COND_EXPR, var, cst1, cst0); pattern_stmt_info = new_stmt_vec_info (pattern_stmt, loop_vinfo, bb_vinfo); diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index d6819ec..b139601 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -7296,6 +7296,19 @@ vect_is_simple_cond (tree cond, gimple *stmt, loop_vec_info loop_vinfo, enum vect_def_type dt; tree vectype1 = NULL_TREE, vectype2 = NULL_TREE; + /* Mask case. */ + if (TREE_CODE (cond) == SSA_NAME + && TREE_CODE (TREE_TYPE (cond)) == BOOLEAN_TYPE) + { + gimple *lhs_def_stmt = SSA_NAME_DEF_STMT (cond); + if (!vect_is_simple_use_1 (cond, stmt, loop_vinfo, bb_vinfo, + &lhs_def_stmt, &def, &dt, comp_vectype) + || !*comp_vectype + || !VECTOR_BOOLEAN_TYPE_P (*comp_vectype)) + return false; + return true; + } + if (!COMPARISON_CLASS_P (cond)) return false; @@ -7368,6 +7381,7 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, vec<tree> vec_oprnds2 = vNULL; vec<tree> vec_oprnds3 = vNULL; tree vec_cmp_type; + bool masked = false; if (reduc_index && STMT_SLP_TYPE (stmt_info)) return false; @@ -7443,7 +7457,13 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, && TREE_CODE (else_clause) != FIXED_CST) return false; - vec_cmp_type = build_same_sized_truth_vector_type (comp_vectype); + if (VECTOR_BOOLEAN_TYPE_P (comp_vectype)) + { + vec_cmp_type = comp_vectype; + masked = true; + } + else + vec_cmp_type = build_same_sized_truth_vector_type (comp_vectype); if (vec_cmp_type == NULL_TREE) return false; @@ -7478,14 +7498,20 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, auto_vec<tree, 4> ops; auto_vec<vec<tree>, 4> vec_defs; - ops.safe_push (TREE_OPERAND (cond_expr, 0)); - ops.safe_push (TREE_OPERAND (cond_expr, 1)); + if (masked) + ops.safe_push (cond_expr); + else + { + ops.safe_push (TREE_OPERAND (cond_expr, 0)); + ops.safe_push (TREE_OPERAND (cond_expr, 1)); + } ops.safe_push (then_clause); ops.safe_push (else_clause); vect_get_slp_defs (ops, slp_node, &vec_defs, -1); vec_oprnds3 = vec_defs.pop (); vec_oprnds2 = vec_defs.pop (); - vec_oprnds1 = vec_defs.pop (); + if (!masked) + vec_oprnds1 = vec_defs.pop (); vec_oprnds0 = vec_defs.pop (); ops.release (); @@ -7494,17 +7520,28 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, else { gimple *gtemp; - vec_cond_lhs = - vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0), - stmt, NULL, comp_vectype); - vect_is_simple_use (TREE_OPERAND (cond_expr, 0), stmt, - loop_vinfo, NULL, >emp, &def, &dts[0]); - - vec_cond_rhs = - vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1), - stmt, NULL, comp_vectype); - vect_is_simple_use (TREE_OPERAND (cond_expr, 1), stmt, - loop_vinfo, NULL, >emp, &def, &dts[1]); + if (masked) + { + vec_cond_lhs + = vect_get_vec_def_for_operand (cond_expr, stmt, + NULL, comp_vectype); + vect_is_simple_use (cond_expr, stmt, loop_vinfo, NULL, + >emp, &def, &dts[0]); + } + else + { + vec_cond_lhs = + vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0), + stmt, NULL, comp_vectype); + vect_is_simple_use (TREE_OPERAND (cond_expr, 0), stmt, + loop_vinfo, NULL, >emp, &def, &dts[0]); + + vec_cond_rhs = + vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1), + stmt, NULL, comp_vectype); + vect_is_simple_use (TREE_OPERAND (cond_expr, 1), stmt, + loop_vinfo, NULL, >emp, &def, &dts[1]); + } if (reduc_index == 1) vec_then_clause = reduc_def; else @@ -7527,10 +7564,14 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, } else { - vec_cond_lhs = vect_get_vec_def_for_stmt_copy (dts[0], - vec_oprnds0.pop ()); - vec_cond_rhs = vect_get_vec_def_for_stmt_copy (dts[1], - vec_oprnds1.pop ()); + vec_cond_lhs + = vect_get_vec_def_for_stmt_copy (dts[0], + vec_oprnds0.pop ()); + if (!masked) + vec_cond_rhs + = vect_get_vec_def_for_stmt_copy (dts[1], + vec_oprnds1.pop ()); + vec_then_clause = vect_get_vec_def_for_stmt_copy (dts[2], vec_oprnds2.pop ()); vec_else_clause = vect_get_vec_def_for_stmt_copy (dts[3], @@ -7540,7 +7581,8 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, if (!slp_node) { vec_oprnds0.quick_push (vec_cond_lhs); - vec_oprnds1.quick_push (vec_cond_rhs); + if (!masked) + vec_oprnds1.quick_push (vec_cond_rhs); vec_oprnds2.quick_push (vec_then_clause); vec_oprnds3.quick_push (vec_else_clause); } @@ -7548,12 +7590,17 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, /* Arguments are ready. Create the new vector stmt. */ FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_cond_lhs) { - vec_cond_rhs = vec_oprnds1[i]; vec_then_clause = vec_oprnds2[i]; vec_else_clause = vec_oprnds3[i]; - vec_compare = build2 (TREE_CODE (cond_expr), vec_cmp_type, - vec_cond_lhs, vec_cond_rhs); + if (masked) + vec_compare = vec_cond_lhs; + else + { + vec_cond_rhs = vec_oprnds1[i]; + vec_compare = build2 (TREE_CODE (cond_expr), vec_cmp_type, + vec_cond_lhs, vec_cond_rhs); + } vec_cond_expr = build3 (VEC_COND_EXPR, vectype, vec_compare, vec_then_clause, vec_else_clause);