@@ -3191,6 +3191,75 @@ adjust_bool_pattern (tree var, tree out_type, tree trueval,
}
+/* Try to determine a proper type for converting bool VAR
+ into an integer value. The type is chosen so that
+ conversion has the same number of elements as a mask
+ producer. */
+
+static tree
+search_type_for_mask (tree var, loop_vec_info loop_vinfo, bb_vec_info bb_vinfo)
+{
+ gimple def_stmt;
+ enum vect_def_type dt;
+ tree def, rhs1;
+ enum tree_code rhs_code;
+ tree res = NULL;
+
+ if (TREE_CODE (var) != SSA_NAME)
+ return NULL;
+
+ if ((TYPE_PRECISION (TREE_TYPE (var)) != 1
+ || !TYPE_UNSIGNED (TREE_TYPE (var)))
+ && TREE_CODE (TREE_TYPE (var)) != BOOLEAN_TYPE)
+ return NULL;
+
+ if (!vect_is_simple_use (var, NULL, loop_vinfo, bb_vinfo, &def_stmt, &def,
+ &dt))
+ return NULL;
+
+ if (dt != vect_internal_def)
+ return NULL;
+
+ if (!is_gimple_assign (def_stmt))
+ return NULL;
+
+ rhs_code = gimple_assign_rhs_code (def_stmt);
+ rhs1 = gimple_assign_rhs1 (def_stmt);
+
+ switch (rhs_code)
+ {
+ case SSA_NAME:
+ case BIT_NOT_EXPR:
+ CASE_CONVERT:
+ res = search_type_for_mask (rhs1, loop_vinfo, bb_vinfo);
+ break;
+
+ case BIT_AND_EXPR:
+ case BIT_IOR_EXPR:
+ case BIT_XOR_EXPR:
+ if (!(res = search_type_for_mask (rhs1, loop_vinfo, bb_vinfo)))
+ res = search_type_for_mask (gimple_assign_rhs2 (def_stmt),
+ loop_vinfo, bb_vinfo);
+ break;
+
+ default:
+ if (TREE_CODE_CLASS (rhs_code) == tcc_comparison)
+ {
+ if (TREE_CODE (TREE_TYPE (rhs1)) != INTEGER_TYPE
+ || !TYPE_UNSIGNED (TREE_TYPE (rhs1)))
+ {
+ machine_mode mode = TYPE_MODE (TREE_TYPE (rhs1));
+ res = build_nonstandard_integer_type (GET_MODE_BITSIZE (mode), 1);
+ }
+ else
+ res = TREE_TYPE (rhs1);
+ }
+ }
+
+ return res;
+}
+
+
/* Function vect_recog_bool_pattern
Try to find pattern like following:
@@ -3248,6 +3317,7 @@ vect_recog_bool_pattern (vec<gimple> *stmts, tree *type_in,
enum tree_code rhs_code;
tree var, lhs, rhs, vectype;
stmt_vec_info stmt_vinfo = vinfo_for_stmt (last_stmt);
+ stmt_vec_info new_stmt_info;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_vinfo);
gimple pattern_stmt;
@@ -3273,27 +3343,56 @@ vect_recog_bool_pattern (vec<gimple> *stmts, tree *type_in,
if (vectype == NULL_TREE)
return NULL;
- if (!check_bool_pattern (var, loop_vinfo, bb_vinfo))
- return NULL;
+ if (targetm.vectorize.use_scalar_mask_p ())
+ {
+ tree type = search_type_for_mask (var, loop_vinfo, bb_vinfo);
+ tree cst0, cst1;
- rhs = adjust_bool_pattern (var, TREE_TYPE (lhs), NULL_TREE, stmts);
- lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
- if (useless_type_conversion_p (TREE_TYPE (lhs), TREE_TYPE (rhs)))
- pattern_stmt = gimple_build_assign (lhs, SSA_NAME, rhs);
+ if (!type || TYPE_MODE (type) == TYPE_MODE (TREE_TYPE (lhs)))
+ type = TREE_TYPE (lhs);
+ cst0 = build_int_cst (type, 0);
+ cst1 = build_int_cst (type, 1);
+ lhs = vect_recog_temp_ssa_var (type, NULL);
+ pattern_stmt = gimple_build_assign (lhs, COND_EXPR, var, cst0, cst1);
+
+ if (!useless_type_conversion_p (type, TREE_TYPE (lhs)))
+ {
+ tree new_vectype = get_vectype_for_scalar_type (type);
+ new_stmt_info = new_stmt_vec_info (pattern_stmt, loop_vinfo,
+ bb_vinfo);
+ set_vinfo_for_stmt (pattern_stmt, new_stmt_info);
+ STMT_VINFO_VECTYPE (new_stmt_info) = new_vectype;
+ new_pattern_def_seq (stmt_vinfo, pattern_stmt);
+
+ rhs = lhs;
+ lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
+ pattern_stmt = gimple_build_assign (lhs, CONVERT_EXPR, rhs);
+ }
+ }
else
- pattern_stmt
- = gimple_build_assign (lhs, NOP_EXPR, rhs);
+ {
+ if (!check_bool_pattern (var, loop_vinfo, bb_vinfo))
+ return NULL;
+
+ rhs = adjust_bool_pattern (var, TREE_TYPE (lhs), NULL_TREE, stmts);
+ lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
+ if (useless_type_conversion_p (TREE_TYPE (lhs), TREE_TYPE (rhs)))
+ pattern_stmt = gimple_build_assign (lhs, SSA_NAME, rhs);
+ else
+ pattern_stmt
+ = gimple_build_assign (lhs, NOP_EXPR, rhs);
+ }
*type_out = vectype;
*type_in = vectype;
stmts->safe_push (last_stmt);
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
"vect_recog_bool_pattern: detected:\n");
-
return pattern_stmt;
}
else if (rhs_code == COND_EXPR
- && TREE_CODE (var) == SSA_NAME)
+ && TREE_CODE (var) == SSA_NAME
+ && !targetm.vectorize.use_scalar_mask_p ())
{
vectype = get_vectype_for_scalar_type (TREE_TYPE (lhs));
if (vectype == NULL_TREE)
@@ -3339,21 +3438,49 @@ vect_recog_bool_pattern (vec<gimple> *stmts, tree *type_in,
gcc_assert (vectype != NULL_TREE);
if (!VECTOR_MODE_P (TYPE_MODE (vectype)))
return NULL;
- if (!check_bool_pattern (var, loop_vinfo, bb_vinfo))
- return NULL;
- rhs = adjust_bool_pattern (var, TREE_TYPE (vectype), NULL_TREE, stmts);
+ STMT_VINFO_PATTERN_DEF_SEQ (stmt_vinfo) = NULL;
+ if (targetm.vectorize.use_scalar_mask_p ())
+ {
+ tree type = search_type_for_mask (var, loop_vinfo, bb_vinfo);
+ tree cst0, cst1, new_vectype;
+
+ if (!type || TYPE_MODE (type) == TYPE_MODE (TREE_TYPE (vectype)))
+ type = TREE_TYPE (vectype);
+
+ cst0 = build_int_cst (type, 0);
+ cst1 = build_int_cst (type, 1);
+ new_vectype = get_vectype_for_scalar_type (type);
+
+ rhs = vect_recog_temp_ssa_var (type, NULL);
+ pattern_stmt = gimple_build_assign (rhs, COND_EXPR, var, cst0, cst1);
+
+ pattern_stmt_info = new_stmt_vec_info (pattern_stmt, loop_vinfo,
+ bb_vinfo);
+ set_vinfo_for_stmt (pattern_stmt, pattern_stmt_info);
+ STMT_VINFO_VECTYPE (pattern_stmt_info) = new_vectype;
+ append_pattern_def_seq (stmt_vinfo, pattern_stmt);
+ }
+ else
+ {
+ if (!check_bool_pattern (var, loop_vinfo, bb_vinfo))
+ return NULL;
+
+ rhs = adjust_bool_pattern (var, TREE_TYPE (vectype), NULL_TREE,
+ stmts);
+ }
+
lhs = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (vectype), lhs);
if (!useless_type_conversion_p (TREE_TYPE (lhs), TREE_TYPE (rhs)))
{
tree rhs2 = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
gimple cast_stmt = gimple_build_assign (rhs2, NOP_EXPR, rhs);
- new_pattern_def_seq (stmt_vinfo, cast_stmt);
+ append_pattern_def_seq (stmt_vinfo, cast_stmt);
rhs = rhs2;
}
pattern_stmt = gimple_build_assign (lhs, SSA_NAME, rhs);
pattern_stmt_info = new_stmt_vec_info (pattern_stmt, loop_vinfo,
- bb_vinfo);
+ bb_vinfo);
set_vinfo_for_stmt (pattern_stmt, pattern_stmt_info);
STMT_VINFO_DATA_REF (pattern_stmt_info)
= STMT_VINFO_DATA_REF (stmt_vinfo);