From patchwork Thu Oct 8 15:50:30 2015 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ilya Enkovich X-Patchwork-Id: 527805 Return-Path: X-Original-To: incoming@patchwork.ozlabs.org Delivered-To: patchwork-incoming@bilbo.ozlabs.org Received: from sourceware.org (server1.sourceware.org [209.132.180.131]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 0DDAF14012C for ; Fri, 9 Oct 2015 02:51:04 +1100 (AEDT) Authentication-Results: ozlabs.org; dkim=pass (1024-bit key; unprotected) header.d=gcc.gnu.org header.i=@gcc.gnu.org header.b=ricpkm0/; dkim-atps=neutral DomainKey-Signature: a=rsa-sha1; c=nofws; d=gcc.gnu.org; h=list-id :list-unsubscribe:list-archive:list-post:list-help:sender:date :from:to:subject:message-id:mime-version:content-type; q=dns; s= default; b=AHvTj1AZYjSQFPHahTTyy8uUViXh447lrgo85oE0/QSZLdzjbhJTD yRboALn68tl5ODYVaAkacYa3ZWyhbtSaSXP0RVQ8zeVKgBvMnVmGJKPSG7tiPX6X sZw8+nDqgeytVReLIaZnMNC08xxT+2QmYAuYiZ6tc7tMiXyHSsYrBc= DKIM-Signature: v=1; a=rsa-sha1; c=relaxed; d=gcc.gnu.org; h=list-id :list-unsubscribe:list-archive:list-post:list-help:sender:date :from:to:subject:message-id:mime-version:content-type; s= default; bh=fmgioZbDJBvngwhq+sKyfce1RYE=; b=ricpkm0/4KPTdLfKO48V 2wJaDioDYQ6ACG8ww039W7l+ODlaSzksl6rGIG8tCfvm6rugB+Y31AFrh6YsJwkq 9Cy8mMRw/GI6/ha2rqiyZt0OhGLuUKXMER/0TDEVlQ19Iq498tHPV6Ubq2mYMhxB OQn6Zio/nb3UMoVS99hamvM= Received: (qmail 25863 invoked by alias); 8 Oct 2015 15:50:58 -0000 Mailing-List: contact gcc-patches-help@gcc.gnu.org; run by ezmlm Precedence: bulk List-Id: List-Unsubscribe: List-Archive: List-Post: List-Help: Sender: gcc-patches-owner@gcc.gnu.org Delivered-To: mailing list gcc-patches@gcc.gnu.org Received: (qmail 25831 invoked by uid 89); 8 Oct 2015 15:50:57 -0000 Authentication-Results: sourceware.org; auth=none X-Virus-Found: No X-Spam-SWARE-Status: No, score=-2.5 required=5.0 tests=AWL, BAYES_00, FREEMAIL_FROM, RCVD_IN_DNSWL_LOW, SPF_PASS autolearn=ham version=3.3.2 X-HELO: mail-qg0-f51.google.com Received: from mail-qg0-f51.google.com (HELO mail-qg0-f51.google.com) (209.85.192.51) by sourceware.org (qpsmtpd/0.93/v0.84-503-g423c35a) with (AES128-GCM-SHA256 encrypted) ESMTPS; Thu, 08 Oct 2015 15:50:54 +0000 Received: by qgt47 with SMTP id 47so45474629qgt.2 for ; Thu, 08 Oct 2015 08:50:52 -0700 (PDT) X-Received: by 10.140.99.117 with SMTP id p108mr8932703qge.91.1444319452751; Thu, 08 Oct 2015 08:50:52 -0700 (PDT) Received: from msticlxl57.ims.intel.com (jfdmzpr02-ext.jf.intel.com. [134.134.137.71]) by smtp.gmail.com with ESMTPSA id q86sm10743336qkl.21.2015.10.08.08.50.50 for (version=TLSv1.2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128); Thu, 08 Oct 2015 08:50:52 -0700 (PDT) Date: Thu, 8 Oct 2015 18:50:30 +0300 From: Ilya Enkovich To: gcc-patches@gcc.gnu.org Subject: [mask-vec_cond, patch 1/2] Support vectorization of VEC_COND_EXPR with no embedded comparison Message-ID: <20151008155030.GJ63757@msticlxl57.ims.intel.com> MIME-Version: 1.0 Content-Disposition: inline User-Agent: Mutt/1.5.23 (2014-03-12) X-IsSubscribed: yes Hi, This patch allows COND_EXPR with no embedded comparison to be vectorized. It's applied on top of vectorized comparison support series. New optab vcond_mask_optab is introduced for such statements. Bool patterns now avoid comparison in COND_EXPR in case vector comparison is supported by target. Thanks, Ilya --- gcc/ 2015-10-08 Ilya Enkovich * optabs-query.h (get_vcond_mask_icode): New. * optabs-tree.c (expand_vec_cond_expr_p): Use get_vcond_mask_icode for VEC_COND_EXPR with mask. * optabs.c (expand_vec_cond_mask_expr): New. (expand_vec_cond_expr): Use get_vcond_mask_icode when possible. * optabs.def (vcond_mask_optab): New. * tree-vect-patterns.c (vect_recog_bool_pattern): Don't generate redundant comparison for COND_EXPR. * tree-vect-stmts.c (vect_is_simple_cond): Allow SSA_NAME as a condition. (vectorizable_condition): Likewise. diff --git a/gcc/optabs-query.h b/gcc/optabs-query.h index 162d2e9..48bcf7c 100644 --- a/gcc/optabs-query.h +++ b/gcc/optabs-query.h @@ -98,6 +98,15 @@ get_vcond_icode (machine_mode vmode, machine_mode cmode, bool uns) return icode; } +/* Return insn code for a conditional operator with a mask mode + MMODE resulting in a value of mode VMODE. */ + +static inline enum insn_code +get_vcond_mask_icode (machine_mode vmode, machine_mode mmode) +{ + return convert_optab_handler (vcond_mask_optab, vmode, mmode); +} + /* Enumerates the possible extraction_insn operations. */ enum extraction_pattern { EP_insv, EP_extv, EP_extzv }; diff --git a/gcc/optabs-tree.c b/gcc/optabs-tree.c index aa863cf..d887619 100644 --- a/gcc/optabs-tree.c +++ b/gcc/optabs-tree.c @@ -342,6 +342,9 @@ expand_vec_cond_expr_p (tree value_type, tree cmp_op_type) { machine_mode value_mode = TYPE_MODE (value_type); machine_mode cmp_op_mode = TYPE_MODE (cmp_op_type); + if (VECTOR_BOOLEAN_TYPE_P (cmp_op_type)) + return get_vcond_mask_icode (TYPE_MODE (value_type), + TYPE_MODE (cmp_op_type)) != CODE_FOR_nothing; if (GET_MODE_SIZE (value_mode) != GET_MODE_SIZE (cmp_op_mode) || GET_MODE_NUNITS (value_mode) != GET_MODE_NUNITS (cmp_op_mode) || get_vcond_icode (TYPE_MODE (value_type), TYPE_MODE (cmp_op_type), diff --git a/gcc/optabs.c b/gcc/optabs.c index ca1a6e7..d26b8f8 100644 --- a/gcc/optabs.c +++ b/gcc/optabs.c @@ -5346,6 +5346,38 @@ expand_vec_perm (machine_mode mode, rtx v0, rtx v1, rtx sel, rtx target) return tmp; } +/* Generate insns for a VEC_COND_EXPR with mask, given its TYPE and its + three operands. */ + +rtx +expand_vec_cond_mask_expr (tree vec_cond_type, tree op0, tree op1, tree op2, + rtx target) +{ + struct expand_operand ops[4]; + machine_mode mode = TYPE_MODE (vec_cond_type); + machine_mode mask_mode = TYPE_MODE (TREE_TYPE (op0)); + enum insn_code icode = get_vcond_mask_icode (mode, mask_mode); + rtx mask, rtx_op1, rtx_op2; + + if (icode == CODE_FOR_nothing) + return 0; + + mask = expand_normal (op0); + rtx_op1 = expand_normal (op1); + rtx_op2 = expand_normal (op2); + + mask = force_reg (GET_MODE (mask), mask); + rtx_op1 = force_reg (GET_MODE (rtx_op1), rtx_op1); + + create_output_operand (&ops[0], target, mode); + create_input_operand (&ops[1], rtx_op1, mode); + create_input_operand (&ops[2], rtx_op2, mode); + create_input_operand (&ops[3], mask, mask_mode); + expand_insn (icode, 4, ops); + + return ops[0].value; +} + /* Generate insns for a VEC_COND_EXPR, given its TYPE and its three operands. */ @@ -5371,12 +5403,21 @@ expand_vec_cond_expr (tree vec_cond_type, tree op0, tree op1, tree op2, } else { - /* Fake op0 < 0. */ gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0))); - op0a = op0; - op0b = build_zero_cst (TREE_TYPE (op0)); - tcode = LT_EXPR; - unsignedp = false; + if (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0))) + != CODE_FOR_nothing) + return expand_vec_cond_mask_expr (vec_cond_type, op0, op1, + op2, target); + /* Fake op0 < 0. */ + else + { + gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0))) + == MODE_VECTOR_INT); + op0a = op0; + op0b = build_zero_cst (TREE_TYPE (op0)); + tcode = LT_EXPR; + unsignedp = false; + } } cmp_op_mode = TYPE_MODE (TREE_TYPE (op0a)); diff --git a/gcc/optabs.def b/gcc/optabs.def index 9804378..70530a6 100644 --- a/gcc/optabs.def +++ b/gcc/optabs.def @@ -61,6 +61,7 @@ OPTAB_CD(vec_load_lanes_optab, "vec_load_lanes$a$b") OPTAB_CD(vec_store_lanes_optab, "vec_store_lanes$a$b") OPTAB_CD(vcond_optab, "vcond$a$b") OPTAB_CD(vcondu_optab, "vcondu$a$b") +OPTAB_CD(vcond_mask_optab, "vcond_mask_$a$b") OPTAB_CD(vec_cmp_optab, "vec_cmp$a$b") OPTAB_CD(vec_cmpu_optab, "vec_cmpu$a$b") OPTAB_CD(maskload_optab, "maskload$a$b") diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c index e3be3d1..6269b69 100644 --- a/gcc/tree-vect-patterns.c +++ b/gcc/tree-vect-patterns.c @@ -3356,7 +3356,7 @@ vect_recog_bool_pattern (vec *stmts, tree *type_in, else { tree type = search_type_for_mask (var, loop_vinfo, bb_vinfo); - tree cst0, cst1, cmp, tmp; + tree cst0, cst1, tmp; if (!type) return NULL; @@ -3372,9 +3372,7 @@ vect_recog_bool_pattern (vec *stmts, tree *type_in, cst0 = build_int_cst (type, 0); cst1 = build_int_cst (type, 1); tmp = vect_recog_temp_ssa_var (type, NULL); - cmp = build2 (NE_EXPR, boolean_type_node, - var, build_int_cst (TREE_TYPE (var), 0)); - pattern_stmt = gimple_build_assign (tmp, COND_EXPR, cmp, cst1, cst0); + pattern_stmt = gimple_build_assign (tmp, COND_EXPR, var, cst1, cst0); if (!useless_type_conversion_p (type, TREE_TYPE (lhs))) { @@ -3418,19 +3416,16 @@ vect_recog_bool_pattern (vec *stmts, tree *type_in, if (get_vectype_for_scalar_type (type) == NULL_TREE) return NULL; - if (check_bool_pattern (var, loop_vinfo, bb_vinfo)) - { - rhs = adjust_bool_pattern (var, type, NULL_TREE, stmts); - rhs = build2 (NE_EXPR, boolean_type_node, - rhs, build_int_cst (type, 0)); - } - else - rhs = build2 (NE_EXPR, boolean_type_node, - var, build_int_cst (TREE_TYPE (var), 0)), + if (!check_bool_pattern (var, loop_vinfo, bb_vinfo)) + return NULL; + + rhs = adjust_bool_pattern (var, type, NULL_TREE, stmts); lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL); pattern_stmt - = gimple_build_assign (lhs, COND_EXPR, rhs, + = gimple_build_assign (lhs, COND_EXPR, + build2 (NE_EXPR, boolean_type_node, + rhs, build_int_cst (type, 0)), gimple_assign_rhs2 (last_stmt), gimple_assign_rhs3 (last_stmt)); *type_out = vectype; @@ -3457,7 +3452,7 @@ vect_recog_bool_pattern (vec *stmts, tree *type_in, else { tree type = search_type_for_mask (var, loop_vinfo, bb_vinfo); - tree cst0, cst1, cmp, new_vectype; + tree cst0, cst1, new_vectype; if (!type) return NULL; @@ -3470,10 +3465,7 @@ vect_recog_bool_pattern (vec *stmts, tree *type_in, new_vectype = get_vectype_for_scalar_type (type); rhs = vect_recog_temp_ssa_var (type, NULL); - cmp = build2 (NE_EXPR, boolean_type_node, - var, build_int_cst (TREE_TYPE (var), 0)); - pattern_stmt = gimple_build_assign (rhs, COND_EXPR, - cmp, cst1, cst0); + pattern_stmt = gimple_build_assign (rhs, COND_EXPR, var, cst1, cst0); pattern_stmt_info = new_stmt_vec_info (pattern_stmt, loop_vinfo, bb_vinfo); diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index d6819ec..b139601 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -7296,6 +7296,19 @@ vect_is_simple_cond (tree cond, gimple *stmt, loop_vec_info loop_vinfo, enum vect_def_type dt; tree vectype1 = NULL_TREE, vectype2 = NULL_TREE; + /* Mask case. */ + if (TREE_CODE (cond) == SSA_NAME + && TREE_CODE (TREE_TYPE (cond)) == BOOLEAN_TYPE) + { + gimple *lhs_def_stmt = SSA_NAME_DEF_STMT (cond); + if (!vect_is_simple_use_1 (cond, stmt, loop_vinfo, bb_vinfo, + &lhs_def_stmt, &def, &dt, comp_vectype) + || !*comp_vectype + || !VECTOR_BOOLEAN_TYPE_P (*comp_vectype)) + return false; + return true; + } + if (!COMPARISON_CLASS_P (cond)) return false; @@ -7368,6 +7381,7 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, vec vec_oprnds2 = vNULL; vec vec_oprnds3 = vNULL; tree vec_cmp_type; + bool masked = false; if (reduc_index && STMT_SLP_TYPE (stmt_info)) return false; @@ -7443,7 +7457,13 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, && TREE_CODE (else_clause) != FIXED_CST) return false; - vec_cmp_type = build_same_sized_truth_vector_type (comp_vectype); + if (VECTOR_BOOLEAN_TYPE_P (comp_vectype)) + { + vec_cmp_type = comp_vectype; + masked = true; + } + else + vec_cmp_type = build_same_sized_truth_vector_type (comp_vectype); if (vec_cmp_type == NULL_TREE) return false; @@ -7478,14 +7498,20 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, auto_vec ops; auto_vec, 4> vec_defs; - ops.safe_push (TREE_OPERAND (cond_expr, 0)); - ops.safe_push (TREE_OPERAND (cond_expr, 1)); + if (masked) + ops.safe_push (cond_expr); + else + { + ops.safe_push (TREE_OPERAND (cond_expr, 0)); + ops.safe_push (TREE_OPERAND (cond_expr, 1)); + } ops.safe_push (then_clause); ops.safe_push (else_clause); vect_get_slp_defs (ops, slp_node, &vec_defs, -1); vec_oprnds3 = vec_defs.pop (); vec_oprnds2 = vec_defs.pop (); - vec_oprnds1 = vec_defs.pop (); + if (!masked) + vec_oprnds1 = vec_defs.pop (); vec_oprnds0 = vec_defs.pop (); ops.release (); @@ -7494,17 +7520,28 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, else { gimple *gtemp; - vec_cond_lhs = - vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0), - stmt, NULL, comp_vectype); - vect_is_simple_use (TREE_OPERAND (cond_expr, 0), stmt, - loop_vinfo, NULL, >emp, &def, &dts[0]); - - vec_cond_rhs = - vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1), - stmt, NULL, comp_vectype); - vect_is_simple_use (TREE_OPERAND (cond_expr, 1), stmt, - loop_vinfo, NULL, >emp, &def, &dts[1]); + if (masked) + { + vec_cond_lhs + = vect_get_vec_def_for_operand (cond_expr, stmt, + NULL, comp_vectype); + vect_is_simple_use (cond_expr, stmt, loop_vinfo, NULL, + >emp, &def, &dts[0]); + } + else + { + vec_cond_lhs = + vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0), + stmt, NULL, comp_vectype); + vect_is_simple_use (TREE_OPERAND (cond_expr, 0), stmt, + loop_vinfo, NULL, >emp, &def, &dts[0]); + + vec_cond_rhs = + vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1), + stmt, NULL, comp_vectype); + vect_is_simple_use (TREE_OPERAND (cond_expr, 1), stmt, + loop_vinfo, NULL, >emp, &def, &dts[1]); + } if (reduc_index == 1) vec_then_clause = reduc_def; else @@ -7527,10 +7564,14 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, } else { - vec_cond_lhs = vect_get_vec_def_for_stmt_copy (dts[0], - vec_oprnds0.pop ()); - vec_cond_rhs = vect_get_vec_def_for_stmt_copy (dts[1], - vec_oprnds1.pop ()); + vec_cond_lhs + = vect_get_vec_def_for_stmt_copy (dts[0], + vec_oprnds0.pop ()); + if (!masked) + vec_cond_rhs + = vect_get_vec_def_for_stmt_copy (dts[1], + vec_oprnds1.pop ()); + vec_then_clause = vect_get_vec_def_for_stmt_copy (dts[2], vec_oprnds2.pop ()); vec_else_clause = vect_get_vec_def_for_stmt_copy (dts[3], @@ -7540,7 +7581,8 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, if (!slp_node) { vec_oprnds0.quick_push (vec_cond_lhs); - vec_oprnds1.quick_push (vec_cond_rhs); + if (!masked) + vec_oprnds1.quick_push (vec_cond_rhs); vec_oprnds2.quick_push (vec_then_clause); vec_oprnds3.quick_push (vec_else_clause); } @@ -7548,12 +7590,17 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, /* Arguments are ready. Create the new vector stmt. */ FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_cond_lhs) { - vec_cond_rhs = vec_oprnds1[i]; vec_then_clause = vec_oprnds2[i]; vec_else_clause = vec_oprnds3[i]; - vec_compare = build2 (TREE_CODE (cond_expr), vec_cmp_type, - vec_cond_lhs, vec_cond_rhs); + if (masked) + vec_compare = vec_cond_lhs; + else + { + vec_cond_rhs = vec_oprnds1[i]; + vec_compare = build2 (TREE_CODE (cond_expr), vec_cmp_type, + vec_cond_lhs, vec_cond_rhs); + } vec_cond_expr = build3 (VEC_COND_EXPR, vectype, vec_compare, vec_then_clause, vec_else_clause);