diff mbox series

[02/10] vect: Move vect_model_store_cost next to the transform in vectorizable_store

Message ID 1539ec7d34af4e38467420b3aed342d708a64a48.1694657494.git.linkw@linux.ibm.com
State New
Headers show
Series vect: Move costing next to the transform for vect store | expand

Commit Message

Kewen.Lin Sept. 14, 2023, 3:11 a.m. UTC 
This patch is an initial patch to move costing next to the
transform, it still adopts vect_model_store_cost for costing
but moves and duplicates it down according to the handlings
of different vect_memory_access_types or some special
handling need, hope it can make the subsequent patches easy
to review.  This patch should not have any functional
changes.

gcc/ChangeLog:

	* tree-vect-stmts.cc (vectorizable_store): Move and duplicate the call
	to vect_model_store_cost down to some different transform paths
	according to the handlings of different vect_memory_access_types
	or some special handling need.
---
 gcc/tree-vect-stmts.cc | 79 ++++++++++++++++++++++++++++++++----------
 1 file changed, 60 insertions(+), 19 deletions(-)

Comments

Richard Biener Sept. 27, 2023, 11:23 a.m. UTC | #1 
On Thu, Sep 14, 2023 at 5:12 AM Kewen Lin <linkw@linux.ibm.com> wrote:
>
> This patch is an initial patch to move costing next to the
> transform, it still adopts vect_model_store_cost for costing
> but moves and duplicates it down according to the handlings
> of different vect_memory_access_types or some special
> handling need, hope it can make the subsequent patches easy
> to review.  This patch should not have any functional
> changes.

OK

> gcc/ChangeLog:
>
>         * tree-vect-stmts.cc (vectorizable_store): Move and duplicate the call
>         to vect_model_store_cost down to some different transform paths
>         according to the handlings of different vect_memory_access_types
>         or some special handling need.
> ---
>  gcc/tree-vect-stmts.cc | 79 ++++++++++++++++++++++++++++++++----------
>  1 file changed, 60 insertions(+), 19 deletions(-)
>
> diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
> index a5caaf0bca2..36f7c5b9f4b 100644
> --- a/gcc/tree-vect-stmts.cc
> +++ b/gcc/tree-vect-stmts.cc
> @@ -8372,7 +8372,8 @@ vectorizable_store (vec_info *vinfo,
>         return false;
>      }
>
> -  if (!vec_stmt) /* transformation not required.  */
> +  bool costing_p = !vec_stmt;
> +  if (costing_p) /* transformation not required.  */
>      {
>        STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info) = memory_access_type;
>
> @@ -8401,11 +8402,6 @@ vectorizable_store (vec_info *vinfo,
>                          "Vectorizing an unaligned access.\n");
>
>        STMT_VINFO_TYPE (stmt_info) = store_vec_info_type;
> -      vect_model_store_cost (vinfo, stmt_info, ncopies,
> -                            memory_access_type, &gs_info,
> -                            alignment_support_scheme,
> -                            misalignment, vls_type, slp_node, cost_vec);
> -      return true;
>      }
>    gcc_assert (memory_access_type == STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info));
>
> @@ -8415,12 +8411,27 @@ vectorizable_store (vec_info *vinfo,
>
>    if (memory_access_type == VMAT_GATHER_SCATTER && gs_info.decl)
>      {
> -      vect_build_scatter_store_calls (vinfo, stmt_info, gsi, vec_stmt,
> -                                     &gs_info, mask);
> +      if (costing_p)
> +       vect_model_store_cost (vinfo, stmt_info, ncopies, memory_access_type,
> +                              &gs_info, alignment_support_scheme, misalignment,
> +                              vls_type, slp_node, cost_vec);
> +      else
> +       vect_build_scatter_store_calls (vinfo, stmt_info, gsi, vec_stmt,
> +                                       &gs_info, mask);
>        return true;
>      }
>    else if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) >= 3)
> -    return vectorizable_scan_store (vinfo, stmt_info, gsi, vec_stmt, ncopies);
> +    {
> +      gcc_assert (memory_access_type == VMAT_CONTIGUOUS);
> +      if (costing_p)
> +       {
> +         vect_model_store_cost (vinfo, stmt_info, ncopies, memory_access_type,
> +                                &gs_info, alignment_support_scheme,
> +                                misalignment, vls_type, slp_node, cost_vec);
> +         return true;
> +       }
> +      return vectorizable_scan_store (vinfo, stmt_info, gsi, vec_stmt, ncopies);
> +    }
>
>    if (grouped_store)
>      {
> @@ -8449,13 +8460,21 @@ vectorizable_store (vec_info *vinfo,
>    else
>      ref_type = reference_alias_ptr_type (DR_REF (first_dr_info->dr));
>
> -  if (dump_enabled_p ())
> -    dump_printf_loc (MSG_NOTE, vect_location,
> -                     "transform store. ncopies = %d\n", ncopies);
> +  if (!costing_p && dump_enabled_p ())
> +    dump_printf_loc (MSG_NOTE, vect_location, "transform store. ncopies = %d\n",
> +                    ncopies);
>
>    if (memory_access_type == VMAT_ELEMENTWISE
>        || memory_access_type == VMAT_STRIDED_SLP)
>      {
> +      if (costing_p)
> +       {
> +         vect_model_store_cost (vinfo, stmt_info, ncopies, memory_access_type,
> +                                &gs_info, alignment_support_scheme,
> +                                misalignment, vls_type, slp_node, cost_vec);
> +         return true;
> +       }
> +
>        gimple_stmt_iterator incr_gsi;
>        bool insert_after;
>        gimple *incr;
> @@ -8718,8 +8737,9 @@ vectorizable_store (vec_info *vinfo,
>    else if (memory_access_type == VMAT_GATHER_SCATTER)
>      {
>        aggr_type = elem_type;
> -      vect_get_strided_load_store_ops (stmt_info, loop_vinfo, gsi, &gs_info,
> -                                      &bump, &vec_offset, loop_lens);
> +      if (!costing_p)
> +       vect_get_strided_load_store_ops (stmt_info, loop_vinfo, gsi, &gs_info,
> +                                        &bump, &vec_offset, loop_lens);
>      }
>    else
>      {
> @@ -8731,7 +8751,7 @@ vectorizable_store (vec_info *vinfo,
>                                           memory_access_type, loop_lens);
>      }
>
> -  if (mask)
> +  if (mask && !costing_p)
>      LOOP_VINFO_HAS_MASK_STORE (loop_vinfo) = true;
>
>    /* In case the vectorization factor (VF) is bigger than the number
> @@ -8782,6 +8802,13 @@ vectorizable_store (vec_info *vinfo,
>    if (memory_access_type == VMAT_LOAD_STORE_LANES)
>      {
>        gcc_assert (!slp && grouped_store);
> +      if (costing_p)
> +       {
> +         vect_model_store_cost (vinfo, stmt_info, ncopies, memory_access_type,
> +                                &gs_info, alignment_support_scheme,
> +                                misalignment, vls_type, slp_node, cost_vec);
> +         return true;
> +       }
>        for (j = 0; j < ncopies; j++)
>         {
>           gimple *new_stmt;
> @@ -8927,6 +8954,13 @@ vectorizable_store (vec_info *vinfo,
>    if (memory_access_type == VMAT_GATHER_SCATTER)
>      {
>        gcc_assert (!slp && !grouped_store);
> +      if (costing_p)
> +       {
> +         vect_model_store_cost (vinfo, stmt_info, ncopies, memory_access_type,
> +                                &gs_info, alignment_support_scheme,
> +                                misalignment, vls_type, slp_node, cost_vec);
> +         return true;
> +       }
>        auto_vec<tree> vec_offsets;
>        for (j = 0; j < ncopies; j++)
>         {
> @@ -9091,7 +9125,7 @@ vectorizable_store (vec_info *vinfo,
>    for (j = 0; j < ncopies; j++)
>      {
>        gimple *new_stmt;
> -      if (j == 0)
> +      if (j == 0 && !costing_p)
>         {
>           if (slp)
>             {
> @@ -9158,7 +9192,7 @@ vectorizable_store (vec_info *vinfo,
>                                           offset, &dummy, gsi, &ptr_incr,
>                                           simd_lane_access_p, bump);
>         }
> -      else
> +      else if (!costing_p)
>         {
>           gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
>           /* DR_CHAIN is then used as an input to vect_permute_store_chain().
> @@ -9179,7 +9213,7 @@ vectorizable_store (vec_info *vinfo,
>         }
>
>        new_stmt = NULL;
> -      if (grouped_store)
> +      if (!costing_p && grouped_store)
>         /* Permute.  */
>         vect_permute_store_chain (vinfo, dr_chain, group_size, stmt_info, gsi,
>                                   &result_chain);
> @@ -9187,6 +9221,8 @@ vectorizable_store (vec_info *vinfo,
>        stmt_vec_info next_stmt_info = first_stmt_info;
>        for (i = 0; i < vec_num; i++)
>         {
> +         if (costing_p)
> +           continue;
>           unsigned misalign;
>           unsigned HOST_WIDE_INT align;
>
> @@ -9361,7 +9397,7 @@ vectorizable_store (vec_info *vinfo,
>           if (!next_stmt_info)
>             break;
>         }
> -      if (!slp)
> +      if (!slp && !costing_p)
>         {
>           if (j == 0)
>             *vec_stmt = new_stmt;
> @@ -9369,6 +9405,11 @@ vectorizable_store (vec_info *vinfo,
>         }
>      }
>
> +  if (costing_p)
> +    vect_model_store_cost (vinfo, stmt_info, ncopies, memory_access_type,
> +                          &gs_info, alignment_support_scheme, misalignment,
> +                          vls_type, slp_node, cost_vec);
> +
>    return true;
>  }
>
> --
> 2.31.1
>
diff mbox series

Patch

diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index a5caaf0bca2..36f7c5b9f4b 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -8372,7 +8372,8 @@  vectorizable_store (vec_info *vinfo,
 	return false;
     }
 
-  if (!vec_stmt) /* transformation not required.  */
+  bool costing_p = !vec_stmt;
+  if (costing_p) /* transformation not required.  */
     {
       STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info) = memory_access_type;
 
@@ -8401,11 +8402,6 @@  vectorizable_store (vec_info *vinfo,
 			 "Vectorizing an unaligned access.\n");
 
       STMT_VINFO_TYPE (stmt_info) = store_vec_info_type;
-      vect_model_store_cost (vinfo, stmt_info, ncopies,
-			     memory_access_type, &gs_info,
-			     alignment_support_scheme,
-			     misalignment, vls_type, slp_node, cost_vec);
-      return true;
     }
   gcc_assert (memory_access_type == STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info));
 
@@ -8415,12 +8411,27 @@  vectorizable_store (vec_info *vinfo,
 
   if (memory_access_type == VMAT_GATHER_SCATTER && gs_info.decl)
     {
-      vect_build_scatter_store_calls (vinfo, stmt_info, gsi, vec_stmt,
-				      &gs_info, mask);
+      if (costing_p)
+	vect_model_store_cost (vinfo, stmt_info, ncopies, memory_access_type,
+			       &gs_info, alignment_support_scheme, misalignment,
+			       vls_type, slp_node, cost_vec);
+      else
+	vect_build_scatter_store_calls (vinfo, stmt_info, gsi, vec_stmt,
+					&gs_info, mask);
       return true;
     }
   else if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) >= 3)
-    return vectorizable_scan_store (vinfo, stmt_info, gsi, vec_stmt, ncopies);
+    {
+      gcc_assert (memory_access_type == VMAT_CONTIGUOUS);
+      if (costing_p)
+	{
+	  vect_model_store_cost (vinfo, stmt_info, ncopies, memory_access_type,
+				 &gs_info, alignment_support_scheme,
+				 misalignment, vls_type, slp_node, cost_vec);
+	  return true;
+	}
+      return vectorizable_scan_store (vinfo, stmt_info, gsi, vec_stmt, ncopies);
+    }
 
   if (grouped_store)
     {
@@ -8449,13 +8460,21 @@  vectorizable_store (vec_info *vinfo,
   else
     ref_type = reference_alias_ptr_type (DR_REF (first_dr_info->dr));
 
-  if (dump_enabled_p ())
-    dump_printf_loc (MSG_NOTE, vect_location,
-                     "transform store. ncopies = %d\n", ncopies);
+  if (!costing_p && dump_enabled_p ())
+    dump_printf_loc (MSG_NOTE, vect_location, "transform store. ncopies = %d\n",
+		     ncopies);
 
   if (memory_access_type == VMAT_ELEMENTWISE
       || memory_access_type == VMAT_STRIDED_SLP)
     {
+      if (costing_p)
+	{
+	  vect_model_store_cost (vinfo, stmt_info, ncopies, memory_access_type,
+				 &gs_info, alignment_support_scheme,
+				 misalignment, vls_type, slp_node, cost_vec);
+	  return true;
+	}
+
       gimple_stmt_iterator incr_gsi;
       bool insert_after;
       gimple *incr;
@@ -8718,8 +8737,9 @@  vectorizable_store (vec_info *vinfo,
   else if (memory_access_type == VMAT_GATHER_SCATTER)
     {
       aggr_type = elem_type;
-      vect_get_strided_load_store_ops (stmt_info, loop_vinfo, gsi, &gs_info,
-				       &bump, &vec_offset, loop_lens);
+      if (!costing_p)
+	vect_get_strided_load_store_ops (stmt_info, loop_vinfo, gsi, &gs_info,
+					 &bump, &vec_offset, loop_lens);
     }
   else
     {
@@ -8731,7 +8751,7 @@  vectorizable_store (vec_info *vinfo,
 					  memory_access_type, loop_lens);
     }
 
-  if (mask)
+  if (mask && !costing_p)
     LOOP_VINFO_HAS_MASK_STORE (loop_vinfo) = true;
 
   /* In case the vectorization factor (VF) is bigger than the number
@@ -8782,6 +8802,13 @@  vectorizable_store (vec_info *vinfo,
   if (memory_access_type == VMAT_LOAD_STORE_LANES)
     {
       gcc_assert (!slp && grouped_store);
+      if (costing_p)
+	{
+	  vect_model_store_cost (vinfo, stmt_info, ncopies, memory_access_type,
+				 &gs_info, alignment_support_scheme,
+				 misalignment, vls_type, slp_node, cost_vec);
+	  return true;
+	}
       for (j = 0; j < ncopies; j++)
 	{
 	  gimple *new_stmt;
@@ -8927,6 +8954,13 @@  vectorizable_store (vec_info *vinfo,
   if (memory_access_type == VMAT_GATHER_SCATTER)
     {
       gcc_assert (!slp && !grouped_store);
+      if (costing_p)
+	{
+	  vect_model_store_cost (vinfo, stmt_info, ncopies, memory_access_type,
+				 &gs_info, alignment_support_scheme,
+				 misalignment, vls_type, slp_node, cost_vec);
+	  return true;
+	}
       auto_vec<tree> vec_offsets;
       for (j = 0; j < ncopies; j++)
 	{
@@ -9091,7 +9125,7 @@  vectorizable_store (vec_info *vinfo,
   for (j = 0; j < ncopies; j++)
     {
       gimple *new_stmt;
-      if (j == 0)
+      if (j == 0 && !costing_p)
 	{
 	  if (slp)
 	    {
@@ -9158,7 +9192,7 @@  vectorizable_store (vec_info *vinfo,
 					  offset, &dummy, gsi, &ptr_incr,
 					  simd_lane_access_p, bump);
 	}
-      else
+      else if (!costing_p)
 	{
 	  gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
 	  /* DR_CHAIN is then used as an input to vect_permute_store_chain().
@@ -9179,7 +9213,7 @@  vectorizable_store (vec_info *vinfo,
 	}
 
       new_stmt = NULL;
-      if (grouped_store)
+      if (!costing_p && grouped_store)
 	/* Permute.  */
 	vect_permute_store_chain (vinfo, dr_chain, group_size, stmt_info, gsi,
 				  &result_chain);
@@ -9187,6 +9221,8 @@  vectorizable_store (vec_info *vinfo,
       stmt_vec_info next_stmt_info = first_stmt_info;
       for (i = 0; i < vec_num; i++)
 	{
+	  if (costing_p)
+	    continue;
 	  unsigned misalign;
 	  unsigned HOST_WIDE_INT align;
 
@@ -9361,7 +9397,7 @@  vectorizable_store (vec_info *vinfo,
 	  if (!next_stmt_info)
 	    break;
 	}
-      if (!slp)
+      if (!slp && !costing_p)
 	{
 	  if (j == 0)
 	    *vec_stmt = new_stmt;
@@ -9369,6 +9405,11 @@  vectorizable_store (vec_info *vinfo,
 	}
     }
 
+  if (costing_p)
+    vect_model_store_cost (vinfo, stmt_info, ncopies, memory_access_type,
+			   &gs_info, alignment_support_scheme, misalignment,
+			   vls_type, slp_node, cost_vec);
+
   return true;
 }