diff mbox series

[2/3] vect: Move VMAT_LOAD_STORE_LANES handlings from final loop nest

Message ID 8a82c294-eaab-bfb2-5e2d-a08d38f3e570@linux.ibm.com
State New
Headers show
Series [1/3] vect: Remove some manual release in vectorizable_store | expand

Commit Message

Kewen.Lin Aug. 22, 2023, 8:49 a.m. UTC 
Hi,

Like commit r14-3214 which moves the handlings on memory
access type VMAT_LOAD_STORE_LANES in vectorizable_load
final loop nest, this one is to deal with the function
vectorizable_store.

Bootstrapped and regtested on x86_64-redhat-linux,
aarch64-linux-gnu and powerpc64{,le}-linux-gnu.

Is it ok for trunk?

BR,
Kewen
-----

gcc/ChangeLog:

	* tree-vect-stmts.cc (vectorizable_store): Move the handlings on
	VMAT_LOAD_STORE_LANES in the final loop nest to its own loop,
	and update the final nest accordingly.
---
 gcc/tree-vect-stmts.cc | 732 ++++++++++++++++++++++-------------------
 1 file changed, 387 insertions(+), 345 deletions(-)

--
2.31.1

Comments

Richard Biener Aug. 22, 2023, 1:27 p.m. UTC | #1 
On Tue, Aug 22, 2023 at 10:49 AM Kewen.Lin <linkw@linux.ibm.com> wrote:
>
> Hi,
>
> Like commit r14-3214 which moves the handlings on memory
> access type VMAT_LOAD_STORE_LANES in vectorizable_load
> final loop nest, this one is to deal with the function
> vectorizable_store.
>
> Bootstrapped and regtested on x86_64-redhat-linux,
> aarch64-linux-gnu and powerpc64{,le}-linux-gnu.
>
> Is it ok for trunk?

OK.

> BR,
> Kewen
> -----
>
> gcc/ChangeLog:
>
>         * tree-vect-stmts.cc (vectorizable_store): Move the handlings on
>         VMAT_LOAD_STORE_LANES in the final loop nest to its own loop,
>         and update the final nest accordingly.
> ---
>  gcc/tree-vect-stmts.cc | 732 ++++++++++++++++++++++-------------------
>  1 file changed, 387 insertions(+), 345 deletions(-)
>
> diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
> index fcaa4127e52..18f5ebcc09c 100644
> --- a/gcc/tree-vect-stmts.cc
> +++ b/gcc/tree-vect-stmts.cc
> @@ -8779,42 +8779,29 @@ vectorizable_store (vec_info *vinfo,
>    */
>
>    auto_vec<tree> dr_chain (group_size);
> -  auto_vec<tree> result_chain (group_size);
>    auto_vec<tree> vec_masks;
>    tree vec_mask = NULL;
> -  auto_vec<tree> vec_offsets;
>    auto_delete_vec<auto_vec<tree>> gvec_oprnds (group_size);
>    for (i = 0; i < group_size; i++)
>      gvec_oprnds.quick_push (new auto_vec<tree> (ncopies));
> -  auto_vec<tree, 1> vec_oprnds;
> -  for (j = 0; j < ncopies; j++)
> +
> +  if (memory_access_type == VMAT_LOAD_STORE_LANES)
>      {
> -      gimple *new_stmt;
> -      if (j == 0)
> +      gcc_assert (!slp && grouped_store);
> +      for (j = 0; j < ncopies; j++)
>         {
> -          if (slp)
> -            {
> -             /* Get vectorized arguments for SLP_NODE.  */
> -             vect_get_vec_defs (vinfo, stmt_info, slp_node, 1,
> -                                op, &vec_oprnds);
> -              vec_oprnd = vec_oprnds[0];
> -            }
> -          else
> -            {
> -             /* For interleaved stores we collect vectorized defs for all the
> -                stores in the group in DR_CHAIN. DR_CHAIN is then used as an
> -                input to vect_permute_store_chain().
> -
> -                If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN
> -                is of size 1.  */
> +         gimple *new_stmt;
> +         if (j == 0)
> +           {
> +             /* For interleaved stores we collect vectorized defs for all
> +                the stores in the group in DR_CHAIN. DR_CHAIN is then used
> +                as an input to vect_permute_store_chain().  */
>               stmt_vec_info next_stmt_info = first_stmt_info;
>               for (i = 0; i < group_size; i++)
>                 {
>                   /* Since gaps are not supported for interleaved stores,
> -                    DR_GROUP_SIZE is the exact number of stmts in the chain.
> -                    Therefore, NEXT_STMT_INFO can't be NULL_TREE.  In case
> -                    that there is no interleaving, DR_GROUP_SIZE is 1,
> -                    and only one iteration of the loop will be executed.  */
> +                    DR_GROUP_SIZE is the exact number of stmts in the
> +                    chain. Therefore, NEXT_STMT_INFO can't be NULL_TREE.  */
>                   op = vect_get_store_rhs (next_stmt_info);
>                   vect_get_vec_defs_for_operand (vinfo, next_stmt_info, ncopies,
>                                                  op, gvec_oprnds[i]);
> @@ -8825,66 +8812,37 @@ vectorizable_store (vec_info *vinfo,
>               if (mask)
>                 {
>                   vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies,
> -                                                mask, &vec_masks, mask_vectype);
> +                                                mask, &vec_masks,
> +                                                mask_vectype);
>                   vec_mask = vec_masks[0];
>                 }
> -           }
>
> -         /* We should have catched mismatched types earlier.  */
> -         gcc_assert (useless_type_conversion_p (vectype,
> -                                                TREE_TYPE (vec_oprnd)));
> -         bool simd_lane_access_p
> -           = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) != 0;
> -         if (simd_lane_access_p
> -             && !loop_masks
> -             && TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR
> -             && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0))
> -             && integer_zerop (get_dr_vinfo_offset (vinfo, first_dr_info))
> -             && integer_zerop (DR_INIT (first_dr_info->dr))
> -             && alias_sets_conflict_p (get_alias_set (aggr_type),
> -                                       get_alias_set (TREE_TYPE (ref_type))))
> -           {
> -             dataref_ptr = unshare_expr (DR_BASE_ADDRESS (first_dr_info->dr));
> -             dataref_offset = build_int_cst (ref_type, 0);
> +             /* We should have catched mismatched types earlier.  */
> +             gcc_assert (
> +               useless_type_conversion_p (vectype, TREE_TYPE (vec_oprnd)));
> +             dataref_ptr
> +               = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
> +                                           NULL, offset, &dummy, gsi,
> +                                           &ptr_incr, false, bump);
>             }
> -         else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> -           vect_get_gather_scatter_ops (loop_vinfo, loop, stmt_info,
> -                                        slp_node, &gs_info, &dataref_ptr,
> -                                        &vec_offsets);
>           else
> -           dataref_ptr
> -             = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
> -                                         simd_lane_access_p ? loop : NULL,
> -                                         offset, &dummy, gsi, &ptr_incr,
> -                                         simd_lane_access_p, bump);
> -       }
> -      else
> -       {
> -         gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
> -         /* DR_CHAIN is then used as an input to vect_permute_store_chain().
> -            If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN is
> -            of size 1.  */
> -         for (i = 0; i < group_size; i++)
>             {
> -             vec_oprnd = (*gvec_oprnds[i])[j];
> -             dr_chain[i] = vec_oprnd;
> +             gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
> +             /* DR_CHAIN is then used as an input to
> +                vect_permute_store_chain().  */
> +             for (i = 0; i < group_size; i++)
> +               {
> +                 vec_oprnd = (*gvec_oprnds[i])[j];
> +                 dr_chain[i] = vec_oprnd;
> +               }
> +             if (mask)
> +               vec_mask = vec_masks[j];
> +             dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
> +                                            stmt_info, bump);
>             }
> -         if (mask)
> -           vec_mask = vec_masks[j];
> -         if (dataref_offset)
> -           dataref_offset
> -             = int_const_binop (PLUS_EXPR, dataref_offset, bump);
> -         else if (!STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> -           dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
> -                                          stmt_info, bump);
> -       }
> -
> -      if (memory_access_type == VMAT_LOAD_STORE_LANES)
> -       {
> -         tree vec_array;
>
>           /* Get an array into which we can store the individual vectors.  */
> -         vec_array = create_vector_array (vectype, vec_num);
> +         tree vec_array = create_vector_array (vectype, vec_num);
>
>           /* Invalidate the current contents of VEC_ARRAY.  This should
>              become an RTL clobber too, which prevents the vector registers
> @@ -8895,8 +8853,8 @@ vectorizable_store (vec_info *vinfo,
>           for (i = 0; i < vec_num; i++)
>             {
>               vec_oprnd = dr_chain[i];
> -             write_vector_array (vinfo, stmt_info,
> -                                 gsi, vec_oprnd, vec_array, i);
> +             write_vector_array (vinfo, stmt_info, gsi, vec_oprnd, vec_array,
> +                                 i);
>             }
>
>           tree final_mask = NULL;
> @@ -8906,8 +8864,8 @@ vectorizable_store (vec_info *vinfo,
>             final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
>                                              ncopies, vectype, j);
>           if (vec_mask)
> -           final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
> -                                          final_mask, vec_mask, gsi);
> +           final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
> +                                          vec_mask, gsi);
>
>           if (lanes_ifn == IFN_MASK_LEN_STORE_LANES)
>             {
> @@ -8955,8 +8913,7 @@ vectorizable_store (vec_info *vinfo,
>               /* Emit:
>                    MEM_REF[...all elements...] = STORE_LANES (VEC_ARRAY).  */
>               data_ref = create_array_ref (aggr_type, dataref_ptr, ref_type);
> -             call = gimple_build_call_internal (IFN_STORE_LANES, 1,
> -                                                vec_array);
> +             call = gimple_build_call_internal (IFN_STORE_LANES, 1, vec_array);
>               gimple_call_set_lhs (call, data_ref);
>             }
>           gimple_call_set_nothrow (call, true);
> @@ -8965,301 +8922,386 @@ vectorizable_store (vec_info *vinfo,
>
>           /* Record that VEC_ARRAY is now dead.  */
>           vect_clobber_variable (vinfo, stmt_info, gsi, vec_array);
> +         if (j == 0)
> +           *vec_stmt = new_stmt;
> +         STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
>         }
> -      else
> -       {
> -         new_stmt = NULL;
> -         if (grouped_store)
> -           /* Permute.  */
> -           vect_permute_store_chain (vinfo, dr_chain, group_size, stmt_info,
> -                                     gsi, &result_chain);
>
> -         stmt_vec_info next_stmt_info = first_stmt_info;
> -         for (i = 0; i < vec_num; i++)
> -           {
> -             unsigned misalign;
> -             unsigned HOST_WIDE_INT align;
> +      return true;
> +    }
>
> -             tree final_mask = NULL_TREE;
> -             tree final_len = NULL_TREE;
> -             tree bias = NULL_TREE;
> -             if (loop_masks)
> -               final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
> -                                                vec_num * ncopies,
> -                                                vectype, vec_num * j + i);
> -             if (vec_mask)
> -               final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
> -                                              final_mask, vec_mask, gsi);
> +  auto_vec<tree> result_chain (group_size);
> +  auto_vec<tree> vec_offsets;
> +  auto_vec<tree, 1> vec_oprnds;
> +  for (j = 0; j < ncopies; j++)
> +    {
> +      gimple *new_stmt;
> +      if (j == 0)
> +       {
> +         if (slp)
> +           {
> +             /* Get vectorized arguments for SLP_NODE.  */
> +             vect_get_vec_defs (vinfo, stmt_info, slp_node, 1, op,
> +                                &vec_oprnds);
> +             vec_oprnd = vec_oprnds[0];
> +           }
> +         else
> +           {
> +             /* For interleaved stores we collect vectorized defs for all the
> +                stores in the group in DR_CHAIN. DR_CHAIN is then used as an
> +                input to vect_permute_store_chain().
>
> -             if (memory_access_type == VMAT_GATHER_SCATTER
> -                 && gs_info.ifn != IFN_LAST)
> +                If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN
> +                is of size 1.  */
> +             stmt_vec_info next_stmt_info = first_stmt_info;
> +             for (i = 0; i < group_size; i++)
>                 {
> -                 if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> -                   vec_offset = vec_offsets[vec_num * j + i];
> -                 tree scale = size_int (gs_info.scale);
> -
> -                 if (gs_info.ifn == IFN_MASK_LEN_SCATTER_STORE)
> -                   {
> -                     if (loop_lens)
> -                       final_len
> -                         = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
> -                                              vec_num * ncopies, vectype,
> -                                              vec_num * j + i, 1);
> -                     else
> -                       final_len
> -                         = build_int_cst (sizetype,
> -                                          TYPE_VECTOR_SUBPARTS (vectype));
> -                     signed char biasval
> -                       = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
> -                     bias = build_int_cst (intQI_type_node, biasval);
> -                     if (!final_mask)
> -                       {
> -                         mask_vectype = truth_type_for (vectype);
> -                         final_mask = build_minus_one_cst (mask_vectype);
> -                       }
> -                   }
> -
> -                 gcall *call;
> -                 if (final_len && final_mask)
> -                   call
> -                     = gimple_build_call_internal (IFN_MASK_LEN_SCATTER_STORE,
> -                                                   7, dataref_ptr, vec_offset,
> -                                                   scale, vec_oprnd, final_mask,
> -                                                   final_len, bias);
> -                 else if (final_mask)
> -                   call = gimple_build_call_internal
> -                     (IFN_MASK_SCATTER_STORE, 5, dataref_ptr, vec_offset,
> -                      scale, vec_oprnd, final_mask);
> -                 else
> -                   call = gimple_build_call_internal
> -                     (IFN_SCATTER_STORE, 4, dataref_ptr, vec_offset,
> -                      scale, vec_oprnd);
> -                 gimple_call_set_nothrow (call, true);
> -                 vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
> -                 new_stmt = call;
> -                 break;
> +                 /* Since gaps are not supported for interleaved stores,
> +                    DR_GROUP_SIZE is the exact number of stmts in the chain.
> +                    Therefore, NEXT_STMT_INFO can't be NULL_TREE.  In case
> +                    that there is no interleaving, DR_GROUP_SIZE is 1,
> +                    and only one iteration of the loop will be executed.  */
> +                 op = vect_get_store_rhs (next_stmt_info);
> +                 vect_get_vec_defs_for_operand (vinfo, next_stmt_info, ncopies,
> +                                                op, gvec_oprnds[i]);
> +                 vec_oprnd = (*gvec_oprnds[i])[0];
> +                 dr_chain.quick_push (vec_oprnd);
> +                 next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
>                 }
> -             else if (memory_access_type == VMAT_GATHER_SCATTER)
> +             if (mask)
>                 {
> -                 /* Emulated scatter.  */
> -                 gcc_assert (!final_mask);
> -                 unsigned HOST_WIDE_INT const_nunits = nunits.to_constant ();
> -                 unsigned HOST_WIDE_INT const_offset_nunits
> -                   = TYPE_VECTOR_SUBPARTS (gs_info.offset_vectype)
> -                       .to_constant ();
> -                 vec<constructor_elt, va_gc> *ctor_elts;
> -                 vec_alloc (ctor_elts, const_nunits);
> -                 gimple_seq stmts = NULL;
> -                 tree elt_type = TREE_TYPE (vectype);
> -                 unsigned HOST_WIDE_INT elt_size
> -                   = tree_to_uhwi (TYPE_SIZE (elt_type));
> -                 /* We support offset vectors with more elements
> -                    than the data vector for now.  */
> -                 unsigned HOST_WIDE_INT factor
> -                   = const_offset_nunits / const_nunits;
> -                 vec_offset = vec_offsets[j / factor];
> -                 unsigned elt_offset = (j % factor) * const_nunits;
> -                 tree idx_type = TREE_TYPE (TREE_TYPE (vec_offset));
> -                 tree scale = size_int (gs_info.scale);
> -                 align = get_object_alignment (DR_REF (first_dr_info->dr));
> -                 tree ltype = build_aligned_type (TREE_TYPE (vectype), align);
> -                 for (unsigned k = 0; k < const_nunits; ++k)
> -                   {
> -                     /* Compute the offsetted pointer.  */
> -                     tree boff = size_binop (MULT_EXPR, TYPE_SIZE (idx_type),
> -                                             bitsize_int (k + elt_offset));
> -                     tree idx = gimple_build (&stmts, BIT_FIELD_REF,
> -                                              idx_type, vec_offset,
> -                                              TYPE_SIZE (idx_type), boff);
> -                     idx = gimple_convert (&stmts, sizetype, idx);
> -                     idx = gimple_build (&stmts, MULT_EXPR,
> -                                         sizetype, idx, scale);
> -                     tree ptr = gimple_build (&stmts, PLUS_EXPR,
> -                                              TREE_TYPE (dataref_ptr),
> -                                              dataref_ptr, idx);
> -                     ptr = gimple_convert (&stmts, ptr_type_node, ptr);
> -                     /* Extract the element to be stored.  */
> -                     tree elt = gimple_build (&stmts, BIT_FIELD_REF,
> -                                              TREE_TYPE (vectype), vec_oprnd,
> -                                              TYPE_SIZE (elt_type),
> -                                              bitsize_int (k * elt_size));
> -                     gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
> -                     stmts = NULL;
> -                     tree ref = build2 (MEM_REF, ltype, ptr,
> -                                        build_int_cst (ref_type, 0));
> -                     new_stmt = gimple_build_assign (ref, elt);
> -                     vect_finish_stmt_generation (vinfo, stmt_info,
> -                                                  new_stmt, gsi);
> -                   }
> -                 break;
> +                 vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies,
> +                                                mask, &vec_masks,
> +                                                mask_vectype);
> +                 vec_mask = vec_masks[0];
>                 }
> +           }
>
> -             if (i > 0)
> -               /* Bump the vector pointer.  */
> -               dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr,
> -                                              gsi, stmt_info, bump);
> +         /* We should have catched mismatched types earlier.  */
> +         gcc_assert (useless_type_conversion_p (vectype,
> +                                                TREE_TYPE (vec_oprnd)));
> +         bool simd_lane_access_p
> +           = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) != 0;
> +         if (simd_lane_access_p
> +             && !loop_masks
> +             && TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR
> +             && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0))
> +             && integer_zerop (get_dr_vinfo_offset (vinfo, first_dr_info))
> +             && integer_zerop (DR_INIT (first_dr_info->dr))
> +             && alias_sets_conflict_p (get_alias_set (aggr_type),
> +                                       get_alias_set (TREE_TYPE (ref_type))))
> +           {
> +             dataref_ptr = unshare_expr (DR_BASE_ADDRESS (first_dr_info->dr));
> +             dataref_offset = build_int_cst (ref_type, 0);
> +           }
> +         else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> +           vect_get_gather_scatter_ops (loop_vinfo, loop, stmt_info, slp_node,
> +                                        &gs_info, &dataref_ptr, &vec_offsets);
> +         else
> +           dataref_ptr
> +             = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
> +                                         simd_lane_access_p ? loop : NULL,
> +                                         offset, &dummy, gsi, &ptr_incr,
> +                                         simd_lane_access_p, bump);
> +       }
> +      else
> +       {
> +         gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
> +         /* DR_CHAIN is then used as an input to vect_permute_store_chain().
> +            If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN is
> +            of size 1.  */
> +         for (i = 0; i < group_size; i++)
> +           {
> +             vec_oprnd = (*gvec_oprnds[i])[j];
> +             dr_chain[i] = vec_oprnd;
> +           }
> +         if (mask)
> +           vec_mask = vec_masks[j];
> +         if (dataref_offset)
> +           dataref_offset = int_const_binop (PLUS_EXPR, dataref_offset, bump);
> +         else if (!STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> +           dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
> +                                          stmt_info, bump);
> +       }
>
> -             if (slp)
> -               vec_oprnd = vec_oprnds[i];
> -             else if (grouped_store)
> -               /* For grouped stores vectorized defs are interleaved in
> -                  vect_permute_store_chain().  */
> -               vec_oprnd = result_chain[i];
> +      new_stmt = NULL;
> +      if (grouped_store)
> +       /* Permute.  */
> +       vect_permute_store_chain (vinfo, dr_chain, group_size, stmt_info, gsi,
> +                                 &result_chain);
>
> -             align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
> -             if (alignment_support_scheme == dr_aligned)
> -               misalign = 0;
> -             else if (misalignment == DR_MISALIGNMENT_UNKNOWN)
> -               {
> -                 align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
> -                 misalign = 0;
> -               }
> -             else
> -               misalign = misalignment;
> -             if (dataref_offset == NULL_TREE
> -                 && TREE_CODE (dataref_ptr) == SSA_NAME)
> -               set_ptr_info_alignment (get_ptr_info (dataref_ptr), align,
> -                                       misalign);
> -             align = least_bit_hwi (misalign | align);
> -
> -             if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
> -               {
> -                 tree perm_mask = perm_mask_for_reverse (vectype);
> -                 tree perm_dest = vect_create_destination_var
> -                   (vect_get_store_rhs (stmt_info), vectype);
> -                 tree new_temp = make_ssa_name (perm_dest);
> -
> -                 /* Generate the permute statement.  */
> -                 gimple *perm_stmt
> -                   = gimple_build_assign (new_temp, VEC_PERM_EXPR, vec_oprnd,
> -                                          vec_oprnd, perm_mask);
> -                 vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
> -
> -                 perm_stmt = SSA_NAME_DEF_STMT (new_temp);
> -                 vec_oprnd = new_temp;
> -               }
> +      stmt_vec_info next_stmt_info = first_stmt_info;
> +      for (i = 0; i < vec_num; i++)
> +       {
> +         unsigned misalign;
> +         unsigned HOST_WIDE_INT align;
>
> -             /* Compute IFN when LOOP_LENS or final_mask valid.  */
> -             machine_mode vmode = TYPE_MODE (vectype);
> -             machine_mode new_vmode = vmode;
> -             internal_fn partial_ifn = IFN_LAST;
> -             if (loop_lens)
> -               {
> -                 opt_machine_mode new_ovmode
> -                   = get_len_load_store_mode (vmode, false, &partial_ifn);
> -                 new_vmode = new_ovmode.require ();
> -                 unsigned factor
> -                   = (new_ovmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vmode);
> -                 final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
> -                                                vec_num * ncopies, vectype,
> -                                                vec_num * j + i, factor);
> -               }
> -             else if (final_mask)
> -               {
> -                 if (!can_vec_mask_load_store_p (vmode,
> -                                                 TYPE_MODE (TREE_TYPE (final_mask)),
> -                                                 false, &partial_ifn))
> -                   gcc_unreachable ();
> -               }
> +         tree final_mask = NULL_TREE;
> +         tree final_len = NULL_TREE;
> +         tree bias = NULL_TREE;
> +         if (loop_masks)
> +           final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
> +                                            vec_num * ncopies, vectype,
> +                                            vec_num * j + i);
> +         if (vec_mask)
> +           final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
> +                                          vec_mask, gsi);
>
> -             if (partial_ifn == IFN_MASK_LEN_STORE)
> +         if (memory_access_type == VMAT_GATHER_SCATTER
> +             && gs_info.ifn != IFN_LAST)
> +           {
> +             if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> +               vec_offset = vec_offsets[vec_num * j + i];
> +             tree scale = size_int (gs_info.scale);
> +
> +             if (gs_info.ifn == IFN_MASK_LEN_SCATTER_STORE)
>                 {
> -                 if (!final_len)
> -                   {
> -                     /* Pass VF value to 'len' argument of
> -                        MASK_LEN_STORE if LOOP_LENS is invalid.  */
> -                     final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
> -                   }
> +                 if (loop_lens)
> +                   final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
> +                                                  vec_num * ncopies, vectype,
> +                                                  vec_num * j + i, 1);
> +                 else
> +                   final_len = build_int_cst (sizetype,
> +                                              TYPE_VECTOR_SUBPARTS (vectype));
> +                 signed char biasval
> +                   = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
> +                 bias = build_int_cst (intQI_type_node, biasval);
>                   if (!final_mask)
>                     {
> -                     /* Pass all ones value to 'mask' argument of
> -                        MASK_LEN_STORE if final_mask is invalid.  */
>                       mask_vectype = truth_type_for (vectype);
>                       final_mask = build_minus_one_cst (mask_vectype);
>                     }
>                 }
> -             if (final_len)
> -               {
> -                 signed char biasval
> -                   = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
>
> -                 bias = build_int_cst (intQI_type_node, biasval);
> +             gcall *call;
> +             if (final_len && final_mask)
> +               call = gimple_build_call_internal (IFN_MASK_LEN_SCATTER_STORE,
> +                                                  7, dataref_ptr, vec_offset,
> +                                                  scale, vec_oprnd, final_mask,
> +                                                  final_len, bias);
> +             else if (final_mask)
> +               call
> +                 = gimple_build_call_internal (IFN_MASK_SCATTER_STORE, 5,
> +                                               dataref_ptr, vec_offset, scale,
> +                                               vec_oprnd, final_mask);
> +             else
> +               call = gimple_build_call_internal (IFN_SCATTER_STORE, 4,
> +                                                  dataref_ptr, vec_offset,
> +                                                  scale, vec_oprnd);
> +             gimple_call_set_nothrow (call, true);
> +             vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
> +             new_stmt = call;
> +             break;
> +           }
> +         else if (memory_access_type == VMAT_GATHER_SCATTER)
> +           {
> +             /* Emulated scatter.  */
> +             gcc_assert (!final_mask);
> +             unsigned HOST_WIDE_INT const_nunits = nunits.to_constant ();
> +             unsigned HOST_WIDE_INT const_offset_nunits
> +               = TYPE_VECTOR_SUBPARTS (gs_info.offset_vectype).to_constant ();
> +             vec<constructor_elt, va_gc> *ctor_elts;
> +             vec_alloc (ctor_elts, const_nunits);
> +             gimple_seq stmts = NULL;
> +             tree elt_type = TREE_TYPE (vectype);
> +             unsigned HOST_WIDE_INT elt_size
> +               = tree_to_uhwi (TYPE_SIZE (elt_type));
> +             /* We support offset vectors with more elements
> +                than the data vector for now.  */
> +             unsigned HOST_WIDE_INT factor
> +               = const_offset_nunits / const_nunits;
> +             vec_offset = vec_offsets[j / factor];
> +             unsigned elt_offset = (j % factor) * const_nunits;
> +             tree idx_type = TREE_TYPE (TREE_TYPE (vec_offset));
> +             tree scale = size_int (gs_info.scale);
> +             align = get_object_alignment (DR_REF (first_dr_info->dr));
> +             tree ltype = build_aligned_type (TREE_TYPE (vectype), align);
> +             for (unsigned k = 0; k < const_nunits; ++k)
> +               {
> +                 /* Compute the offsetted pointer.  */
> +                 tree boff = size_binop (MULT_EXPR, TYPE_SIZE (idx_type),
> +                                         bitsize_int (k + elt_offset));
> +                 tree idx
> +                   = gimple_build (&stmts, BIT_FIELD_REF, idx_type, vec_offset,
> +                                   TYPE_SIZE (idx_type), boff);
> +                 idx = gimple_convert (&stmts, sizetype, idx);
> +                 idx = gimple_build (&stmts, MULT_EXPR, sizetype, idx, scale);
> +                 tree ptr
> +                   = gimple_build (&stmts, PLUS_EXPR, TREE_TYPE (dataref_ptr),
> +                                   dataref_ptr, idx);
> +                 ptr = gimple_convert (&stmts, ptr_type_node, ptr);
> +                 /* Extract the element to be stored.  */
> +                 tree elt
> +                   = gimple_build (&stmts, BIT_FIELD_REF, TREE_TYPE (vectype),
> +                                   vec_oprnd, TYPE_SIZE (elt_type),
> +                                   bitsize_int (k * elt_size));
> +                 gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
> +                 stmts = NULL;
> +                 tree ref
> +                   = build2 (MEM_REF, ltype, ptr, build_int_cst (ref_type, 0));
> +                 new_stmt = gimple_build_assign (ref, elt);
> +                 vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
>                 }
> +             break;
> +           }
>
> -             /* Arguments are ready.  Create the new vector stmt.  */
> -             if (final_len)
> -               {
> -                 gcall *call;
> -                 tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
> -                 /* Need conversion if it's wrapped with VnQI.  */
> -                 if (vmode != new_vmode)
> -                   {
> -                     tree new_vtype
> -                       = build_vector_type_for_mode (unsigned_intQI_type_node,
> -                                                     new_vmode);
> -                     tree var
> -                       = vect_get_new_ssa_name (new_vtype, vect_simple_var);
> -                     vec_oprnd
> -                       = build1 (VIEW_CONVERT_EXPR, new_vtype, vec_oprnd);
> -                     gassign *new_stmt
> -                       = gimple_build_assign (var, VIEW_CONVERT_EXPR,
> -                                              vec_oprnd);
> -                     vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
> -                                                  gsi);
> -                     vec_oprnd = var;
> -                   }
> +         if (i > 0)
> +           /* Bump the vector pointer.  */
> +           dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
> +                                          stmt_info, bump);
>
> -                 if (partial_ifn == IFN_MASK_LEN_STORE)
> -                   call = gimple_build_call_internal (IFN_MASK_LEN_STORE, 6,
> -                                                      dataref_ptr, ptr,
> -                                                      final_mask, final_len,
> -                                                      bias, vec_oprnd);
> -                 else
> -                   call
> -                     = gimple_build_call_internal (IFN_LEN_STORE, 5,
> -                                                   dataref_ptr, ptr,
> -                                                   final_len, bias,
> -                                                   vec_oprnd);
> -                 gimple_call_set_nothrow (call, true);
> -                 vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
> -                 new_stmt = call;
> +         if (slp)
> +           vec_oprnd = vec_oprnds[i];
> +         else if (grouped_store)
> +           /* For grouped stores vectorized defs are interleaved in
> +              vect_permute_store_chain().  */
> +           vec_oprnd = result_chain[i];
> +
> +         align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
> +         if (alignment_support_scheme == dr_aligned)
> +           misalign = 0;
> +         else if (misalignment == DR_MISALIGNMENT_UNKNOWN)
> +           {
> +             align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
> +             misalign = 0;
> +           }
> +         else
> +           misalign = misalignment;
> +         if (dataref_offset == NULL_TREE
> +             && TREE_CODE (dataref_ptr) == SSA_NAME)
> +           set_ptr_info_alignment (get_ptr_info (dataref_ptr), align,
> +                                   misalign);
> +         align = least_bit_hwi (misalign | align);
> +
> +         if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
> +           {
> +             tree perm_mask = perm_mask_for_reverse (vectype);
> +             tree perm_dest
> +               = vect_create_destination_var (vect_get_store_rhs (stmt_info),
> +                                              vectype);
> +             tree new_temp = make_ssa_name (perm_dest);
> +
> +             /* Generate the permute statement.  */
> +             gimple *perm_stmt
> +               = gimple_build_assign (new_temp, VEC_PERM_EXPR, vec_oprnd,
> +                                      vec_oprnd, perm_mask);
> +             vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
> +
> +             perm_stmt = SSA_NAME_DEF_STMT (new_temp);
> +             vec_oprnd = new_temp;
> +           }
> +
> +         /* Compute IFN when LOOP_LENS or final_mask valid.  */
> +         machine_mode vmode = TYPE_MODE (vectype);
> +         machine_mode new_vmode = vmode;
> +         internal_fn partial_ifn = IFN_LAST;
> +         if (loop_lens)
> +           {
> +             opt_machine_mode new_ovmode
> +               = get_len_load_store_mode (vmode, false, &partial_ifn);
> +             new_vmode = new_ovmode.require ();
> +             unsigned factor
> +               = (new_ovmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vmode);
> +             final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
> +                                            vec_num * ncopies, vectype,
> +                                            vec_num * j + i, factor);
> +           }
> +         else if (final_mask)
> +           {
> +             if (!can_vec_mask_load_store_p (
> +                   vmode, TYPE_MODE (TREE_TYPE (final_mask)), false,
> +                   &partial_ifn))
> +               gcc_unreachable ();
> +           }
> +
> +         if (partial_ifn == IFN_MASK_LEN_STORE)
> +           {
> +             if (!final_len)
> +               {
> +                 /* Pass VF value to 'len' argument of
> +                    MASK_LEN_STORE if LOOP_LENS is invalid.  */
> +                 final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
>                 }
> -             else if (final_mask)
> +             if (!final_mask)
>                 {
> -                 tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
> -                 gcall *call
> -                   = gimple_build_call_internal (IFN_MASK_STORE, 4,
> -                                                 dataref_ptr, ptr,
> -                                                 final_mask, vec_oprnd);
> -                 gimple_call_set_nothrow (call, true);
> -                 vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
> -                 new_stmt = call;
> +                 /* Pass all ones value to 'mask' argument of
> +                    MASK_LEN_STORE if final_mask is invalid.  */
> +                 mask_vectype = truth_type_for (vectype);
> +                 final_mask = build_minus_one_cst (mask_vectype);
>                 }
> -             else
> +           }
> +         if (final_len)
> +           {
> +             signed char biasval
> +               = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
> +
> +             bias = build_int_cst (intQI_type_node, biasval);
> +           }
> +
> +         /* Arguments are ready.  Create the new vector stmt.  */
> +         if (final_len)
> +           {
> +             gcall *call;
> +             tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
> +             /* Need conversion if it's wrapped with VnQI.  */
> +             if (vmode != new_vmode)
>                 {
> -                 data_ref = fold_build2 (MEM_REF, vectype,
> -                                         dataref_ptr,
> -                                         dataref_offset
> -                                         ? dataref_offset
> -                                         : build_int_cst (ref_type, 0));
> -                 if (alignment_support_scheme == dr_aligned)
> -                   ;
> -                 else
> -                   TREE_TYPE (data_ref)
> -                     = build_aligned_type (TREE_TYPE (data_ref),
> -                                           align * BITS_PER_UNIT);
> -                 vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
> -                 new_stmt = gimple_build_assign (data_ref, vec_oprnd);
> +                 tree new_vtype
> +                   = build_vector_type_for_mode (unsigned_intQI_type_node,
> +                                                 new_vmode);
> +                 tree var = vect_get_new_ssa_name (new_vtype, vect_simple_var);
> +                 vec_oprnd = build1 (VIEW_CONVERT_EXPR, new_vtype, vec_oprnd);
> +                 gassign *new_stmt
> +                   = gimple_build_assign (var, VIEW_CONVERT_EXPR, vec_oprnd);
>                   vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
> +                 vec_oprnd = var;
>                 }
>
> -             if (slp)
> -               continue;
> -
> -             next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
> -             if (!next_stmt_info)
> -               break;
> +             if (partial_ifn == IFN_MASK_LEN_STORE)
> +               call = gimple_build_call_internal (IFN_MASK_LEN_STORE, 6,
> +                                                  dataref_ptr, ptr, final_mask,
> +                                                  final_len, bias, vec_oprnd);
> +             else
> +               call = gimple_build_call_internal (IFN_LEN_STORE, 5,
> +                                                  dataref_ptr, ptr, final_len,
> +                                                  bias, vec_oprnd);
> +             gimple_call_set_nothrow (call, true);
> +             vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
> +             new_stmt = call;
> +           }
> +         else if (final_mask)
> +           {
> +             tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
> +             gcall *call
> +               = gimple_build_call_internal (IFN_MASK_STORE, 4, dataref_ptr,
> +                                             ptr, final_mask, vec_oprnd);
> +             gimple_call_set_nothrow (call, true);
> +             vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
> +             new_stmt = call;
> +           }
> +         else
> +           {
> +             data_ref
> +               = fold_build2 (MEM_REF, vectype, dataref_ptr,
> +                              dataref_offset ? dataref_offset
> +                                             : build_int_cst (ref_type, 0));
> +             if (alignment_support_scheme == dr_aligned)
> +               ;
> +             else
> +               TREE_TYPE (data_ref)
> +                 = build_aligned_type (TREE_TYPE (data_ref),
> +                                       align * BITS_PER_UNIT);
> +             vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
> +             new_stmt = gimple_build_assign (data_ref, vec_oprnd);
> +             vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
>             }
> +
> +         if (slp)
> +           continue;
> +
> +         next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
> +         if (!next_stmt_info)
> +           break;
>         }
>        if (!slp)
>         {
> --
> 2.31.1
diff mbox series

Patch

diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index fcaa4127e52..18f5ebcc09c 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -8779,42 +8779,29 @@  vectorizable_store (vec_info *vinfo,
   */

   auto_vec<tree> dr_chain (group_size);
-  auto_vec<tree> result_chain (group_size);
   auto_vec<tree> vec_masks;
   tree vec_mask = NULL;
-  auto_vec<tree> vec_offsets;
   auto_delete_vec<auto_vec<tree>> gvec_oprnds (group_size);
   for (i = 0; i < group_size; i++)
     gvec_oprnds.quick_push (new auto_vec<tree> (ncopies));
-  auto_vec<tree, 1> vec_oprnds;
-  for (j = 0; j < ncopies; j++)
+
+  if (memory_access_type == VMAT_LOAD_STORE_LANES)
     {
-      gimple *new_stmt;
-      if (j == 0)
+      gcc_assert (!slp && grouped_store);
+      for (j = 0; j < ncopies; j++)
 	{
-          if (slp)
-            {
-	      /* Get vectorized arguments for SLP_NODE.  */
-	      vect_get_vec_defs (vinfo, stmt_info, slp_node, 1,
-				 op, &vec_oprnds);
-              vec_oprnd = vec_oprnds[0];
-            }
-          else
-            {
-	      /* For interleaved stores we collect vectorized defs for all the
-		 stores in the group in DR_CHAIN. DR_CHAIN is then used as an
-		 input to vect_permute_store_chain().
-
-		 If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN
-		 is of size 1.  */
+	  gimple *new_stmt;
+	  if (j == 0)
+	    {
+	      /* For interleaved stores we collect vectorized defs for all
+		 the stores in the group in DR_CHAIN. DR_CHAIN is then used
+		 as an input to vect_permute_store_chain().  */
 	      stmt_vec_info next_stmt_info = first_stmt_info;
 	      for (i = 0; i < group_size; i++)
 		{
 		  /* Since gaps are not supported for interleaved stores,
-		     DR_GROUP_SIZE is the exact number of stmts in the chain.
-		     Therefore, NEXT_STMT_INFO can't be NULL_TREE.  In case
-		     that there is no interleaving, DR_GROUP_SIZE is 1,
-		     and only one iteration of the loop will be executed.  */
+		     DR_GROUP_SIZE is the exact number of stmts in the
+		     chain. Therefore, NEXT_STMT_INFO can't be NULL_TREE.  */
 		  op = vect_get_store_rhs (next_stmt_info);
 		  vect_get_vec_defs_for_operand (vinfo, next_stmt_info, ncopies,
 						 op, gvec_oprnds[i]);
@@ -8825,66 +8812,37 @@  vectorizable_store (vec_info *vinfo,
 	      if (mask)
 		{
 		  vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies,
-						 mask, &vec_masks, mask_vectype);
+						 mask, &vec_masks,
+						 mask_vectype);
 		  vec_mask = vec_masks[0];
 		}
-	    }

-	  /* We should have catched mismatched types earlier.  */
-	  gcc_assert (useless_type_conversion_p (vectype,
-						 TREE_TYPE (vec_oprnd)));
-	  bool simd_lane_access_p
-	    = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) != 0;
-	  if (simd_lane_access_p
-	      && !loop_masks
-	      && TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR
-	      && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0))
-	      && integer_zerop (get_dr_vinfo_offset (vinfo, first_dr_info))
-	      && integer_zerop (DR_INIT (first_dr_info->dr))
-	      && alias_sets_conflict_p (get_alias_set (aggr_type),
-					get_alias_set (TREE_TYPE (ref_type))))
-	    {
-	      dataref_ptr = unshare_expr (DR_BASE_ADDRESS (first_dr_info->dr));
-	      dataref_offset = build_int_cst (ref_type, 0);
+	      /* We should have catched mismatched types earlier.  */
+	      gcc_assert (
+		useless_type_conversion_p (vectype, TREE_TYPE (vec_oprnd)));
+	      dataref_ptr
+		= vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
+					    NULL, offset, &dummy, gsi,
+					    &ptr_incr, false, bump);
 	    }
-	  else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
-	    vect_get_gather_scatter_ops (loop_vinfo, loop, stmt_info,
-					 slp_node, &gs_info, &dataref_ptr,
-					 &vec_offsets);
 	  else
-	    dataref_ptr
-	      = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
-					  simd_lane_access_p ? loop : NULL,
-					  offset, &dummy, gsi, &ptr_incr,
-					  simd_lane_access_p, bump);
-	}
-      else
-	{
-	  gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
-	  /* DR_CHAIN is then used as an input to vect_permute_store_chain().
-	     If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN is
-	     of size 1.  */
-	  for (i = 0; i < group_size; i++)
 	    {
-	      vec_oprnd = (*gvec_oprnds[i])[j];
-	      dr_chain[i] = vec_oprnd;
+	      gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
+	      /* DR_CHAIN is then used as an input to
+		 vect_permute_store_chain().  */
+	      for (i = 0; i < group_size; i++)
+		{
+		  vec_oprnd = (*gvec_oprnds[i])[j];
+		  dr_chain[i] = vec_oprnd;
+		}
+	      if (mask)
+		vec_mask = vec_masks[j];
+	      dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
+					     stmt_info, bump);
 	    }
-	  if (mask)
-	    vec_mask = vec_masks[j];
-	  if (dataref_offset)
-	    dataref_offset
-	      = int_const_binop (PLUS_EXPR, dataref_offset, bump);
-	  else if (!STMT_VINFO_GATHER_SCATTER_P (stmt_info))
-	    dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
-					   stmt_info, bump);
-	}
-
-      if (memory_access_type == VMAT_LOAD_STORE_LANES)
-	{
-	  tree vec_array;

 	  /* Get an array into which we can store the individual vectors.  */
-	  vec_array = create_vector_array (vectype, vec_num);
+	  tree vec_array = create_vector_array (vectype, vec_num);

 	  /* Invalidate the current contents of VEC_ARRAY.  This should
 	     become an RTL clobber too, which prevents the vector registers
@@ -8895,8 +8853,8 @@  vectorizable_store (vec_info *vinfo,
 	  for (i = 0; i < vec_num; i++)
 	    {
 	      vec_oprnd = dr_chain[i];
-	      write_vector_array (vinfo, stmt_info,
-				  gsi, vec_oprnd, vec_array, i);
+	      write_vector_array (vinfo, stmt_info, gsi, vec_oprnd, vec_array,
+				  i);
 	    }

 	  tree final_mask = NULL;
@@ -8906,8 +8864,8 @@  vectorizable_store (vec_info *vinfo,
 	    final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
 					     ncopies, vectype, j);
 	  if (vec_mask)
-	    final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
-					   final_mask, vec_mask, gsi);
+	    final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
+					   vec_mask, gsi);

 	  if (lanes_ifn == IFN_MASK_LEN_STORE_LANES)
 	    {
@@ -8955,8 +8913,7 @@  vectorizable_store (vec_info *vinfo,
 	      /* Emit:
 		   MEM_REF[...all elements...] = STORE_LANES (VEC_ARRAY).  */
 	      data_ref = create_array_ref (aggr_type, dataref_ptr, ref_type);
-	      call = gimple_build_call_internal (IFN_STORE_LANES, 1,
-						 vec_array);
+	      call = gimple_build_call_internal (IFN_STORE_LANES, 1, vec_array);
 	      gimple_call_set_lhs (call, data_ref);
 	    }
 	  gimple_call_set_nothrow (call, true);
@@ -8965,301 +8922,386 @@  vectorizable_store (vec_info *vinfo,

 	  /* Record that VEC_ARRAY is now dead.  */
 	  vect_clobber_variable (vinfo, stmt_info, gsi, vec_array);
+	  if (j == 0)
+	    *vec_stmt = new_stmt;
+	  STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
 	}
-      else
-	{
-	  new_stmt = NULL;
-	  if (grouped_store)
-	    /* Permute.  */
-	    vect_permute_store_chain (vinfo, dr_chain, group_size, stmt_info,
-				      gsi, &result_chain);

-	  stmt_vec_info next_stmt_info = first_stmt_info;
-	  for (i = 0; i < vec_num; i++)
-	    {
-	      unsigned misalign;
-	      unsigned HOST_WIDE_INT align;
+      return true;
+    }

-	      tree final_mask = NULL_TREE;
-	      tree final_len = NULL_TREE;
-	      tree bias = NULL_TREE;
-	      if (loop_masks)
-		final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
-						 vec_num * ncopies,
-						 vectype, vec_num * j + i);
-	      if (vec_mask)
-		final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
-					       final_mask, vec_mask, gsi);
+  auto_vec<tree> result_chain (group_size);
+  auto_vec<tree> vec_offsets;
+  auto_vec<tree, 1> vec_oprnds;
+  for (j = 0; j < ncopies; j++)
+    {
+      gimple *new_stmt;
+      if (j == 0)
+	{
+	  if (slp)
+	    {
+	      /* Get vectorized arguments for SLP_NODE.  */
+	      vect_get_vec_defs (vinfo, stmt_info, slp_node, 1, op,
+				 &vec_oprnds);
+	      vec_oprnd = vec_oprnds[0];
+	    }
+	  else
+	    {
+	      /* For interleaved stores we collect vectorized defs for all the
+		 stores in the group in DR_CHAIN. DR_CHAIN is then used as an
+		 input to vect_permute_store_chain().

-	      if (memory_access_type == VMAT_GATHER_SCATTER
-		  && gs_info.ifn != IFN_LAST)
+		 If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN
+		 is of size 1.  */
+	      stmt_vec_info next_stmt_info = first_stmt_info;
+	      for (i = 0; i < group_size; i++)
 		{
-		  if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
-		    vec_offset = vec_offsets[vec_num * j + i];
-		  tree scale = size_int (gs_info.scale);
-
-		  if (gs_info.ifn == IFN_MASK_LEN_SCATTER_STORE)
-		    {
-		      if (loop_lens)
-			final_len
-			  = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
-					       vec_num * ncopies, vectype,
-					       vec_num * j + i, 1);
-		      else
-			final_len
-			  = build_int_cst (sizetype,
-					   TYPE_VECTOR_SUBPARTS (vectype));
-		      signed char biasval
-			= LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
-		      bias = build_int_cst (intQI_type_node, biasval);
-		      if (!final_mask)
-			{
-			  mask_vectype = truth_type_for (vectype);
-			  final_mask = build_minus_one_cst (mask_vectype);
-			}
-		    }
-
-		  gcall *call;
-		  if (final_len && final_mask)
-		    call
-		      = gimple_build_call_internal (IFN_MASK_LEN_SCATTER_STORE,
-						    7, dataref_ptr, vec_offset,
-						    scale, vec_oprnd, final_mask,
-						    final_len, bias);
-		  else if (final_mask)
-		    call = gimple_build_call_internal
-		      (IFN_MASK_SCATTER_STORE, 5, dataref_ptr, vec_offset,
-		       scale, vec_oprnd, final_mask);
-		  else
-		    call = gimple_build_call_internal
-		      (IFN_SCATTER_STORE, 4, dataref_ptr, vec_offset,
-		       scale, vec_oprnd);
-		  gimple_call_set_nothrow (call, true);
-		  vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
-		  new_stmt = call;
-		  break;
+		  /* Since gaps are not supported for interleaved stores,
+		     DR_GROUP_SIZE is the exact number of stmts in the chain.
+		     Therefore, NEXT_STMT_INFO can't be NULL_TREE.  In case
+		     that there is no interleaving, DR_GROUP_SIZE is 1,
+		     and only one iteration of the loop will be executed.  */
+		  op = vect_get_store_rhs (next_stmt_info);
+		  vect_get_vec_defs_for_operand (vinfo, next_stmt_info, ncopies,
+						 op, gvec_oprnds[i]);
+		  vec_oprnd = (*gvec_oprnds[i])[0];
+		  dr_chain.quick_push (vec_oprnd);
+		  next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
 		}
-	      else if (memory_access_type == VMAT_GATHER_SCATTER)
+	      if (mask)
 		{
-		  /* Emulated scatter.  */
-		  gcc_assert (!final_mask);
-		  unsigned HOST_WIDE_INT const_nunits = nunits.to_constant ();
-		  unsigned HOST_WIDE_INT const_offset_nunits
-		    = TYPE_VECTOR_SUBPARTS (gs_info.offset_vectype)
-			.to_constant ();
-		  vec<constructor_elt, va_gc> *ctor_elts;
-		  vec_alloc (ctor_elts, const_nunits);
-		  gimple_seq stmts = NULL;
-		  tree elt_type = TREE_TYPE (vectype);
-		  unsigned HOST_WIDE_INT elt_size
-		    = tree_to_uhwi (TYPE_SIZE (elt_type));
-		  /* We support offset vectors with more elements
-		     than the data vector for now.  */
-		  unsigned HOST_WIDE_INT factor
-		    = const_offset_nunits / const_nunits;
-		  vec_offset = vec_offsets[j / factor];
-		  unsigned elt_offset = (j % factor) * const_nunits;
-		  tree idx_type = TREE_TYPE (TREE_TYPE (vec_offset));
-		  tree scale = size_int (gs_info.scale);
-		  align = get_object_alignment (DR_REF (first_dr_info->dr));
-		  tree ltype = build_aligned_type (TREE_TYPE (vectype), align);
-		  for (unsigned k = 0; k < const_nunits; ++k)
-		    {
-		      /* Compute the offsetted pointer.  */
-		      tree boff = size_binop (MULT_EXPR, TYPE_SIZE (idx_type),
-					      bitsize_int (k + elt_offset));
-		      tree idx = gimple_build (&stmts, BIT_FIELD_REF,
-					       idx_type, vec_offset,
-					       TYPE_SIZE (idx_type), boff);
-		      idx = gimple_convert (&stmts, sizetype, idx);
-		      idx = gimple_build (&stmts, MULT_EXPR,
-					  sizetype, idx, scale);
-		      tree ptr = gimple_build (&stmts, PLUS_EXPR,
-					       TREE_TYPE (dataref_ptr),
-					       dataref_ptr, idx);
-		      ptr = gimple_convert (&stmts, ptr_type_node, ptr);
-		      /* Extract the element to be stored.  */
-		      tree elt = gimple_build (&stmts, BIT_FIELD_REF,
-					       TREE_TYPE (vectype), vec_oprnd,
-					       TYPE_SIZE (elt_type),
-					       bitsize_int (k * elt_size));
-		      gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
-		      stmts = NULL;
-		      tree ref = build2 (MEM_REF, ltype, ptr,
-					 build_int_cst (ref_type, 0));
-		      new_stmt = gimple_build_assign (ref, elt);
-		      vect_finish_stmt_generation (vinfo, stmt_info,
-						   new_stmt, gsi);
-		    }
-		  break;
+		  vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies,
+						 mask, &vec_masks,
+						 mask_vectype);
+		  vec_mask = vec_masks[0];
 		}
+	    }

-	      if (i > 0)
-		/* Bump the vector pointer.  */
-		dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr,
-					       gsi, stmt_info, bump);
+	  /* We should have catched mismatched types earlier.  */
+	  gcc_assert (useless_type_conversion_p (vectype,
+						 TREE_TYPE (vec_oprnd)));
+	  bool simd_lane_access_p
+	    = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) != 0;
+	  if (simd_lane_access_p
+	      && !loop_masks
+	      && TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR
+	      && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0))
+	      && integer_zerop (get_dr_vinfo_offset (vinfo, first_dr_info))
+	      && integer_zerop (DR_INIT (first_dr_info->dr))
+	      && alias_sets_conflict_p (get_alias_set (aggr_type),
+					get_alias_set (TREE_TYPE (ref_type))))
+	    {
+	      dataref_ptr = unshare_expr (DR_BASE_ADDRESS (first_dr_info->dr));
+	      dataref_offset = build_int_cst (ref_type, 0);
+	    }
+	  else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
+	    vect_get_gather_scatter_ops (loop_vinfo, loop, stmt_info, slp_node,
+					 &gs_info, &dataref_ptr, &vec_offsets);
+	  else
+	    dataref_ptr
+	      = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
+					  simd_lane_access_p ? loop : NULL,
+					  offset, &dummy, gsi, &ptr_incr,
+					  simd_lane_access_p, bump);
+	}
+      else
+	{
+	  gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
+	  /* DR_CHAIN is then used as an input to vect_permute_store_chain().
+	     If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN is
+	     of size 1.  */
+	  for (i = 0; i < group_size; i++)
+	    {
+	      vec_oprnd = (*gvec_oprnds[i])[j];
+	      dr_chain[i] = vec_oprnd;
+	    }
+	  if (mask)
+	    vec_mask = vec_masks[j];
+	  if (dataref_offset)
+	    dataref_offset = int_const_binop (PLUS_EXPR, dataref_offset, bump);
+	  else if (!STMT_VINFO_GATHER_SCATTER_P (stmt_info))
+	    dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
+					   stmt_info, bump);
+	}

-	      if (slp)
-		vec_oprnd = vec_oprnds[i];
-	      else if (grouped_store)
-		/* For grouped stores vectorized defs are interleaved in
-		   vect_permute_store_chain().  */
-		vec_oprnd = result_chain[i];
+      new_stmt = NULL;
+      if (grouped_store)
+	/* Permute.  */
+	vect_permute_store_chain (vinfo, dr_chain, group_size, stmt_info, gsi,
+				  &result_chain);

-	      align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
-	      if (alignment_support_scheme == dr_aligned)
-		misalign = 0;
-	      else if (misalignment == DR_MISALIGNMENT_UNKNOWN)
-		{
-		  align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
-		  misalign = 0;
-		}
-	      else
-		misalign = misalignment;
-	      if (dataref_offset == NULL_TREE
-		  && TREE_CODE (dataref_ptr) == SSA_NAME)
-		set_ptr_info_alignment (get_ptr_info (dataref_ptr), align,
-					misalign);
-	      align = least_bit_hwi (misalign | align);
-
-	      if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
-		{
-		  tree perm_mask = perm_mask_for_reverse (vectype);
-		  tree perm_dest = vect_create_destination_var
-		    (vect_get_store_rhs (stmt_info), vectype);
-		  tree new_temp = make_ssa_name (perm_dest);
-
-		  /* Generate the permute statement.  */
-		  gimple *perm_stmt
-		    = gimple_build_assign (new_temp, VEC_PERM_EXPR, vec_oprnd,
-					   vec_oprnd, perm_mask);
-		  vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-
-		  perm_stmt = SSA_NAME_DEF_STMT (new_temp);
-		  vec_oprnd = new_temp;
-		}
+      stmt_vec_info next_stmt_info = first_stmt_info;
+      for (i = 0; i < vec_num; i++)
+	{
+	  unsigned misalign;
+	  unsigned HOST_WIDE_INT align;

-	      /* Compute IFN when LOOP_LENS or final_mask valid.  */
-	      machine_mode vmode = TYPE_MODE (vectype);
-	      machine_mode new_vmode = vmode;
-	      internal_fn partial_ifn = IFN_LAST;
-	      if (loop_lens)
-		{
-		  opt_machine_mode new_ovmode
-		    = get_len_load_store_mode (vmode, false, &partial_ifn);
-		  new_vmode = new_ovmode.require ();
-		  unsigned factor
-		    = (new_ovmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vmode);
-		  final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
-						 vec_num * ncopies, vectype,
-						 vec_num * j + i, factor);
-		}
-	      else if (final_mask)
-		{
-		  if (!can_vec_mask_load_store_p (vmode,
-						  TYPE_MODE (TREE_TYPE (final_mask)),
-						  false, &partial_ifn))
-		    gcc_unreachable ();
-		}
+	  tree final_mask = NULL_TREE;
+	  tree final_len = NULL_TREE;
+	  tree bias = NULL_TREE;
+	  if (loop_masks)
+	    final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
+					     vec_num * ncopies, vectype,
+					     vec_num * j + i);
+	  if (vec_mask)
+	    final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
+					   vec_mask, gsi);

-	      if (partial_ifn == IFN_MASK_LEN_STORE)
+	  if (memory_access_type == VMAT_GATHER_SCATTER
+	      && gs_info.ifn != IFN_LAST)
+	    {
+	      if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
+		vec_offset = vec_offsets[vec_num * j + i];
+	      tree scale = size_int (gs_info.scale);
+
+	      if (gs_info.ifn == IFN_MASK_LEN_SCATTER_STORE)
 		{
-		  if (!final_len)
-		    {
-		      /* Pass VF value to 'len' argument of
-			 MASK_LEN_STORE if LOOP_LENS is invalid.  */
-		      final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
-		    }
+		  if (loop_lens)
+		    final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
+						   vec_num * ncopies, vectype,
+						   vec_num * j + i, 1);
+		  else
+		    final_len = build_int_cst (sizetype,
+					       TYPE_VECTOR_SUBPARTS (vectype));
+		  signed char biasval
+		    = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
+		  bias = build_int_cst (intQI_type_node, biasval);
 		  if (!final_mask)
 		    {
-		      /* Pass all ones value to 'mask' argument of
-			 MASK_LEN_STORE if final_mask is invalid.  */
 		      mask_vectype = truth_type_for (vectype);
 		      final_mask = build_minus_one_cst (mask_vectype);
 		    }
 		}
-	      if (final_len)
-		{
-		  signed char biasval
-		    = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);

-		  bias = build_int_cst (intQI_type_node, biasval);
+	      gcall *call;
+	      if (final_len && final_mask)
+		call = gimple_build_call_internal (IFN_MASK_LEN_SCATTER_STORE,
+						   7, dataref_ptr, vec_offset,
+						   scale, vec_oprnd, final_mask,
+						   final_len, bias);
+	      else if (final_mask)
+		call
+		  = gimple_build_call_internal (IFN_MASK_SCATTER_STORE, 5,
+						dataref_ptr, vec_offset, scale,
+						vec_oprnd, final_mask);
+	      else
+		call = gimple_build_call_internal (IFN_SCATTER_STORE, 4,
+						   dataref_ptr, vec_offset,
+						   scale, vec_oprnd);
+	      gimple_call_set_nothrow (call, true);
+	      vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
+	      new_stmt = call;
+	      break;
+	    }
+	  else if (memory_access_type == VMAT_GATHER_SCATTER)
+	    {
+	      /* Emulated scatter.  */
+	      gcc_assert (!final_mask);
+	      unsigned HOST_WIDE_INT const_nunits = nunits.to_constant ();
+	      unsigned HOST_WIDE_INT const_offset_nunits
+		= TYPE_VECTOR_SUBPARTS (gs_info.offset_vectype).to_constant ();
+	      vec<constructor_elt, va_gc> *ctor_elts;
+	      vec_alloc (ctor_elts, const_nunits);
+	      gimple_seq stmts = NULL;
+	      tree elt_type = TREE_TYPE (vectype);
+	      unsigned HOST_WIDE_INT elt_size
+		= tree_to_uhwi (TYPE_SIZE (elt_type));
+	      /* We support offset vectors with more elements
+		 than the data vector for now.  */
+	      unsigned HOST_WIDE_INT factor
+		= const_offset_nunits / const_nunits;
+	      vec_offset = vec_offsets[j / factor];
+	      unsigned elt_offset = (j % factor) * const_nunits;
+	      tree idx_type = TREE_TYPE (TREE_TYPE (vec_offset));
+	      tree scale = size_int (gs_info.scale);
+	      align = get_object_alignment (DR_REF (first_dr_info->dr));
+	      tree ltype = build_aligned_type (TREE_TYPE (vectype), align);
+	      for (unsigned k = 0; k < const_nunits; ++k)
+		{
+		  /* Compute the offsetted pointer.  */
+		  tree boff = size_binop (MULT_EXPR, TYPE_SIZE (idx_type),
+					  bitsize_int (k + elt_offset));
+		  tree idx
+		    = gimple_build (&stmts, BIT_FIELD_REF, idx_type, vec_offset,
+				    TYPE_SIZE (idx_type), boff);
+		  idx = gimple_convert (&stmts, sizetype, idx);
+		  idx = gimple_build (&stmts, MULT_EXPR, sizetype, idx, scale);
+		  tree ptr
+		    = gimple_build (&stmts, PLUS_EXPR, TREE_TYPE (dataref_ptr),
+				    dataref_ptr, idx);
+		  ptr = gimple_convert (&stmts, ptr_type_node, ptr);
+		  /* Extract the element to be stored.  */
+		  tree elt
+		    = gimple_build (&stmts, BIT_FIELD_REF, TREE_TYPE (vectype),
+				    vec_oprnd, TYPE_SIZE (elt_type),
+				    bitsize_int (k * elt_size));
+		  gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
+		  stmts = NULL;
+		  tree ref
+		    = build2 (MEM_REF, ltype, ptr, build_int_cst (ref_type, 0));
+		  new_stmt = gimple_build_assign (ref, elt);
+		  vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
 		}
+	      break;
+	    }

-	      /* Arguments are ready.  Create the new vector stmt.  */
-	      if (final_len)
-		{
-		  gcall *call;
-		  tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
-		  /* Need conversion if it's wrapped with VnQI.  */
-		  if (vmode != new_vmode)
-		    {
-		      tree new_vtype
-			= build_vector_type_for_mode (unsigned_intQI_type_node,
-						      new_vmode);
-		      tree var
-			= vect_get_new_ssa_name (new_vtype, vect_simple_var);
-		      vec_oprnd
-			= build1 (VIEW_CONVERT_EXPR, new_vtype, vec_oprnd);
-		      gassign *new_stmt
-			= gimple_build_assign (var, VIEW_CONVERT_EXPR,
-					       vec_oprnd);
-		      vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
-						   gsi);
-		      vec_oprnd = var;
-		    }
+	  if (i > 0)
+	    /* Bump the vector pointer.  */
+	    dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
+					   stmt_info, bump);

-		  if (partial_ifn == IFN_MASK_LEN_STORE)
-		    call = gimple_build_call_internal (IFN_MASK_LEN_STORE, 6,
-						       dataref_ptr, ptr,
-						       final_mask, final_len,
-						       bias, vec_oprnd);
-		  else
-		    call
-		      = gimple_build_call_internal (IFN_LEN_STORE, 5,
-						    dataref_ptr, ptr,
-						    final_len, bias,
-						    vec_oprnd);
-		  gimple_call_set_nothrow (call, true);
-		  vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
-		  new_stmt = call;
+	  if (slp)
+	    vec_oprnd = vec_oprnds[i];
+	  else if (grouped_store)
+	    /* For grouped stores vectorized defs are interleaved in
+	       vect_permute_store_chain().  */
+	    vec_oprnd = result_chain[i];
+
+	  align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
+	  if (alignment_support_scheme == dr_aligned)
+	    misalign = 0;
+	  else if (misalignment == DR_MISALIGNMENT_UNKNOWN)
+	    {
+	      align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
+	      misalign = 0;
+	    }
+	  else
+	    misalign = misalignment;
+	  if (dataref_offset == NULL_TREE
+	      && TREE_CODE (dataref_ptr) == SSA_NAME)
+	    set_ptr_info_alignment (get_ptr_info (dataref_ptr), align,
+				    misalign);
+	  align = least_bit_hwi (misalign | align);
+
+	  if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
+	    {
+	      tree perm_mask = perm_mask_for_reverse (vectype);
+	      tree perm_dest
+		= vect_create_destination_var (vect_get_store_rhs (stmt_info),
+					       vectype);
+	      tree new_temp = make_ssa_name (perm_dest);
+
+	      /* Generate the permute statement.  */
+	      gimple *perm_stmt
+		= gimple_build_assign (new_temp, VEC_PERM_EXPR, vec_oprnd,
+				       vec_oprnd, perm_mask);
+	      vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
+
+	      perm_stmt = SSA_NAME_DEF_STMT (new_temp);
+	      vec_oprnd = new_temp;
+	    }
+
+	  /* Compute IFN when LOOP_LENS or final_mask valid.  */
+	  machine_mode vmode = TYPE_MODE (vectype);
+	  machine_mode new_vmode = vmode;
+	  internal_fn partial_ifn = IFN_LAST;
+	  if (loop_lens)
+	    {
+	      opt_machine_mode new_ovmode
+		= get_len_load_store_mode (vmode, false, &partial_ifn);
+	      new_vmode = new_ovmode.require ();
+	      unsigned factor
+		= (new_ovmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vmode);
+	      final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
+					     vec_num * ncopies, vectype,
+					     vec_num * j + i, factor);
+	    }
+	  else if (final_mask)
+	    {
+	      if (!can_vec_mask_load_store_p (
+		    vmode, TYPE_MODE (TREE_TYPE (final_mask)), false,
+		    &partial_ifn))
+		gcc_unreachable ();
+	    }
+
+	  if (partial_ifn == IFN_MASK_LEN_STORE)
+	    {
+	      if (!final_len)
+		{
+		  /* Pass VF value to 'len' argument of
+		     MASK_LEN_STORE if LOOP_LENS is invalid.  */
+		  final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
 		}
-	      else if (final_mask)
+	      if (!final_mask)
 		{
-		  tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
-		  gcall *call
-		    = gimple_build_call_internal (IFN_MASK_STORE, 4,
-						  dataref_ptr, ptr,
-						  final_mask, vec_oprnd);
-		  gimple_call_set_nothrow (call, true);
-		  vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
-		  new_stmt = call;
+		  /* Pass all ones value to 'mask' argument of
+		     MASK_LEN_STORE if final_mask is invalid.  */
+		  mask_vectype = truth_type_for (vectype);
+		  final_mask = build_minus_one_cst (mask_vectype);
 		}
-	      else
+	    }
+	  if (final_len)
+	    {
+	      signed char biasval
+		= LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
+
+	      bias = build_int_cst (intQI_type_node, biasval);
+	    }
+
+	  /* Arguments are ready.  Create the new vector stmt.  */
+	  if (final_len)
+	    {
+	      gcall *call;
+	      tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
+	      /* Need conversion if it's wrapped with VnQI.  */
+	      if (vmode != new_vmode)
 		{
-		  data_ref = fold_build2 (MEM_REF, vectype,
-					  dataref_ptr,
-					  dataref_offset
-					  ? dataref_offset
-					  : build_int_cst (ref_type, 0));
-		  if (alignment_support_scheme == dr_aligned)
-		    ;
-		  else
-		    TREE_TYPE (data_ref)
-		      = build_aligned_type (TREE_TYPE (data_ref),
-					    align * BITS_PER_UNIT);
-		  vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
-		  new_stmt = gimple_build_assign (data_ref, vec_oprnd);
+		  tree new_vtype
+		    = build_vector_type_for_mode (unsigned_intQI_type_node,
+						  new_vmode);
+		  tree var = vect_get_new_ssa_name (new_vtype, vect_simple_var);
+		  vec_oprnd = build1 (VIEW_CONVERT_EXPR, new_vtype, vec_oprnd);
+		  gassign *new_stmt
+		    = gimple_build_assign (var, VIEW_CONVERT_EXPR, vec_oprnd);
 		  vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+		  vec_oprnd = var;
 		}

-	      if (slp)
-		continue;
-
-	      next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
-	      if (!next_stmt_info)
-		break;
+	      if (partial_ifn == IFN_MASK_LEN_STORE)
+		call = gimple_build_call_internal (IFN_MASK_LEN_STORE, 6,
+						   dataref_ptr, ptr, final_mask,
+						   final_len, bias, vec_oprnd);
+	      else
+		call = gimple_build_call_internal (IFN_LEN_STORE, 5,
+						   dataref_ptr, ptr, final_len,
+						   bias, vec_oprnd);
+	      gimple_call_set_nothrow (call, true);
+	      vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
+	      new_stmt = call;
+	    }
+	  else if (final_mask)
+	    {
+	      tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
+	      gcall *call
+		= gimple_build_call_internal (IFN_MASK_STORE, 4, dataref_ptr,
+					      ptr, final_mask, vec_oprnd);
+	      gimple_call_set_nothrow (call, true);
+	      vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
+	      new_stmt = call;
+	    }
+	  else
+	    {
+	      data_ref
+		= fold_build2 (MEM_REF, vectype, dataref_ptr,
+			       dataref_offset ? dataref_offset
+					      : build_int_cst (ref_type, 0));
+	      if (alignment_support_scheme == dr_aligned)
+		;
+	      else
+		TREE_TYPE (data_ref)
+		  = build_aligned_type (TREE_TYPE (data_ref),
+					align * BITS_PER_UNIT);
+	      vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
+	      new_stmt = gimple_build_assign (data_ref, vec_oprnd);
+	      vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
 	    }
+
+	  if (slp)
+	    continue;
+
+	  next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
+	  if (!next_stmt_info)
+	    break;
 	}
       if (!slp)
 	{