@@ -2131,7 +2131,10 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi,
ptr, vec_mask, vec_rhs);
vect_finish_stmt_generation (stmt, new_stmt, gsi);
if (i == 0)
- STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
+ {
+ STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
+ STMT_VINFO_FIRST_COPY_P (vinfo_for_stmt (new_stmt)) = true;
+ }
else
STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
prev_stmt_info = vinfo_for_stmt (new_stmt);
@@ -6203,7 +6206,16 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
if (!slp)
{
if (j == 0)
- STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
+ {
+ STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
+ STMT_VINFO_FIRST_COPY_P (vinfo_for_stmt (new_stmt)) = true;
+ /* Original statement is replaced with the first vector one.
+ Keep data reference and original vectype in the first
+ vector copy for masking purposes. */
+ STMT_VINFO_DATA_REF (vinfo_for_stmt (new_stmt))
+ = STMT_VINFO_DATA_REF (stmt_info);
+ STMT_VINFO_VECTYPE (vinfo_for_stmt (new_stmt)) = vectype;
+ }
else
STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
prev_stmt_info = vinfo_for_stmt (new_stmt);
@@ -633,6 +633,10 @@ typedef struct _stmt_vec_info {
/* For both loads and stores. */
bool simd_lane_access_p;
+ /* True for the first vector statement copy when scalar
+ statement is vectorized into several vector ones. */
+ bool first_copy_p;
+
/* For reduction loops, this is the type of reduction. */
enum vect_reduction_type v_reduc_type;
@@ -666,6 +670,7 @@ STMT_VINFO_BB_VINFO (stmt_vec_info stmt_vinfo)
#define STMT_VINFO_GATHER_SCATTER_P(S) (S)->gather_scatter_p
#define STMT_VINFO_STRIDED_P(S) (S)->strided_p
#define STMT_VINFO_SIMD_LANE_ACCESS_P(S) (S)->simd_lane_access_p
+#define STMT_VINFO_FIRST_COPY_P(S) (S)->first_copy_p
#define STMT_VINFO_VEC_REDUCTION_TYPE(S) (S)->v_reduc_type
#define STMT_VINFO_DR_BASE_ADDRESS(S) (S)->dr_base_address