@@ -746,6 +746,7 @@ lower_stmt (gimple_stmt_iterator *gsi, struct lower_data *data)
case GIMPLE_EH_MUST_NOT_THROW:
case GIMPLE_OMP_FOR:
case GIMPLE_OMP_SCOPE:
+ case GIMPLE_OMP_DISPATCH:
case GIMPLE_OMP_SECTIONS:
case GIMPLE_OMP_SECTIONS_SWITCH:
case GIMPLE_OMP_SECTION:
@@ -1726,6 +1726,35 @@ dump_gimple_omp_scope (pretty_printer *pp, const gimple *gs,
}
}
+/* Dump a GIMPLE_OMP_DISPATCH tuple on the pretty_printer BUFFER. */
+
+static void
+dump_gimple_omp_dispatch (pretty_printer *buffer, const gimple *gs, int spc,
+ dump_flags_t flags)
+{
+ if (flags & TDF_RAW)
+ {
+ dump_gimple_fmt (buffer, spc, flags, "%G <%+BODY <%S>%nCLAUSES <", gs,
+ gimple_omp_body (gs));
+ dump_omp_clauses (buffer, gimple_omp_dispatch_clauses (gs), spc, flags);
+ dump_gimple_fmt (buffer, spc, flags, " >");
+ }
+ else
+ {
+ pp_string (buffer, "#pragma omp dispatch");
+ dump_omp_clauses (buffer, gimple_omp_dispatch_clauses (gs), spc, flags);
+ if (!gimple_seq_empty_p (gimple_omp_body (gs)))
+ {
+ newline_and_indent (buffer, spc + 2);
+ pp_left_brace (buffer);
+ pp_newline (buffer);
+ dump_gimple_seq (buffer, gimple_omp_body (gs), spc + 4, flags);
+ newline_and_indent (buffer, spc + 2);
+ pp_right_brace (buffer);
+ }
+ }
+}
+
/* Dump a GIMPLE_OMP_TARGET tuple on the pretty_printer PP. */
static void
@@ -2805,6 +2834,10 @@ pp_gimple_stmt_1 (pretty_printer *pp, const gimple *gs, int spc,
dump_gimple_omp_scope (pp, gs, spc, flags);
break;
+ case GIMPLE_OMP_DISPATCH:
+ dump_gimple_omp_dispatch(pp, gs, spc, flags);
+ break;
+
case GIMPLE_OMP_MASTER:
case GIMPLE_OMP_SECTION:
case GIMPLE_OMP_STRUCTURED_BLOCK:
@@ -707,6 +707,7 @@ walk_gimple_stmt (gimple_stmt_iterator *gsi, walk_stmt_fn callback_stmt,
case GIMPLE_OMP_PARALLEL:
case GIMPLE_OMP_TASK:
case GIMPLE_OMP_SCOPE:
+ case GIMPLE_OMP_DISPATCH:
case GIMPLE_OMP_SECTIONS:
case GIMPLE_OMP_SINGLE:
case GIMPLE_OMP_TARGET:
@@ -1235,6 +1235,21 @@ gimple_build_omp_scope (gimple_seq body, tree clauses)
return p;
}
+/* Build a GIMPLE_OMP_DISPATCH statement.
+
+ BODY is the target function call to be dispatched.
+ CLAUSES are any of the OMP dispatch construct's clauses: ... */
+
+gimple *
+gimple_build_omp_dispatch (gimple_seq body, tree clauses)
+{
+ gimple *p = gimple_alloc (GIMPLE_OMP_DISPATCH, 0);
+ gimple_omp_dispatch_set_clauses (p, clauses);
+ if (body)
+ gimple_omp_set_body (p, body);
+
+ return p;
+}
/* Build a GIMPLE_OMP_TARGET statement.
@@ -2148,6 +2163,11 @@ gimple_copy (gimple *stmt)
gimple_omp_scope_set_clauses (copy, t);
goto copy_omp_body;
+ case GIMPLE_OMP_DISPATCH:
+ t = unshare_expr (gimple_omp_dispatch_clauses (stmt));
+ gimple_omp_dispatch_set_clauses (copy, t);
+ goto copy_omp_body;
+
case GIMPLE_OMP_TARGET:
{
gomp_target *omp_target_stmt = as_a <gomp_target *> (stmt);
@@ -350,6 +350,11 @@ DEFGSCODE(GIMPLE_OMP_SCAN, "gimple_omp_scan", GSS_OMP_SINGLE_LAYOUT)
CLAUSES is an OMP_CLAUSE chain holding the associated clauses. */
DEFGSCODE(GIMPLE_OMP_SCOPE, "gimple_omp_scope", GSS_OMP_SINGLE_LAYOUT)
+/* GIMPLE_OMP_DISPATCH <BODY, CLAUSES> represents #pragma omp dispatch
+ BODY is the target function call to be dispatched.
+ CLAUSES is an OMP_CLAUSE chain holding the associated clauses. */
+DEFGSCODE(GIMPLE_OMP_DISPATCH, "gimple_omp_dispatch", GSS_OMP_SINGLE_LAYOUT)
+
/* OMP_SECTION <BODY> represents #pragma omp section.
BODY is the sequence of statements in the section body. */
DEFGSCODE(GIMPLE_OMP_SECTION, "gimple_omp_section", GSS_OMP)
@@ -742,7 +742,7 @@ struct GTY((tag("GSS_OMP_CONTINUE")))
};
/* GIMPLE_OMP_SINGLE, GIMPLE_OMP_ORDERED, GIMPLE_OMP_TASKGROUP,
- GIMPLE_OMP_SCAN, GIMPLE_OMP_MASKED, GIMPLE_OMP_SCOPE. */
+ GIMPLE_OMP_SCAN, GIMPLE_OMP_MASKED, GIMPLE_OMP_SCOPE, GIMPLE_OMP_DISPATCH. */
struct GTY((tag("GSS_OMP_SINGLE_LAYOUT")))
gimple_statement_omp_single_layout : public gimple_statement_omp
@@ -1591,6 +1591,7 @@ gomp_task *gimple_build_omp_task (gimple_seq, tree, tree, tree, tree,
gimple *gimple_build_omp_section (gimple_seq);
gimple *gimple_build_omp_structured_block (gimple_seq);
gimple *gimple_build_omp_scope (gimple_seq, tree);
+gimple *gimple_build_omp_dispatch (gimple_seq, tree);
gimple *gimple_build_omp_master (gimple_seq);
gimple *gimple_build_omp_masked (gimple_seq, tree);
gimple *gimple_build_omp_taskgroup (gimple_seq, tree);
@@ -1882,6 +1883,7 @@ gimple_has_substatements (gimple *g)
case GIMPLE_OMP_PARALLEL:
case GIMPLE_OMP_TASK:
case GIMPLE_OMP_SCOPE:
+ case GIMPLE_OMP_DISPATCH:
case GIMPLE_OMP_SECTIONS:
case GIMPLE_OMP_SINGLE:
case GIMPLE_OMP_TARGET:
@@ -5434,6 +5436,34 @@ gimple_omp_scope_set_clauses (gimple *gs, tree clauses)
= clauses;
}
+/* Return the clauses associated with OMP_DISPATCH statement GS. */
+
+inline tree
+gimple_omp_dispatch_clauses (const gimple *gs)
+{
+ GIMPLE_CHECK (gs, GIMPLE_OMP_DISPATCH);
+ return static_cast<const gimple_statement_omp_single_layout *> (gs)->clauses;
+}
+
+/* Return a pointer to the clauses associated with OMP dispatch statement
+ GS. */
+
+inline tree *
+gimple_omp_dispatch_clauses_ptr (gimple *gs)
+{
+ GIMPLE_CHECK (gs, GIMPLE_OMP_DISPATCH);
+ return &static_cast<gimple_statement_omp_single_layout *> (gs)->clauses;
+}
+
+/* Set CLAUSES to be the clauses associated with OMP dispatch statement
+ GS. */
+
+inline void
+gimple_omp_dispatch_set_clauses (gimple *gs, tree clauses)
+{
+ GIMPLE_CHECK (gs, GIMPLE_OMP_DISPATCH);
+ static_cast<gimple_statement_omp_single_layout *> (gs)->clauses = clauses;
+}
/* Return the kind of the OMP_FOR statemement G. */
@@ -6768,6 +6798,7 @@ gimple_return_set_retval (greturn *gs, tree retval)
case GIMPLE_OMP_TARGET: \
case GIMPLE_OMP_TEAMS: \
case GIMPLE_OMP_SCOPE: \
+ case GIMPLE_OMP_DISPATCH: \
case GIMPLE_OMP_SECTION: \
case GIMPLE_OMP_STRUCTURED_BLOCK: \
case GIMPLE_OMP_MASTER: \
@@ -161,7 +161,8 @@ enum omp_region_type
{
ORT_WORKSHARE = 0x00,
ORT_TASKGROUP = 0x01,
- ORT_SIMD = 0x04,
+ ORT_DISPATCH = 0x02,
+ ORT_SIMD = 0x04,
ORT_PARALLEL = 0x08,
ORT_COMBINED_PARALLEL = ORT_PARALLEL | 1,
@@ -258,6 +259,7 @@ struct gimplify_omp_ctx
bool order_concurrent;
bool has_depend;
bool in_for_exprs;
+ bool in_call_args;
int defaultmap[5];
};
@@ -4067,23 +4069,125 @@ gimplify_call_expr (tree *expr_p, gimple_seq *pre_p, bool want_value)
/* Gimplify the function arguments. */
if (nargs > 0)
{
+ tree device_num = NULL_TREE;
for (i = (PUSH_ARGS_REVERSED ? nargs - 1 : 0);
- PUSH_ARGS_REVERSED ? i >= 0 : i < nargs;
- PUSH_ARGS_REVERSED ? i-- : i++)
- {
- enum gimplify_status t;
+ PUSH_ARGS_REVERSED ? i >= 0 : i < nargs;
+ PUSH_ARGS_REVERSED ? i-- : i++)
+ {
+ enum gimplify_status t;
- /* Avoid gimplifying the second argument to va_start, which needs to
- be the plain PARM_DECL. */
- if ((i != 1) || !builtin_va_start_p)
- {
- t = gimplify_arg (&CALL_EXPR_ARG (*expr_p, i), pre_p,
- EXPR_LOCATION (*expr_p), ! returns_twice);
+ /* Avoid gimplifying the second argument to va_start, which needs to
+ be the plain PARM_DECL. */
+ if ((i != 1) || !builtin_va_start_p)
+ {
+ tree *arg_p = &CALL_EXPR_ARG (*expr_p, i);
+ tree adjust_args_list;
+ if (flag_openmp && EXPR_P (CALL_EXPR_FN (*expr_p))
+ && DECL_P (TREE_OPERAND (CALL_EXPR_FN (*expr_p), 0))
+ && (adjust_args_list = lookup_attribute (
+ "omp declare variant variant adjust_args",
+ DECL_ATTRIBUTES (
+ TREE_OPERAND (CALL_EXPR_FN (*expr_p), 0))))
+ != NULL_TREE
+ && gimplify_omp_ctxp != NULL
+ && gimplify_omp_ctxp->code == OMP_DISPATCH
+ && !gimplify_omp_ctxp->in_call_args)
+ {
+ tree param
+ = DECL_ARGUMENTS (TREE_OPERAND (CALL_EXPR_FN (*expr_p), 0));
- if (t == GS_ERROR)
- ret = GS_ERROR;
- }
- }
+ if (param != NULL_TREE)
+ {
+ for (int param_idx = 0; param_idx < i; param_idx++)
+ param = TREE_CHAIN (param);
+
+ bool is_device_ptr = false;
+ for (tree c = gimplify_omp_ctxp->clauses; c;
+ c = TREE_CHAIN (c))
+ {
+ if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IS_DEVICE_PTR)
+ {
+ tree decl1 = DECL_NAME (OMP_CLAUSE_DECL (c));
+ tree decl2 = tree_strip_nop_conversions (*arg_p);
+ if (TREE_CODE (decl2) == ADDR_EXPR)
+ decl2 = TREE_OPERAND (decl2, 0);
+ gcc_assert (TREE_CODE (decl2) == VAR_DECL
+ || TREE_CODE (decl2) == PARM_DECL);
+ decl2 = DECL_NAME (decl2);
+ if (decl1 == decl2)
+ is_device_ptr = true;
+ }
+ else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEVICE)
+ device_num = OMP_CLAUSE_OPERAND (c, 0);
+ }
+
+ bool need_device_ptr = false;
+ for (tree arg
+ = TREE_PURPOSE (TREE_VALUE (adjust_args_list));
+ arg != NULL; arg = TREE_CHAIN (arg))
+ {
+ if (TREE_VALUE (arg)
+ && TREE_CODE (TREE_VALUE (arg)) == INTEGER_CST
+ && wi::eq_p (i, wi::to_wide (TREE_VALUE (arg))))
+ {
+ need_device_ptr = true;
+ break;
+ }
+ }
+
+ if (need_device_ptr && !is_device_ptr)
+ {
+ if (device_num == NULL_TREE)
+ {
+ // device_num = omp_get_default_device ()
+ tree fn = builtin_decl_explicit (
+ BUILT_IN_OMP_GET_DEFAULT_DEVICE);
+ gcall *call = gimple_build_call (fn, 0);
+ device_num = create_tmp_var (
+ gimple_call_return_type (call));
+ gimple_call_set_lhs (call, device_num);
+ gimplify_seq_add_stmt (pre_p, call);
+ }
+
+ // mapped_arg = omp_get_mapped_ptr (arg, device_num)
+ tree fn = builtin_decl_explicit (
+ BUILT_IN_OMP_GET_MAPPED_PTR);
+ *arg_p = (TREE_CODE (*arg_p) == NOP_EXPR)
+ ? TREE_OPERAND (*arg_p, 0)
+ : *arg_p;
+ gimplify_arg (arg_p, pre_p, loc);
+ gimplify_arg (&device_num, pre_p, loc);
+ call = gimple_build_call (fn, 2, *arg_p, device_num);
+ tree mapped_arg
+ = create_tmp_var (gimple_call_return_type (call));
+ gimple_call_set_lhs (call, mapped_arg);
+ gimplify_seq_add_stmt (pre_p, call);
+
+ *arg_p = mapped_arg;
+
+ // Mark mapped argument as device pointer to ensure
+ // idempotency in gimplification
+ gcc_assert (gimplify_omp_ctxp->code == OMP_DISPATCH);
+ tree c = build_omp_clause (input_location,
+ OMP_CLAUSE_IS_DEVICE_PTR);
+ OMP_CLAUSE_DECL (c) = *arg_p;
+ OMP_CLAUSE_CHAIN (c) = gimplify_omp_ctxp->clauses;
+ gimplify_omp_ctxp->clauses = c;
+ }
+ }
+ }
+
+ if (gimplify_omp_ctxp && gimplify_omp_ctxp->code == OMP_DISPATCH)
+ gimplify_omp_ctxp->in_call_args = true;
+ t = gimplify_arg (arg_p, pre_p, EXPR_LOCATION (*expr_p),
+ !returns_twice);
+ if (gimplify_omp_ctxp && gimplify_omp_ctxp->code == OMP_DISPATCH)
+ gimplify_omp_ctxp->in_call_args = false;
+
+ if (t == GS_ERROR)
+ ret = GS_ERROR;
+ }
+ }
}
/* Gimplify the static chain. */
@@ -6337,6 +6441,7 @@ is_gimple_stmt (tree t)
case OACC_LOOP:
case OMP_SCAN:
case OMP_SCOPE:
+ case OMP_DISPATCH:
case OMP_SECTIONS:
case OMP_SECTION:
case OMP_STRUCTURED_BLOCK:
@@ -13157,6 +13262,21 @@ gimplify_scan_omp_clauses (tree *list_p, gimple_seq *pre_p,
break;
}
}
+ else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEVICE
+ && code == OMP_DISPATCH)
+ {
+ bool saved_into_ssa = gimplify_ctxp->into_ssa;
+ gimplify_ctxp->into_ssa = false;
+ if (gimplify_expr (&OMP_CLAUSE_DEVICE_ID (c), pre_p, NULL,
+ is_gimple_val, fb_rvalue)
+ == GS_ERROR)
+ remove = true;
+ else if (DECL_P (OMP_CLAUSE_DEVICE_ID (c)))
+ omp_add_variable (ctx, OMP_CLAUSE_DEVICE_ID (c),
+ GOVD_SHARED | GOVD_SEEN);
+ gimplify_ctxp->into_ssa = saved_into_ssa;
+ break;
+ }
/* Fall through. */
case OMP_CLAUSE_PRIORITY:
@@ -13386,6 +13506,14 @@ gimplify_scan_omp_clauses (tree *list_p, gimple_seq *pre_p,
}
break;
+ case OMP_CLAUSE_NOVARIANTS:
+ OMP_CLAUSE_NOVARIANTS_EXPR (c)
+ = gimple_boolify (OMP_CLAUSE_NOVARIANTS_EXPR (c));
+ break;
+ case OMP_CLAUSE_NOCONTEXT:
+ OMP_CLAUSE_NOCONTEXT_EXPR (c)
+ = gimple_boolify (OMP_CLAUSE_NOCONTEXT_EXPR (c));
+ break;
case OMP_CLAUSE_NOHOST:
default:
gcc_unreachable ();
@@ -14713,6 +14841,60 @@ gimplify_adjust_omp_clauses (gimple_seq *pre_p, gimple_seq body, tree *list_p,
delete_omp_context (ctx);
}
+/* Try to evaluate a novariants clause. Return 1 if true, 0 if false or absent,
+ * -1 if run-time evaluation is needed. */
+
+int
+omp_has_novariants (void)
+{
+ for (struct gimplify_omp_ctx *ctx = gimplify_omp_ctxp; ctx;
+ ctx = ctx->outer_context)
+ {
+ if (ctx->code == OMP_DISPATCH && !ctx->in_call_args)
+ {
+ tree c = omp_find_clause (ctx->clauses, OMP_CLAUSE_NOVARIANTS);
+ if (c != NULL_TREE)
+ {
+ if (integer_nonzerop (OMP_CLAUSE_NOVARIANTS_EXPR (c)))
+ return 1;
+ else if (integer_zerop (OMP_CLAUSE_NOVARIANTS_EXPR (c)))
+ return 0;
+ else
+ return -1;
+ }
+ return 0;
+ }
+ }
+ return 0;
+}
+
+/* Try to evaluate a nocontext clause. Return 1 if true, 0 if false or absent,
+ * -1 if run-time evaluation is needed. */
+
+static int
+omp_has_nocontext (void)
+{
+ for (struct gimplify_omp_ctx *ctx = gimplify_omp_ctxp; ctx;
+ ctx = ctx->outer_context)
+ {
+ if (ctx->code == OMP_DISPATCH)
+ {
+ tree c = omp_find_clause (ctx->clauses, OMP_CLAUSE_NOCONTEXT);
+ if (c != NULL_TREE)
+ {
+ if (integer_nonzerop (OMP_CLAUSE_NOCONTEXT_EXPR (c)))
+ return 1;
+ else if (integer_zerop (OMP_CLAUSE_NOCONTEXT_EXPR (c)))
+ return 0;
+ else
+ return -1;
+ }
+ return 0;
+ }
+ }
+ return 0;
+}
+
/* Return 0 if CONSTRUCTS selectors don't match the OpenMP context,
-1 if unknown yet (simd is involved, won't be known until vectorization)
and 1 if they do. If SCORES is non-NULL, it should point to an array
@@ -14740,9 +14922,9 @@ omp_construct_selector_matches (enum tree_code *constructs, int nconstructs,
== ORT_TARGET && ctx->code == OMP_TARGET)
|| ((ctx->region_type & ORT_TEAMS) && ctx->code == OMP_TEAMS)
|| (ctx->region_type == ORT_WORKSHARE && ctx->code == OMP_FOR)
- || (ctx->region_type == ORT_SIMD
- && ctx->code == OMP_SIMD
- && !omp_find_clause (ctx->clauses, OMP_CLAUSE_BIND)))
+ || (ctx->region_type == ORT_SIMD && ctx->code == OMP_SIMD
+ && !omp_find_clause (ctx->clauses, OMP_CLAUSE_BIND))
+ || (ctx->code == OMP_DISPATCH && omp_has_nocontext () != 1))
{
++cnt;
if (scores)
@@ -17853,6 +18035,265 @@ gimplify_omp_ordered (tree expr, gimple_seq body)
return gimple_build_omp_ordered (body, OMP_ORDERED_CLAUSES (expr));
}
+/* Callback for walk_tree to find a IFN_GOMP_DISPATCH. */
+
+static tree
+find_ifn_gomp_dispatch (tree *tp, int *, void *modify)
+{
+ tree t = *tp;
+
+ if (TREE_CODE (t) == CALL_EXPR && CALL_EXPR_IFN (t) == IFN_GOMP_DISPATCH)
+ {
+ *tp = CALL_EXPR_ARG (t, 0);
+ return *(tree *) modify ? *(tree *) modify : *tp;
+ }
+
+ if (TREE_CODE (t) == MODIFY_EXPR)
+ *(tree *) modify = *tp;
+
+ return NULL_TREE;
+}
+
+/* Gimplify an OMP_DISPATCH construct. */
+
+static enum gimplify_status
+gimplify_omp_dispatch (tree *expr_p, gimple_seq *pre_p)
+{
+ tree expr = *expr_p;
+ gimple_seq body = NULL;
+
+ gimplify_scan_omp_clauses (&OMP_DISPATCH_CLAUSES (expr), pre_p, ORT_DISPATCH,
+ OMP_DISPATCH);
+ push_gimplify_context ();
+
+ // If device clause, adjust ICV
+ tree device
+ = omp_find_clause (OMP_DISPATCH_CLAUSES (expr), OMP_CLAUSE_DEVICE);
+ tree saved_device_icv;
+ if (device)
+ {
+ // Save current default-device-var ICV
+ saved_device_icv = create_tmp_var (integer_type_node);
+ tree fn = builtin_decl_explicit (BUILT_IN_OMP_GET_DEFAULT_DEVICE);
+ gcall *call = gimple_build_call (fn, 0);
+ gimple_call_set_lhs (call, saved_device_icv);
+ gimplify_seq_add_stmt (&body, call);
+
+ // Set default device
+ fn = builtin_decl_explicit (BUILT_IN_OMP_SET_DEFAULT_DEVICE);
+ call = gimple_build_call (fn, 1, OMP_CLAUSE_DEVICE_ID (device));
+ gimplify_seq_add_stmt (&body, call);
+ }
+
+ // If the novariants and nocontext clauses are not compile-time constants,
+ // we need to generate code for all possible cases:
+ // if (novariants) // implies nocontext
+ // base()
+ // else if (nocontext)
+ // variant1()
+ // else
+ // variant2()
+ tree *dispatch_body_p = &OMP_DISPATCH_BODY (expr);
+ if (TREE_CODE (*dispatch_body_p) == BIND_EXPR)
+ dispatch_body_p = &BIND_EXPR_BODY (*dispatch_body_p);
+ tree dispatch_body = *dispatch_body_p;
+
+ // Look for IFN_GOMP_DISPATCH and extract the base function call
+ tree base_call_expr = NULL_TREE;
+ if (TREE_CODE (dispatch_body) == STATEMENT_LIST)
+ for (tree_stmt_iterator tsi = tsi_start (dispatch_body); !tsi_end_p (tsi);
+ tsi_next (&tsi))
+ {
+ tree modify = NULL_TREE;
+ tree stmt = tsi_stmt (tsi);
+ base_call_expr
+ = walk_tree (&stmt, find_ifn_gomp_dispatch, &modify, NULL);
+ if (base_call_expr != NULL_TREE)
+ {
+ tsi_link_before (&tsi, base_call_expr, TSI_CONTINUE_LINKING);
+ tsi_next (&tsi);
+ tsi_delink (&tsi);
+ break;
+ }
+ }
+ else
+ {
+ tree modify = NULL_TREE;
+ base_call_expr
+ = walk_tree (dispatch_body_p, find_ifn_gomp_dispatch, &modify, NULL);
+ }
+ gcc_assert (base_call_expr != NULL_TREE);
+
+ tree dst = NULL_TREE;
+ if (TREE_CODE (base_call_expr) == MODIFY_EXPR)
+ {
+ dst = TREE_OPERAND (base_call_expr, 0);
+ base_call_expr = TREE_OPERAND (base_call_expr, 1);
+ }
+ while (TREE_CODE (base_call_expr) == FLOAT_EXPR
+ || TREE_CODE (base_call_expr) == CONVERT_EXPR
+ || TREE_CODE (base_call_expr) == COMPLEX_EXPR
+ || TREE_CODE (base_call_expr) == INDIRECT_REF)
+ base_call_expr = TREE_OPERAND (base_call_expr, 0);
+
+ tree base_fndecl = get_callee_fndecl (STRIP_NOPS (base_call_expr));
+ if (base_fndecl != NULL_TREE)
+ {
+ if (DECL_VIRTUAL_P (base_fndecl))
+ {
+ error_at (
+ EXPR_LOCATION (base_call_expr),
+ "%qD is a virtual function but only a direct call is allowed "
+ "in a dispatch construct",
+ DECL_NAME (base_fndecl));
+ }
+
+ tree variant_fndecl = omp_resolve_declare_variant (base_fndecl);
+ if (base_fndecl != variant_fndecl
+ && (omp_has_novariants () == -1 || omp_has_nocontext () == -1))
+ {
+ tree novariants_clause = NULL_TREE, nocontext_clause = NULL_TREE,
+ novariants_cond = NULL_TREE, nocontext_cond = NULL_TREE;
+ for (tree c = OMP_DISPATCH_CLAUSES (expr); c; c = TREE_CHAIN (c))
+ {
+ if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NOVARIANTS)
+ {
+ gcc_assert (novariants_cond == NULL_TREE);
+ novariants_clause = c;
+ novariants_cond = OMP_CLAUSE_NOVARIANTS_EXPR (c);
+ }
+ else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NOCONTEXT)
+ {
+ gcc_assert (nocontext_cond == NULL_TREE);
+ nocontext_clause = c;
+ nocontext_cond = OMP_CLAUSE_NOCONTEXT_EXPR (c);
+ }
+ }
+ gcc_assert (novariants_cond != NULL_TREE
+ || nocontext_cond != NULL_TREE);
+
+ enum gimplify_status ret
+ = gimplify_expr (&novariants_cond, &body, NULL, is_gimple_val,
+ fb_rvalue);
+ if (ret == GS_ERROR || ret == GS_UNHANDLED)
+ return ret;
+ ret = gimplify_expr (&nocontext_cond, &body, NULL, is_gimple_val,
+ fb_rvalue);
+ if (ret == GS_ERROR || ret == GS_UNHANDLED)
+ return ret;
+
+ tree base_label = create_artificial_label (UNKNOWN_LOCATION);
+ tree variant1_label = create_artificial_label (UNKNOWN_LOCATION);
+ tree cond_label = create_artificial_label (UNKNOWN_LOCATION);
+ tree variant2_label = create_artificial_label (UNKNOWN_LOCATION);
+ tree end_label = create_artificial_label (UNKNOWN_LOCATION);
+
+ if (novariants_cond != NULL_TREE)
+ {
+ gcond *novariants_cond_stmt
+ = gimple_build_cond_from_tree (novariants_cond, base_label,
+ cond_label);
+ gimplify_seq_add_stmt (&body, novariants_cond_stmt);
+
+ gimplify_seq_add_stmt (&body, gimple_build_label (base_label));
+ tree base_call_expr2 = copy_node (base_call_expr);
+ if (TREE_CODE (dispatch_body) == MODIFY_EXPR)
+ {
+ base_call_expr2 = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst,
+ base_call_expr2);
+ }
+ OMP_CLAUSE_NOVARIANTS_EXPR (novariants_clause)
+ = boolean_true_node;
+ gimplify_and_add (base_call_expr2, &body);
+ gimplify_seq_add_stmt (&body, gimple_build_goto (end_label));
+
+ OMP_CLAUSE_NOVARIANTS_EXPR (novariants_clause)
+ = boolean_false_node;
+ }
+
+ gimplify_seq_add_stmt (&body, gimple_build_label (cond_label));
+ if (nocontext_cond != NULL_TREE)
+ {
+ gcond *nocontext_cond_stmt
+ = gimple_build_cond_from_tree (nocontext_cond, variant1_label,
+ variant2_label);
+ gimplify_seq_add_stmt (&body, nocontext_cond_stmt);
+
+ gimplify_seq_add_stmt (&body,
+ gimple_build_label (variant1_label));
+ tree variant_call_expr = copy_node (base_call_expr);
+ if (TREE_CODE (dispatch_body) == MODIFY_EXPR)
+ {
+ variant_call_expr = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst,
+ variant_call_expr);
+ }
+ OMP_CLAUSE_NOCONTEXT_EXPR (nocontext_clause) = boolean_true_node;
+ gimplify_and_add (variant_call_expr, &body);
+ gimplify_seq_add_stmt (&body, gimple_build_goto (end_label));
+ OMP_CLAUSE_NOCONTEXT_EXPR (nocontext_clause) = boolean_false_node;
+ }
+
+ gimplify_seq_add_stmt (&body, gimple_build_label (variant2_label));
+ tree variant_call_expr = copy_node (base_call_expr);
+ if (TREE_CODE (dispatch_body) == MODIFY_EXPR)
+ {
+ variant_call_expr
+ = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, variant_call_expr);
+ }
+ gimplify_and_add (variant_call_expr, &body);
+ gimplify_seq_add_stmt (&body, gimple_build_goto (end_label));
+ gimplify_seq_add_stmt (&body, gimple_build_label (end_label));
+ }
+ else
+ gimplify_and_add (OMP_DISPATCH_BODY (expr), &body);
+ }
+ else
+ gimplify_and_add (OMP_DISPATCH_BODY (expr), &body);
+
+ // Restore default-device-var ICV
+ if (device)
+ {
+ tree fn = builtin_decl_explicit (BUILT_IN_OMP_SET_DEFAULT_DEVICE);
+ gcall *call = gimple_build_call (fn, 1, saved_device_icv);
+ gimplify_seq_add_stmt (&body, call);
+ }
+
+ // Wrap dispatch body into a bind
+ gimple *bind = gimple_build_bind (NULL_TREE, body, NULL_TREE);
+ pop_gimplify_context (bind);
+
+ // Manually tear down context created by gimplify_scan_omp_clauses to avoid a
+ // call to gimplify_adjust_omp_clauses
+ gimplify_omp_ctx *ctx = gimplify_omp_ctxp;
+ if (ctx != NULL)
+ {
+ gcc_assert (ctx->code == OMP_DISPATCH);
+ gimplify_omp_ctxp = ctx->outer_context;
+ delete_omp_context (ctx);
+ }
+
+ // Remove nowait as it has no effect on dispatch (OpenMP 5.2), and depend and
+ // device clauses as they have already been handled
+ tree *dispatch_clauses_ptr = &OMP_DISPATCH_CLAUSES (expr);
+ for (tree c = *dispatch_clauses_ptr; c; c = *dispatch_clauses_ptr)
+ {
+ if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NOWAIT
+ || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND
+ || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEVICE)
+ {
+ *dispatch_clauses_ptr = OMP_CLAUSE_CHAIN (c);
+ break;
+ }
+ else
+ dispatch_clauses_ptr = &OMP_CLAUSE_CHAIN (c);
+ }
+
+ gimple *stmt = gimple_build_omp_dispatch (bind, OMP_DISPATCH_CLAUSES (expr));
+ gimplify_seq_add_stmt (pre_p, stmt);
+ *expr_p = NULL_TREE;
+ return GS_ALL_DONE;
+}
+
/* Convert the GENERIC expression tree *EXPR_P to GIMPLE. If the
expression produces a value to be used as an operand inside a GIMPLE
statement, the value will be stored back in *EXPR_P. This value will
@@ -18791,6 +19232,10 @@ gimplify_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p,
ret = gimplify_omp_atomic (expr_p, pre_p);
break;
+ case OMP_DISPATCH:
+ ret = gimplify_omp_dispatch (expr_p, pre_p);
+ break;
+
case TRANSACTION_EXPR:
ret = gimplify_transaction (expr_p, pre_p);
break;
@@ -19116,7 +19561,8 @@ gimplify_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p,
&& code != OMP_SECTION
&& code != OMP_STRUCTURED_BLOCK
&& code != OMP_SINGLE
- && code != OMP_SCOPE);
+ && code != OMP_SCOPE
+ && code != OMP_DISPATCH);
}
#endif
@@ -77,6 +77,7 @@ extern enum gimplify_status gimplify_expr (tree *, gimple_seq *, gimple_seq *,
bool (*) (tree), fallback_t);
int omp_construct_selector_matches (enum tree_code *, int, int *);
+int omp_has_novariants (void);
extern void gimplify_type_sizes (tree, gimple_seq *);
extern void gimplify_one_sizepos (tree *, gimple_seq *);
@@ -662,6 +662,14 @@ expand_GOMP_SIMD_ORDERED_END (internal_fn, gcall *)
gcc_unreachable ();
}
+/* This should get expanded in gimplify_omp_dispatch. */
+
+static void
+expand_GOMP_DISPATCH (internal_fn, gcall *)
+{
+ gcc_unreachable ();
+}
+
/* This should get expanded in the sanopt pass. */
static void
@@ -469,6 +469,7 @@ DEF_INTERNAL_FN (GOMP_SIMD_VF, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL)
DEF_INTERNAL_FN (GOMP_SIMD_LAST_LANE, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL)
DEF_INTERNAL_FN (GOMP_SIMD_ORDERED_START, ECF_LEAF | ECF_NOTHROW, NULL)
DEF_INTERNAL_FN (GOMP_SIMD_ORDERED_END, ECF_LEAF | ECF_NOTHROW, NULL)
+DEF_INTERNAL_FN (GOMP_DISPATCH, ECF_LEAF | ECF_NOTHROW, NULL)
DEF_INTERNAL_FN (LOOP_VECTORIZED, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW, NULL)
DEF_INTERNAL_FN (LOOP_DIST_ALIAS, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW, NULL)
DEF_INTERNAL_FN (ANNOTATE, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL)
@@ -80,6 +80,12 @@ DEF_GOMP_BUILTIN (BUILT_IN_OMP_GET_TEAM_NUM, "omp_get_team_num",
BT_FN_INT, ATTR_CONST_NOTHROW_LEAF_LIST)
DEF_GOMP_BUILTIN (BUILT_IN_OMP_GET_NUM_TEAMS, "omp_get_num_teams",
BT_FN_INT, ATTR_CONST_NOTHROW_LEAF_LIST)
+DEF_GOMP_BUILTIN (BUILT_IN_OMP_GET_MAPPED_PTR, "omp_get_mapped_ptr",
+ BT_FN_PTR_CONST_PTR_INT, ATTR_NOTHROW_LEAF_LIST)
+DEF_GOMP_BUILTIN (BUILT_IN_OMP_GET_DEFAULT_DEVICE, "omp_get_default_device",
+ BT_FN_INT, ATTR_NOTHROW_LEAF_LIST)
+DEF_GOMP_BUILTIN (BUILT_IN_OMP_SET_DEFAULT_DEVICE, "omp_set_default_device",
+ BT_FN_INT, ATTR_NOTHROW_LEAF_LIST)
DEF_GOMP_BUILTIN (BUILT_IN_GOMP_ATOMIC_START, "GOMP_atomic_start",
BT_FN_VOID, ATTR_NOTHROW_LEAF_LIST)
@@ -1049,7 +1049,7 @@ omp_construct_traits_to_codes (tree ctx, int nconstructs,
/* Order must match the OMP_TRAIT_CONSTRUCT_* enumerators in
enum omp_ts_code. */
static enum tree_code code_map[]
- = { OMP_TARGET, OMP_TEAMS, OMP_PARALLEL, OMP_FOR, OMP_SIMD };
+ = { OMP_TARGET, OMP_TEAMS, OMP_PARALLEL, OMP_FOR, OMP_SIMD, OMP_DISPATCH };
for (tree ts = ctx; ts; ts = TREE_CHAIN (ts), i--)
{
@@ -1142,6 +1142,7 @@ const char *omp_tss_map[] =
"target_device",
"implementation",
"user",
+ "need_device_ptr",
NULL
};
@@ -1253,10 +1254,14 @@ struct omp_ts_info omp_ts_map[] =
OMP_TRAIT_PROPERTY_CLAUSE_LIST, false,
NULL
},
+ { "dispatch",
+ (1 << OMP_TRAIT_SET_CONSTRUCT),
+ OMP_TRAIT_PROPERTY_NONE, false,
+ NULL
+ },
{ NULL, 0, OMP_TRAIT_PROPERTY_NONE, false, NULL } /* OMP_TRAIT_LAST */
};
-
/* Return a name from PROP, a property in selectors accepting
name lists. */
@@ -2579,6 +2584,9 @@ omp_resolve_declare_variant (tree base)
if (cfun && (cfun->curr_properties & PROP_gimple_any) != 0)
return omp_resolve_late_declare_variant (base);
+ if (omp_has_novariants () == 1)
+ return base;
+
auto_vec <tree, 16> variants;
auto_vec <bool, 16> defer;
bool any_deferred = false;
@@ -2725,6 +2733,8 @@ omp_resolve_declare_variant (tree base)
(*slot)->variants = entry.variants;
tree alt = build_decl (DECL_SOURCE_LOCATION (base), FUNCTION_DECL,
DECL_NAME (base), TREE_TYPE (base));
+ if (DECL_ASSEMBLER_NAME_SET_P (base))
+ SET_DECL_ASSEMBLER_NAME (alt, DECL_ASSEMBLER_NAME (base));
DECL_ARTIFICIAL (alt) = 1;
DECL_IGNORED_P (alt) = 1;
TREE_STATIC (alt) = 1;
@@ -4205,6 +4205,11 @@ scan_omp_1_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p,
scan_omp (gimple_omp_body_ptr (stmt), ctx);
break;
+ case GIMPLE_OMP_DISPATCH:
+ ctx = new_omp_context (stmt, ctx);
+ scan_omp (gimple_omp_body_ptr (stmt), ctx);
+ break;
+
case GIMPLE_OMP_SECTIONS:
scan_omp_sections (as_a <gomp_sections *> (stmt), ctx);
break;
@@ -8946,6 +8951,31 @@ lower_omp_scope (gimple_stmt_iterator *gsi_p, omp_context *ctx)
if (BLOCK_VARS (block))
TREE_USED (block) = 1;
}
+
+/* Lower code for an OMP dispatch directive. */
+
+static void
+lower_omp_dispatch (gimple_stmt_iterator *gsi_p, omp_context *ctx)
+{
+ tree block;
+ gimple *stmt = gsi_stmt (*gsi_p);
+ gbind *bind;
+
+ push_gimplify_context ();
+
+ block = make_node (BLOCK);
+ bind = gimple_build_bind (NULL, NULL, block);
+ gsi_replace (gsi_p, bind, true);
+
+ lower_omp (gimple_omp_body_ptr (stmt), ctx);
+ gimple_bind_set_body (bind, maybe_catch_exception (gimple_omp_body (stmt)));
+
+ pop_gimplify_context (bind);
+
+ gimple_bind_append_vars (bind, ctx->block_vars);
+ BLOCK_VARS (block) = ctx->block_vars;
+}
+
/* Expand code for an OpenMP master or masked directive. */
static void
@@ -14577,6 +14607,11 @@ lower_omp_1 (gimple_stmt_iterator *gsi_p, omp_context *ctx)
gcc_assert (ctx);
lower_omp_scope (gsi_p, ctx);
break;
+ case GIMPLE_OMP_DISPATCH:
+ ctx = maybe_lookup_ctx (stmt);
+ gcc_assert (ctx);
+ lower_omp_dispatch (gsi_p, ctx);
+ break;
case GIMPLE_OMP_SINGLE:
ctx = maybe_lookup_ctx (stmt);
gcc_assert (ctx);
@@ -1679,6 +1679,12 @@ remap_gimple_stmt (gimple *stmt, copy_body_data *id)
(s1, gimple_omp_scope_clauses (stmt));
break;
+ case GIMPLE_OMP_DISPATCH:
+ s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
+ copy = gimple_build_omp_dispatch (s1,
+ gimple_omp_dispatch_clauses (stmt));
+ break;
+
case GIMPLE_OMP_TASKGROUP:
s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
copy = gimple_build_omp_taskgroup
@@ -4609,6 +4615,7 @@ estimate_num_insns (gimple *stmt, eni_weights *weights)
case GIMPLE_OMP_MASTER:
case GIMPLE_OMP_MASKED:
case GIMPLE_OMP_SCOPE:
+ case GIMPLE_OMP_DISPATCH:
case GIMPLE_OMP_TASKGROUP:
case GIMPLE_OMP_ORDERED:
case GIMPLE_OMP_SCAN: