@@ -216,6 +216,8 @@ lib_libopenvswitch_la_SOURCES = \
lib/object-collection.h \
lib/odp-execute.c \
lib/odp-execute.h \
+ lib/odp-execute-private.c \
+ lib/odp-execute-private.h \
lib/odp-util.c \
lib/odp-util.h \
lib/ofp-actions.c \
new file mode 100644
@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2022 Intel.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <config.h>
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "dpdk.h"
+#include "dp-packet.h"
+#include "odp-execute-private.h"
+#include "odp-netlink.h"
+#include "odp-util.h"
+#include "openvswitch/vlog.h"
+
+VLOG_DEFINE_THIS_MODULE(odp_execute_impl);
+static int active_action_impl_index;
+
+static struct odp_execute_action_impl action_impls[] = {
+ [ACTION_IMPL_SCALAR] = {
+ .available = false,
+ .name = "scalar",
+ .init_func = NULL,
+ },
+};
+
+static void
+action_impl_copy_funcs(struct odp_execute_action_impl *dest,
+ const struct odp_execute_action_impl *src)
+{
+ for (int i = 0; i < __OVS_ACTION_ATTR_MAX; i++) {
+ atomic_store_relaxed(&dest->funcs[i], src->funcs[i]);
+ }
+}
+
+struct odp_execute_action_impl *
+odp_execute_action_set(const char *name)
+{
+ for (int i = 0; i < ACTION_IMPL_MAX; i++) {
+ /* String compare, and set ptrs atomically. */
+ if (!strcmp(action_impls[i].name, name)) {
+ active_action_impl_index = i;
+
+ VLOG_INFO("Action implementation set to %s", name);
+ return &action_impls[i];
+ }
+ }
+ return NULL;
+}
+
+void
+odp_execute_action_init(void)
+{
+ /* Each impl's function array is initialized to reflect the scalar
+ * implementation. This simplifies adding optimized implementations,
+ * as the autovalidator can always compare all actions.
+ *
+ * Below will check if impl is available and copies the scalar functions
+ * to all other implementations. */
+ for (int i = 0; i < ACTION_IMPL_MAX; i++) {
+ bool avail = true;
+
+ if (i != ACTION_IMPL_SCALAR) {
+ action_impl_copy_funcs(&action_impls[i],
+ &action_impls[ACTION_IMPL_SCALAR]);
+ }
+
+ if (action_impls[i].init_func) {
+ /* Return zero is success, non-zero means error. */
+ avail = (action_impls[i].init_func(&action_impls[i]) == 0);
+ }
+
+ action_impls[i].available = avail;
+
+ VLOG_INFO("Action implementation %s (available: %s)",
+ action_impls[i].name, avail ? "Yes" : "No");
+ }
+}
new file mode 100644
@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2022 Intel.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ODP_EXTRACT_PRIVATE
+#define ODP_EXTRACT_PRIVATE 1
+
+#include "dp-packet.h"
+#include "odp-execute.h"
+#include "odp-netlink.h"
+#include "ovs-atomic.h"
+
+/* Forward declaration for typedef. */
+struct odp_execute_action_impl;
+
+/* Typedef for an initialization function that can initialize each
+ * implementation, checking requirements such as CPU ISA. */
+typedef int (*odp_execute_action_init_func)
+ (struct odp_execute_action_impl *self);
+
+/* Structure represents an implementation of the odp actions. */
+struct odp_execute_action_impl {
+ /* When set, the CPU ISA required for this implementation is available
+ * and the implementation can be used. */
+ bool available;
+
+ /* Name of the implementation. */
+ const char *name;
+
+ /* Function is used to detect if this CPU has the ISA required
+ * to run the optimized action implementation and if available, initializes
+ * the implementation for use. */
+ odp_execute_action_init_func init_func;
+
+ /* An array of callback functions, one for each action. */
+ ATOMIC(odp_execute_action_cb) funcs[__OVS_ACTION_ATTR_MAX];
+};
+
+/* Order of Actions implementations. */
+enum odp_execute_action_impl_idx {
+ ACTION_IMPL_SCALAR,
+ /* See ACTION_IMPL_BEGIN below, for "first to-be-validated" impl.
+ * Do not change the autovalidator position in this list without updating
+ * the define below. */
+
+ ACTION_IMPL_MAX,
+};
+
+/* Index to start verifying implementations from. */
+BUILD_ASSERT_DECL(ACTION_IMPL_SCALAR == 0);
+
+/* Odp execute init handles setting up the state of the actions functions at
+ * initialization time. It cannot return errors, as it must always succeed in
+ * initializing the scalar/generic codepath. */
+void odp_execute_action_init(void);
+
+struct odp_execute_action_impl * odp_execute_action_set(const char *name);
+
+#endif /* ODP_EXTRACT_PRIVATE */
@@ -17,6 +17,7 @@
#include <config.h>
#include "odp-execute.h"
+#include "odp-execute-private.h"
#include <sys/types.h>
#include <netinet/in.h>
#include <arpa/inet.h>
@@ -833,6 +834,36 @@ requires_datapath_assistance(const struct nlattr *a)
return false;
}
+/* The active function pointers on the datapath. ISA optimized implementations
+ * are enabled by plugging them into this static arary, which is consulted when
+ * applying actions on the datapath. */
+static ATOMIC(struct odp_execute_action_impl *) actions_active_impl;
+
+static int
+odp_actions_impl_set(const char *name)
+{
+ struct odp_execute_action_impl *active;
+ active = odp_execute_action_set(name);
+ if (!active) {
+ VLOG_ERR("Failed setting action implementation to %s", name);
+ return 1;
+ }
+
+ atomic_store_relaxed(&actions_active_impl, active);
+ return 0;
+}
+
+void
+odp_execute_init(void)
+{
+ static struct ovsthread_once once = OVSTHREAD_ONCE_INITIALIZER;
+ if (ovsthread_once_start(&once)) {
+ odp_execute_action_init();
+ odp_actions_impl_set("scalar");
+ ovsthread_once_done(&once);
+ }
+}
+
/* Executes all of the 'actions_len' bytes of datapath actions in 'actions' on
* the packets in 'batch'. If 'steal' is true, possibly modifies and
* definitely free the packets in 'batch', otherwise leaves 'batch' unchanged.
@@ -857,6 +888,7 @@ odp_execute_actions(void *dp, struct dp_packet_batch *batch, bool steal,
NL_ATTR_FOR_EACH_UNSAFE (a, left, actions, actions_len) {
int type = nl_attr_type(a);
+ enum ovs_action_attr attr_type = (enum ovs_action_attr) type;
bool last_action = (left <= NLA_ALIGN(a->nla_len));
if (requires_datapath_assistance(a)) {
@@ -879,8 +911,25 @@ odp_execute_actions(void *dp, struct dp_packet_batch *batch, bool steal,
continue;
}
- switch ((enum ovs_action_attr) type) {
+ /* If type is set in the active actions implementation, call the
+ * function-pointer and continue to the next action. */
+ if (attr_type <= OVS_ACTION_ATTR_MAX) {
+ /* Read the action implementation pointer atomically to avoid
+ * non-atomic read causing corruption if being written by another
+ * thread simultaneously. */
+ struct odp_execute_action_impl *actions_impl;
+ atomic_read_relaxed(&actions_active_impl, &actions_impl);
+
+ if (actions_impl && actions_impl->funcs[attr_type]) {
+ actions_impl->funcs[attr_type](batch, a);
+ continue;
+ }
+ }
+
+ /* If the action was not handled by the active function pointers above,
+ * process them by switching on the type below. */
+ switch (attr_type) {
case OVS_ACTION_ATTR_HASH: {
const struct ovs_action_hash *hash_act = nl_attr_get(a);
@@ -1094,6 +1143,9 @@ odp_execute_actions(void *dp, struct dp_packet_batch *batch, bool steal,
case __OVS_ACTION_ATTR_MAX:
OVS_NOT_REACHED();
}
+
+ /* Do not add any generic processing here, as it won't be executed when
+ * an ISA-specific action implementation exists. */
}
dp_packet_delete_batch(batch, steal);
@@ -28,6 +28,13 @@ struct dp_packet;
struct pkt_metadata;
struct dp_packet_batch;
+
+/* Called once at initialization time. */
+void odp_execute_init(void);
+
+typedef void (*odp_execute_action_cb)(struct dp_packet_batch *batch,
+ const struct nlattr *action);
+
typedef void (*odp_execute_cb)(void *dp, struct dp_packet_batch *batch,
const struct nlattr *action, bool should_steal);
@@ -182,6 +182,7 @@ m4_define([_OVS_VSWITCHD_START],
on_exit "kill_ovs_vswitchd `cat ovs-vswitchd.pid`"
AT_CHECK([[sed < stderr '
/ovs_numa|INFO|Discovered /d
+/odp_execute_impl|INFO|Action implementation /d
/vlog|INFO|opened log file/d
/vswitchd|INFO|ovs-vswitchd (Open vSwitch)/d
/reconnect|INFO|/d
@@ -40,6 +40,7 @@
#include "netdev.h"
#include "netdev-offload.h"
#include "nx-match.h"
+#include "odp-execute.h"
#include "ofproto/bond.h"
#include "ofproto/ofproto.h"
#include "openvswitch/dynamic-string.h"
@@ -530,6 +531,8 @@ bridge_init(const char *remote)
stp_init();
lldp_init();
rstp_init();
+ odp_execute_init();
+
ifaces_changed = seq_create();
last_ifaces_changed = seq_read(ifaces_changed);
ifnotifier = if_notifier_create(if_change_cb, NULL);