diff mbox series

[ovs-dev,v2,5/5] controller: Introduce route-exchange module.

Message ID 20240719020943.380924-5-fnordahl@ubuntu.com
State Superseded
Headers show
Series [ovs-dev,v2,1/5] controller: Move address with port parser to lib. | expand

Checks

Context Check Description
ovsrobot/apply-robot success apply and check: success
ovsrobot/github-robot-_ovn-kubernetes success github build: passed
ovsrobot/github-robot-_Build_and_Test fail github build: failed

Commit Message

Frode Nordahl July 19, 2024, 2:09 a.m. UTC
Introduce route-exchange module that depending on Logical Router
Port options maintains a VRF in the system for redistribution of
host routes to NAT addresses and LB VIPs attached to local gateway
router datapaths.

The route-exchange module requires input from both runtime_data
and lb_data engine nodes.  Consequently it needs its own I-P
engine node.

TODO:
* Make detection of route exchange enabled ports/datapaths for recompute
  decision less expensive.
* Keep track of created VRFs and clean up on exit.
* This patch adds LB system test, a NAT one is also needed.
* E2E test together with the bgp-mirror patch.
* E2E docs and NEWS items.

Signed-off-by: Frode Nordahl <fnordahl@ubuntu.com>
---
 controller/automake.mk           |   9 +-
 controller/ovn-controller.c      | 159 +++++++++++++++++++++++
 controller/route-exchange-stub.c |  31 +++++
 controller/route-exchange.c      | 216 +++++++++++++++++++++++++++++++
 controller/route-exchange.h      |  38 ++++++
 tests/system-ovn.at              | 188 ++++++++++++++++++++++++++-
 6 files changed, 637 insertions(+), 4 deletions(-)
 create mode 100644 controller/route-exchange-stub.c
 create mode 100644 controller/route-exchange.c
 create mode 100644 controller/route-exchange.h
diff mbox series

Patch

diff --git a/controller/automake.mk b/controller/automake.mk
index 006e884dc..3e91e97e6 100644
--- a/controller/automake.mk
+++ b/controller/automake.mk
@@ -49,13 +49,18 @@  controller_ovn_controller_SOURCES = \
 	controller/statctrl.h \
 	controller/statctrl.c \
 	controller/ct-zone.h \
-	controller/ct-zone.c
+	controller/ct-zone.c \
+	controller/route-exchange.h
 
 if HAVE_NETLINK
 controller_ovn_controller_SOURCES += \
 	controller/route-exchange-netlink.h \
 	controller/route-exchange-netlink-private.h \
-	controller/route-exchange-netlink.c
+	controller/route-exchange-netlink.c \
+	controller/route-exchange.c
+else
+controller_ovn_controller_SOURCES += \
+	controller/route-exchange-stub.c
 endif
 
 controller_ovn_controller_LDADD = lib/libovn.la $(OVS_LIBDIR)/libopenvswitch.la
diff --git a/controller/ovn-controller.c b/controller/ovn-controller.c
index f46edd22d..52deae439 100644
--- a/controller/ovn-controller.c
+++ b/controller/ovn-controller.c
@@ -87,6 +87,7 @@ 
 #include "statctrl.h"
 #include "lib/dns-resolve.h"
 #include "ct-zone.h"
+#include "route-exchange.h"
 
 VLOG_DEFINE_THIS_MODULE(main);
 
@@ -4572,6 +4573,14 @@  controller_output_mac_cache_handler(struct engine_node *node,
     return true;
 }
 
+static bool
+controller_output_route_exchange_handler(struct engine_node *node,
+                                         void *data OVS_UNUSED)
+{
+    engine_set_node_state(node, EN_UPDATED);
+    return true;
+}
+
 /* Handles sbrec_chassis changes.
  * If a new chassis is added or removed return false, so that
  * flows are recomputed.  For any updates, there is no need for
@@ -4595,6 +4604,142 @@  pflow_lflow_output_sb_chassis_handler(struct engine_node *node,
     return true;
 }
 
+struct ed_type_route_exchange {
+};
+
+static void
+en_route_exchange_run(struct engine_node *node,
+                      void *data OVS_UNUSED)
+{
+    const struct ovsrec_open_vswitch_table *ovs_table =
+        EN_OVSDB_GET(engine_get_input("OVS_open_vswitch", node));
+    const char *chassis_id = get_ovs_chassis_id(ovs_table);
+    ovs_assert(chassis_id);
+
+    struct ovsdb_idl_index *sbrec_chassis_by_name =
+        engine_ovsdb_node_get_index(
+                engine_get_input("SB_chassis", node),
+                "name");
+    const struct sbrec_chassis *chassis
+        = chassis_lookup_by_name(sbrec_chassis_by_name, chassis_id);
+    ovs_assert(chassis);
+
+    struct ovsdb_idl_index *sbrec_port_binding_by_name =
+        engine_ovsdb_node_get_index(
+                engine_get_input("SB_port_binding", node),
+                "name");
+    struct ed_type_runtime_data *rt_data =
+        engine_get_input_data("runtime_data", node);
+
+    const struct sbrec_load_balancer_table *lb_table =
+        EN_OVSDB_GET(engine_get_input("SB_load_balancer", node));
+    struct ed_type_lb_data *lb_data =
+        engine_get_input_data("lb_data", node);
+
+    struct route_exchange_ctx_in r_ctx_in = {
+        .sbrec_port_binding_by_name = sbrec_port_binding_by_name,
+        .lb_table = lb_table,
+        .chassis_rec = chassis,
+        .active_tunnels = &rt_data->active_tunnels,
+        .local_datapaths = &rt_data->local_datapaths,
+        .local_lbs = &lb_data->local_lbs,
+    };
+
+    route_exchange_run(&r_ctx_in);
+
+    engine_set_node_state(node, EN_UPDATED);
+}
+
+
+static void *
+en_route_exchange_init(struct engine_node *node OVS_UNUSED,
+                       struct engine_arg *arg OVS_UNUSED)
+{
+    return NULL;
+}
+
+static void
+en_route_exchange_cleanup(void *data OVS_UNUSED)
+{
+}
+
+static bool
+route_exchange_runtime_data_handler(struct engine_node *node,
+                                    void *data OVS_UNUSED)
+{
+    struct ed_type_runtime_data *rt_data =
+        engine_get_input_data("runtime_data", node);
+
+    if (!rt_data->tracked) {
+        return false;
+    }
+
+    struct tracked_datapath *tdp;
+    HMAP_FOR_EACH (tdp, node, &rt_data->tracked_dp_bindings) {
+        struct shash_node *shash_node;
+        SHASH_FOR_EACH (shash_node, &tdp->lports) {
+            struct tracked_lport *lport = shash_node->data;
+            if (route_exchange_relevant_port(lport->pb)) {
+                /* Until we get I-P support for route exchange we need to
+                 * request recompute. */
+                return false;
+            }
+        }
+    }
+
+    return true;
+}
+
+static bool
+route_exchange_lb_data_handler(struct engine_node *node,
+                               void *data OVS_UNUSED)
+{
+    struct ed_type_runtime_data *rt_data =
+        engine_get_input_data("runtime_data", node);
+    struct ed_type_lb_data *lb_data =
+        engine_get_input_data("lb_data", node);
+    const struct sbrec_load_balancer_table *lb_table =
+        EN_OVSDB_GET(engine_get_input("SB_load_balancer", node));
+
+    if (!lb_data->change_tracked) {
+        return false;
+    }
+
+    if (!rt_data->tracked) {
+        return false;
+    }
+
+    struct hmap *tracked_dp_bindings = &rt_data->tracked_dp_bindings;
+    if (hmap_is_empty(tracked_dp_bindings)) {
+        return true;
+    }
+
+    struct hmap *lbs = NULL;
+
+    struct tracked_datapath *tdp;
+    HMAP_FOR_EACH (tdp, node, tracked_dp_bindings) {
+        if (tdp->tracked_type != TRACKED_RESOURCE_NEW) {
+            continue;
+        }
+
+        if (!lbs) {
+            lbs = load_balancers_by_dp_init(&rt_data->local_datapaths,
+                                            lb_table);
+        }
+
+        struct load_balancers_by_dp *lbs_by_dp =
+            load_balancers_by_dp_find(lbs, tdp->dp);
+        if (lbs_by_dp) {
+            /* Until we get I-P support for route exchange we need to
+             * request recompute. */
+            load_balancers_by_dp_cleanup(lbs);
+            return false;
+        }
+    }
+    load_balancers_by_dp_cleanup(lbs);
+    return true;
+}
+
 /* Returns false if the northd internal version stored in SB_Global
  * and ovn-controller internal version don't match.
  */
@@ -4881,6 +5026,7 @@  main(int argc, char *argv[])
     ENGINE_NODE(if_status_mgr, "if_status_mgr");
     ENGINE_NODE_WITH_CLEAR_TRACK_DATA(lb_data, "lb_data");
     ENGINE_NODE(mac_cache, "mac_cache");
+    ENGINE_NODE(route_exchange, "route_exchange");
 
 #define SB_NODE(NAME, NAME_STR) ENGINE_NODE_SB(NAME, NAME_STR);
     SB_NODES
@@ -4903,6 +5049,17 @@  main(int argc, char *argv[])
     engine_add_input(&en_lb_data, &en_runtime_data,
                      lb_data_runtime_data_handler);
 
+    engine_add_input(&en_route_exchange, &en_ovs_open_vswitch, NULL);
+    engine_add_input(&en_route_exchange, &en_sb_chassis, NULL);
+    engine_add_input(&en_route_exchange, &en_sb_port_binding,
+                     engine_noop_handler);
+    engine_add_input(&en_route_exchange, &en_runtime_data,
+                     route_exchange_runtime_data_handler);
+    engine_add_input(&en_route_exchange, &en_sb_load_balancer,
+                     engine_noop_handler);
+    engine_add_input(&en_route_exchange, &en_lb_data,
+                     route_exchange_lb_data_handler);
+
     engine_add_input(&en_addr_sets, &en_sb_address_set,
                      addr_sets_sb_address_set_handler);
     engine_add_input(&en_port_groups, &en_sb_port_group,
@@ -5076,6 +5233,8 @@  main(int argc, char *argv[])
                      controller_output_pflow_output_handler);
     engine_add_input(&en_controller_output, &en_mac_cache,
                      controller_output_mac_cache_handler);
+    engine_add_input(&en_controller_output, &en_route_exchange,
+                     controller_output_route_exchange_handler);
 
     struct engine_arg engine_arg = {
         .sb_idl = ovnsb_idl_loop.idl,
diff --git a/controller/route-exchange-stub.c b/controller/route-exchange-stub.c
new file mode 100644
index 000000000..c9e4c8144
--- /dev/null
+++ b/controller/route-exchange-stub.c
@@ -0,0 +1,31 @@ 
+/*
+ * Copyright (c) 2024 Canonical
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <config.h>
+
+#include <stdbool.h>
+
+#include "openvswitch/compiler.h"
+#include "route-exchange.h"
+
+bool
+route_exchange_relevant_port(const struct sbrec_port_binding *pb OVS_UNUSED) {
+    return false;
+}
+
+void
+route_exchange_run(struct route_exchange_ctx_in *r_ctx_in OVS_UNUSED) {
+}
diff --git a/controller/route-exchange.c b/controller/route-exchange.c
new file mode 100644
index 000000000..f8bd265ae
--- /dev/null
+++ b/controller/route-exchange.c
@@ -0,0 +1,216 @@ 
+/*
+ * Copyright (c) 2024 Canonical
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <config.h>
+
+#include <net/if.h>
+
+#include "openvswitch/vlog.h"
+
+#include "lib/ovn-sb-idl.h"
+
+#include "binding.h"
+#include "ha-chassis.h"
+#include "lb.h"
+#include "local_data.h"
+#include "route-exchange.h"
+#include "route-exchange-netlink.h"
+
+
+VLOG_DEFINE_THIS_MODULE(route_exchange);
+static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 20);
+
+/* While the linux kernel can handle 2^32 routing tables, only so many can fit
+ * in the corresponding VRF interface name. */
+#define MAX_TABLE_ID 1000000000
+
+bool
+route_exchange_relevant_port(const struct sbrec_port_binding *pb) {
+    return (pb && pb->type && !strcmp(pb->type, "l3gateway") &&
+                (smap_get_bool(&pb->options, "redistribute-lb-vips", false) ||
+                 smap_get_bool(&pb->options, "redistribute-nat", false)));
+}
+
+static void
+extract_nat_addresses(const struct sbrec_port_binding *pb,
+                      struct route_exchange_ctx_in *r_ctx_in,
+                      uint32_t table_id, struct hmap *host_routes)
+{
+    if (!pb || !pb->n_nat_addresses) {
+        return;
+    }
+    VLOG_DBG("extract_nat_addresses: considering lport %s", pb->logical_port);
+
+    for (size_t i = 0; i < pb->n_nat_addresses; i++) {
+        struct lport_addresses *laddrs = xzalloc(sizeof *laddrs);
+        char *lport = NULL;
+
+        if (!extract_addresses_with_port(
+                pb->nat_addresses[i], laddrs, &lport)) {
+            VLOG_DBG("extract_nat_addresses: no addresses");
+            goto cleanup;
+        }
+        if (lport) {
+            const struct sbrec_port_binding *lport_pb = lport_lookup_by_name(
+                    r_ctx_in->sbrec_port_binding_by_name, lport);
+            if (!lport_pb || !lport_pb->chassis) {
+                VLOG_DBG("extract_nat_addresses: cannot find lport %s",
+                         lport);
+                goto cleanup;
+            }
+            enum en_lport_type lport_pb_type = get_lport_type(lport_pb);
+            if (((lport_pb_type == LP_VIF ||
+                  lport_pb_type == LP_CHASSISREDIRECT) &&
+                 lport_pb->chassis != r_ctx_in->chassis_rec) ||
+                 !ha_chassis_group_is_active(lport_pb->ha_chassis_group,
+                                             r_ctx_in->active_tunnels,
+                                             r_ctx_in->chassis_rec)) {
+                VLOG_DBG("extract_nat_addresses: ignoring non-local lport %s",
+                         lport);
+                goto cleanup;
+            }
+        }
+        for (size_t j = 0; j < laddrs->n_ipv4_addrs; j++) {
+            struct in6_addr addr;
+            in6_addr_set_mapped_ipv4(&addr, laddrs->ipv4_addrs[j].addr);
+            host_route_insert(host_routes, table_id, &addr);
+        }
+        for (size_t j = 0; j < laddrs->n_ipv6_addrs; j++) {
+            host_route_insert(host_routes, table_id,
+                              &laddrs->ipv6_addrs[j].addr);
+        }
+
+cleanup:
+        destroy_lport_addresses(laddrs);
+        free(laddrs);
+        if (lport) {
+            free(lport);
+        }
+    }
+}
+
+static void
+extract_lb_vips(const struct sbrec_datapath_binding *dpb,
+                struct hmap *lbs_by_dp_hmap,
+                const struct route_exchange_ctx_in *r_ctx_in,
+                uint32_t table_id, struct hmap *host_routes)
+{
+    struct load_balancers_by_dp *lbs_by_dp
+        = load_balancers_by_dp_find(lbs_by_dp_hmap, dpb);
+    if (!lbs_by_dp) {
+        return;
+    }
+
+    for (size_t i = 0; i < lbs_by_dp->n_dp_lbs; i++) {
+        const struct sbrec_load_balancer *sbrec_lb
+            = lbs_by_dp->dp_lbs[i];
+
+        if (!sbrec_lb) {
+            return;
+        }
+
+        struct ovn_controller_lb *lb
+            = ovn_controller_lb_find(r_ctx_in->local_lbs,
+                                     &sbrec_lb->header_.uuid);
+
+        if (!lb || !lb->slb) {
+            return;
+        }
+
+        VLOG_DBG("considering lb for route leaking: %s", lb->slb->name);
+        for (i = 0; i < lb->n_vips; i++) {
+            VLOG_DBG("considering lb for route leaking: %s vip_str=%s",
+                      lb->slb->name, lb->vips[i].vip_str);
+            host_route_insert(host_routes, table_id, &lb->vips[i].vip);
+        }
+    }
+}
+
+void
+route_exchange_run(struct route_exchange_ctx_in *r_ctx_in)
+{
+    struct hmap *lbs_by_dp_hmap
+        = load_balancers_by_dp_init(r_ctx_in->local_datapaths,
+                                    r_ctx_in->lb_table);
+
+    /* Extract all NAT- and LB VIP-addresses associated with lports resident on
+     * the current chassis to allow full sync of leaked routing tables. */
+    const struct local_datapath *ld;
+    HMAP_FOR_EACH (ld, hmap_node, r_ctx_in->local_datapaths) {
+        if (!ld->n_peer_ports || ld->is_switch) {
+            continue;
+        }
+
+        bool maintain_vrf = false;
+        bool lbs_sync = false;
+        struct hmap local_host_routes_for_current_dp
+            = HMAP_INITIALIZER(&local_host_routes_for_current_dp);
+
+        /* This is a LR datapath, find LRPs with route exchange options. */
+        for (size_t i = 0; i < ld->n_peer_ports; i++) {
+            const struct sbrec_port_binding *local_peer
+                = ld->peer_ports[i].local;
+            if (!local_peer || !route_exchange_relevant_port(local_peer)) {
+                continue;
+            }
+
+            maintain_vrf |= smap_get_bool(&local_peer->options,
+                                          "maintain-vrf", false);
+            lbs_sync |= smap_get_bool(&local_peer->options,
+                                    "redistribute-lb-vips",
+                                    false);
+            if (smap_get_bool(&local_peer->options,
+                              "redistribute-nat",
+                              false)) {
+                extract_nat_addresses(local_peer, r_ctx_in,
+                                      ld->datapath->tunnel_key,
+                                      &local_host_routes_for_current_dp);
+            }
+        }
+
+        if (lbs_sync) {
+            extract_lb_vips(ld->datapath, lbs_by_dp_hmap, r_ctx_in,
+                            ld->datapath->tunnel_key,
+                            &local_host_routes_for_current_dp);
+        }
+
+        /* While tunnel_key would most likely never be negative, the compiler
+         * has opinions if we don't check before using it in snprintf below. */
+        if (ld->datapath->tunnel_key < 0 ||
+            ld->datapath->tunnel_key > MAX_TABLE_ID) {
+            VLOG_WARN_RL(&rl,
+                         "skip route sync for datapath "UUID_FMT", "
+                         "tunnel_key %ld would make VRF interface name "
+                         "overflow.",
+                         UUID_ARGS(&ld->datapath->header_.uuid),
+                         ld->datapath->tunnel_key);
+            goto out;
+        }
+        char vrf_name[IFNAMSIZ + 1];
+        snprintf(vrf_name, sizeof vrf_name, "ovnvrf%ld",
+                 ld->datapath->tunnel_key);
+
+        if (maintain_vrf) {
+            re_nl_create_vrf(vrf_name, ld->datapath->tunnel_key);
+        }
+        re_nl_sync_routes(ld->datapath->tunnel_key, vrf_name,
+                          &local_host_routes_for_current_dp);
+
+out:
+        host_routes_destroy(&local_host_routes_for_current_dp);
+    }
+    load_balancers_by_dp_cleanup(lbs_by_dp_hmap);
+}
diff --git a/controller/route-exchange.h b/controller/route-exchange.h
new file mode 100644
index 000000000..7798874be
--- /dev/null
+++ b/controller/route-exchange.h
@@ -0,0 +1,38 @@ 
+/*
+ * Copyright (c) 2024 Canonical
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ROUTE_EXCHANGE_H
+#define ROUTE_EXCHANGE_H 1
+
+struct hmap;
+struct ovsdb_idl_index;
+struct sbrec_chassis;
+struct sbrec_port_binding;
+struct sset;
+
+struct route_exchange_ctx_in {
+    struct ovsdb_idl_index *sbrec_port_binding_by_name;
+    const struct sbrec_load_balancer_table *lb_table;
+    const struct sbrec_chassis *chassis_rec;
+    const struct sset *active_tunnels;
+    struct hmap *local_datapaths;
+    struct hmap *local_lbs;
+};
+
+bool route_exchange_relevant_port(const struct sbrec_port_binding *pb);
+void route_exchange_run(struct route_exchange_ctx_in *);
+
+#endif /* ROUTE_EXCHANGE_H */
diff --git a/tests/system-ovn.at b/tests/system-ovn.at
index c24ede7c5..5ad4684e0 100644
--- a/tests/system-ovn.at
+++ b/tests/system-ovn.at
@@ -2337,13 +2337,18 @@  ovs-vsctl \
 # Start ovn-controller
 start_daemon ovn-controller
 
-check ovn-nbctl lr-add R1
+ovn-appctl vlog/set route_exchange
+check ovn-nbctl -- lr-add R1 \
+                -- set Logical_Router R1 options:requested-tnl-key=1000
 
 check ovn-nbctl ls-add sw0
 check ovn-nbctl ls-add public
 
 check ovn-nbctl lrp-add R1 rp-sw0 00:00:01:01:02:03 192.168.1.1/24
-check ovn-nbctl lrp-add R1 rp-public 00:00:02:01:02:03 172.16.1.1/24
+check ovn-nbctl -- lrp-add R1 rp-public 00:00:02:01:02:03 172.16.1.1/24 \
+                -- lrp-set-options rp-public \
+                       maintain-vrf=true \
+                       redistribute-lb-vips=true
 
 check ovn-nbctl set logical_router R1 options:chassis=hv1
 
@@ -2379,6 +2384,13 @@  check ovn-nbctl lr-lb-add R1 lb1
 
 check ovn-nbctl --wait=hv sync
 
+ovn-sbctl list port-binding
+ovn-sbctl list datapath-binding
+ovn-sbctl list logical-dp-group
+ip li
+ip vrf
+ip route show table 1000
+
 for i in $(seq 1 5); do
     echo Request $i
     NS_CHECK_EXEC([client], [wget 172.16.1.100 -t 5 -T 1 --retry-connrefused -v -o wget$i.log])
@@ -13027,3 +13039,175 @@  OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d
 /connection dropped.*/d"])
 AT_CLEANUP
 ])
+
+OVN_FOR_EACH_NORTHD([
+AT_SETUP([route-exchange for LB VIPs with gateway router IPv4])
+AT_KEYWORDS([route-exchange])
+
+CHECK_CONNTRACK()
+CHECK_CONNTRACK_NAT()
+ovn_start
+OVS_TRAFFIC_VSWITCHD_START()
+ADD_BR([br-int])
+ADD_BR([br-ext], [set Bridge br-ext fail-mode=standalone])
+
+# Set external-ids in br-int needed for ovn-controller
+ovs-vsctl \
+        -- set Open_vSwitch . external-ids:system-id=hv1 \
+        -- set Open_vSwitch . external-ids:ovn-remote=unix:$ovs_base/ovn-sb/ovn-sb.sock \
+        -- set Open_vSwitch . external-ids:ovn-encap-type=geneve \
+        -- set Open_vSwitch . external-ids:ovn-encap-ip=169.0.0.1 \
+        -- set bridge br-int fail-mode=secure other-config:disable-in-band=true
+
+# Start ovn-controller
+start_daemon ovn-controller
+
+ovn-appctl vlog/set route_exchange
+check ovn-nbctl -- lr-add R1 \
+                -- set Logical_Router R1 options:requested-tnl-key=1000
+
+check ovn-nbctl ls-add sw0
+check ovn-nbctl ls-add public
+
+check ovn-nbctl lrp-add R1 rp-sw0 00:00:01:01:02:03 192.168.1.1/24
+check ovn-nbctl -- lrp-add R1 rp-public 00:00:02:01:02:03 172.16.1.1/24 \
+                -- lrp-set-options rp-public \
+                       maintain-vrf=true \
+                       redistribute-lb-vips=true
+
+check ovn-nbctl set logical_router R1 options:chassis=hv1
+
+check ovn-nbctl lsp-add sw0 sw0-rp -- set Logical_Switch_Port sw0-rp \
+    type=router options:router-port=rp-sw0 \
+    -- lsp-set-addresses sw0-rp router
+
+check ovn-nbctl lsp-add public public-rp -- set Logical_Switch_Port public-rp \
+    type=router options:router-port=rp-public \
+    -- lsp-set-addresses public-rp router
+
+check ovs-vsctl set Open_vSwitch . external-ids:ovn-bridge-mappings=phynet:br-ext
+
+check ovn-nbctl lsp-add public public1 \
+        -- lsp-set-addresses public1 unknown \
+        -- lsp-set-type public1 localnet \
+        -- lsp-set-options public1 network_name=phynet
+
+# Create a load balancer and associate to R1
+check ovn-nbctl lb-add lb1 172.16.1.150:80 172.16.1.100:80
+check ovn-nbctl lr-lb-add R1 lb1
+
+check ovn-nbctl --wait=hv sync
+
+AT_CHECK([ip link | grep -q ovnvrf1000:.*UP])
+AT_CHECK([test `ip route show table 1000 | wc -l` -eq 1])
+AT_CHECK([ip route show table 1000 | grep -q 172.16.1.150])
+
+
+OVS_APP_EXIT_AND_WAIT([ovn-controller])
+
+# XXX
+# Ensure system resources are cleaned up
+#AT_CHECK([ip link | grep -q ovnvrf1000:.*UP], [1])
+#AT_CHECK([test `ip route show table 1000 | wc -l` -eq 1], [1])
+
+as ovn-sb
+OVS_APP_EXIT_AND_WAIT([ovsdb-server])
+
+as ovn-nb
+OVS_APP_EXIT_AND_WAIT([ovsdb-server])
+
+as northd
+OVS_APP_EXIT_AND_WAIT([ovn-northd])
+
+as
+OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d
+/Failed to acquire.*/d
+/connection dropped.*/d"])
+AT_CLEANUP
+])
+
+OVN_FOR_EACH_NORTHD([
+AT_SETUP([route-exchange for LB VIPs with gateway router IPv6])
+AT_KEYWORDS([route-exchange])
+
+CHECK_CONNTRACK()
+CHECK_CONNTRACK_NAT()
+ovn_start
+OVS_TRAFFIC_VSWITCHD_START()
+ADD_BR([br-int])
+ADD_BR([br-ext], [set Bridge br-ext fail-mode=standalone])
+
+# Set external-ids in br-int needed for ovn-controller
+ovs-vsctl \
+        -- set Open_vSwitch . external-ids:system-id=hv1 \
+        -- set Open_vSwitch . external-ids:ovn-remote=unix:$ovs_base/ovn-sb/ovn-sb.sock \
+        -- set Open_vSwitch . external-ids:ovn-encap-type=geneve \
+        -- set Open_vSwitch . external-ids:ovn-encap-ip=169.0.0.1 \
+        -- set bridge br-int fail-mode=secure other-config:disable-in-band=true
+
+# Start ovn-controller
+start_daemon ovn-controller
+
+ovn-appctl vlog/set route_exchange
+check ovn-nbctl -- lr-add R1 \
+                -- set Logical_Router R1 options:requested-tnl-key=1001
+
+check ovn-nbctl ls-add sw0
+check ovn-nbctl ls-add public
+
+check ovn-nbctl lrp-add R1 rp-sw0 00:00:01:01:02:03 2001:db8:100::1/64
+check ovn-nbctl -- lrp-add R1 rp-public 00:00:02:01:02:03 2001:db8:1001::1/64 \
+                -- lrp-set-options rp-public \
+                       maintain-vrf=true \
+                       redistribute-lb-vips=true
+
+check ovn-nbctl set logical_router R1 options:chassis=hv1
+
+check ovn-nbctl lsp-add sw0 sw0-rp -- set Logical_Switch_Port sw0-rp \
+    type=router options:router-port=rp-sw0 \
+    -- lsp-set-addresses sw0-rp router
+
+check ovn-nbctl lsp-add public public-rp -- set Logical_Switch_Port public-rp \
+    type=router options:router-port=rp-public \
+    -- lsp-set-addresses public-rp router
+
+check ovs-vsctl set Open_vSwitch . external-ids:ovn-bridge-mappings=phynet:br-ext
+
+check ovn-nbctl lsp-add public public1 \
+        -- lsp-set-addresses public1 unknown \
+        -- lsp-set-type public1 localnet \
+        -- lsp-set-options public1 network_name=phynet
+
+# Create a load balancer and associate to R1
+check ovn-nbctl lb-add lb1 [[2001:db8:1001::150]]:80 [[2001:db8:1001::100]]:80
+check ovn-nbctl lr-lb-add R1 lb1
+
+check ovn-nbctl --wait=hv sync
+
+AT_CHECK([ip link | grep -q ovnvrf1001:.*UP])
+AT_CHECK([test `ip -6 route show table 1001 | wc -l` -eq 1])
+AT_CHECK([ip -6 route show table 1001 | grep -q 2001:db8:1001::150])
+
+
+OVS_APP_EXIT_AND_WAIT([ovn-controller])
+
+# XXX
+# Ensure system resources are cleaned up
+#AT_CHECK([ip link | grep -q ovnvrf1001:.*UP], [1])
+#AT_CHECK([test `ip -6 route show table 1001 | wc -l` -eq 1], [1])
+
+as ovn-sb
+OVS_APP_EXIT_AND_WAIT([ovsdb-server])
+
+as ovn-nb
+OVS_APP_EXIT_AND_WAIT([ovsdb-server])
+
+as northd
+OVS_APP_EXIT_AND_WAIT([ovn-northd])
+
+as
+OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d
+/Failed to acquire.*/d
+/connection dropped.*/d"])
+AT_CLEANUP
+])