Message ID | 20231128023527.569693-1-numans@ovn.org |
---|---|
State | Changes Requested |
Headers | show |
Series | northd lflow incremental processing | expand |
Context | Check | Description |
---|---|---|
ovsrobot/apply-robot | warning | apply and check: warning |
ovsrobot/github-robot-_Build_and_Test | fail | github build: failed |
ovsrobot/github-robot-_ovn-kubernetes | fail | github build: failed |
ovsrobot/github-robot-_Build_and_Test | fail | github build: failed |
ovsrobot/github-robot-_ovn-kubernetes | fail | github build: failed |
On 11/28/23 03:35, numans@ovn.org wrote: > From: Numan Siddique <numans@ovn.org> > > This new engine now maintains the load balancer and NAT data of a > logical router which was earlier part of northd engine node data. > The main inputs to this engine are: > - northd node > - lr_nat node > - lb_data node > > A record for each logical router is maintained in the 'lr_stateful_table' > hmap table and this record > - stores the lb related data > - embeds the 'lr_nat' record. > > This engine node becomes an input to 'lflow' node. > > Signed-off-by: Numan Siddique <numans@ovn.org> > --- > lib/stopwatch-names.h | 1 + > northd/automake.mk | 2 + > northd/en-lflow.c | 4 + > northd/en-lr-nat.h | 3 + > northd/en-lr-stateful.c | 647 ++++++++++++++++++++++++++++++++++++++ > northd/en-lr-stateful.h | 104 +++++++ > northd/en-sync-sb.c | 49 +-- > northd/inc-proc-northd.c | 13 +- > northd/northd.c | 655 ++++++++++++--------------------------- > northd/northd.h | 136 +++++++- > northd/ovn-northd.c | 1 + > tests/ovn-northd.at | 62 ++++ > 12 files changed, 1191 insertions(+), 486 deletions(-) > create mode 100644 northd/en-lr-stateful.c > create mode 100644 northd/en-lr-stateful.h > > diff --git a/lib/stopwatch-names.h b/lib/stopwatch-names.h > index 0a16da211e..63a04db9ca 100644 > --- a/lib/stopwatch-names.h > +++ b/lib/stopwatch-names.h > @@ -33,5 +33,6 @@ > #define PORT_GROUP_RUN_STOPWATCH_NAME "port_group_run" > #define SYNC_METERS_RUN_STOPWATCH_NAME "sync_meters_run" > #define LR_NAT_RUN_STOPWATCH_NAME "lr_nat_run" > +#define LR_STATEFUL_RUN_STOPWATCH_NAME "lr_stateful" > > #endif > diff --git a/northd/automake.mk b/northd/automake.mk > index ae367a2a8b..3666cd452c 100644 > --- a/northd/automake.mk > +++ b/northd/automake.mk > @@ -26,6 +26,8 @@ northd_ovn_northd_SOURCES = \ > northd/en-lb-data.h \ > northd/en-lr-nat.c \ > northd/en-lr-nat.h \ > + northd/en-lr-stateful.c \ > + northd/en-lr-stateful.h \ > northd/inc-proc-northd.c \ > northd/inc-proc-northd.h \ > northd/ipam.c \ > diff --git a/northd/en-lflow.c b/northd/en-lflow.c > index e4f875ef7c..0aee073520 100644 > --- a/northd/en-lflow.c > +++ b/northd/en-lflow.c > @@ -20,6 +20,7 @@ > > #include "en-lflow.h" > #include "en-lr-nat.h" > +#include "en-lr-stateful.h" > #include "en-northd.h" > #include "en-meters.h" > > @@ -43,6 +44,8 @@ lflow_get_input_data(struct engine_node *node, > engine_get_input_data("sync_meters", node); > struct ed_type_lr_nat_data *lr_nat_data = > engine_get_input_data("lr_nat", node); > + struct ed_type_lr_stateful *lr_sful_data = > + engine_get_input_data("lr_stateful", node); > > lflow_input->nbrec_bfd_table = > EN_OVSDB_GET(engine_get_input("NB_bfd", node)); > @@ -66,6 +69,7 @@ lflow_get_input_data(struct engine_node *node, > lflow_input->lr_ports = &northd_data->lr_ports; > lflow_input->ls_port_groups = &pg_data->ls_port_groups; > lflow_input->lr_nats = &lr_nat_data->lr_nats; > + lflow_input->lr_sful_table = &lr_sful_data->lr_sful_table; > lflow_input->meter_groups = &sync_meters_data->meter_groups; > lflow_input->lb_datapaths_map = &northd_data->lb_datapaths_map; > lflow_input->svc_monitor_map = &northd_data->svc_monitor_map; > diff --git a/northd/en-lr-nat.h b/northd/en-lr-nat.h > index 3ec4c7b506..4d8e142ae6 100644 > --- a/northd/en-lr-nat.h > +++ b/northd/en-lr-nat.h > @@ -86,6 +86,9 @@ struct lr_nat_table { > const struct lr_nat_record * lr_nat_table_find_by_index( > const struct lr_nat_table *, size_t od_index); > > +#define LR_NAT_TABLE_FOR_EACH(LR_NAT_REC, TABLE) \ > + HMAP_FOR_EACH (LR_NAT_REC, key_node, &(TABLE)->entries) > + > struct ed_type_lr_nat_data { > struct lr_nat_table lr_nats; > > diff --git a/northd/en-lr-stateful.c b/northd/en-lr-stateful.c > new file mode 100644 > index 0000000000..bd03c586da > --- /dev/null > +++ b/northd/en-lr-stateful.c > @@ -0,0 +1,647 @@ > +/* > + * Copyright (c) 2023, Red Hat, Inc. > + * > + * Licensed under the Apache License, Version 2.0 (the "License"); > + * you may not use this file except in compliance with the License. > + * You may obtain a copy of the License at: > + * > + * http://www.apache.org/licenses/LICENSE-2.0 > + * > + * Unless required by applicable law or agreed to in writing, software > + * distributed under the License is distributed on an "AS IS" BASIS, > + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. > + * See the License for the specific language governing permissions and > + * limitations under the License. > + */ > + > +#include <config.h> > + > +#include <getopt.h> > +#include <stdlib.h> > +#include <stdio.h> > + > +/* OVS includes */ > +#include "include/openvswitch/hmap.h" > +#include "lib/bitmap.h" > +#include "lib/socket-util.h" > +#include "lib/uuidset.h" > +#include "openvswitch/util.h" > +#include "openvswitch/vlog.h" > +#include "stopwatch.h" > + > +/* OVN includes */ > +#include "en-lb-data.h" > +#include "en-lr-nat.h" > +#include "en-lr-stateful.h" > +#include "lib/inc-proc-eng.h" > +#include "lib/lb.h" > +#include "lib/ovn-nb-idl.h" > +#include "lib/ovn-sb-idl.h" > +#include "lib/ovn-util.h" > +#include "lib/stopwatch-names.h" > +#include "northd.h" > + > +VLOG_DEFINE_THIS_MODULE(en_lr_stateful); > + > +/* Static function declarations. */ > +static void lr_stateful_table_init(struct lr_stateful_table *); > +static void lr_stateful_table_clear(struct lr_stateful_table *); > +static void lr_stateful_table_destroy(struct lr_stateful_table *); > +static struct lr_stateful_record *lr_stateful_table_find_( > + const struct lr_stateful_table *, const struct nbrec_logical_router *); > +static struct lr_stateful_record *lr_stateful_table_find_by_index_( > + const struct lr_stateful_table *table, size_t od_index); > + > +static void lr_stateful_table_build(struct lr_stateful_table *, > + const struct lr_nat_table *, > + const struct ovn_datapaths *lr_datapaths, > + const struct hmap *lb_datapaths_map, > + const struct hmap *lbgrp_datapaths_map); > + > +static struct lr_stateful_input lr_stateful_get_input_data( > + struct engine_node *); > + > +static struct lr_stateful_record *lr_stateful_record_create( > + struct lr_stateful_table *, const struct lr_nat_record *, > + const struct hmap *lb_datapaths_map, > + const struct hmap *lbgrp_datapaths_map); > +static void lr_stateful_record_destroy(struct lr_stateful_record *); > +static void lr_stateful_record_init( > + struct lr_stateful_record *, > + const struct hmap *lb_datapaths_map, > + const struct hmap *lbgrp_datapaths_map); > + > +static void build_lrouter_lb_reachable_ips(struct lr_stateful_record *, > + const struct ovn_northd_lb *); > +static void add_neigh_ips_to_lrouter(struct lr_stateful_record *, > + enum lb_neighbor_responder_mode, > + const struct sset *lb_ips_v4, > + const struct sset *lb_ips_v6); > +static void remove_lrouter_lb_reachable_ips(struct lr_stateful_record *, > + enum lb_neighbor_responder_mode, > + const struct sset *lb_ips_v4, > + const struct sset *lb_ips_v6); > +static void lr_stateful_build_vip_nats(struct lr_stateful_record *); > + > +/* 'lr_stateful' engine node manages the NB logical router LB data. > + */ > +void * > +en_lr_stateful_init(struct engine_node *node OVS_UNUSED, > + struct engine_arg *arg OVS_UNUSED) > +{ > + struct ed_type_lr_stateful *data = xzalloc(sizeof *data); > + lr_stateful_table_init(&data->lr_sful_table); > + hmapx_init(&data->trk_data.crupdated); > + return data; > +} > + > +void > +en_lr_stateful_cleanup(void *data_) > +{ > + struct ed_type_lr_stateful *data = > + (struct ed_type_lr_stateful *) data_; > + lr_stateful_table_destroy(&data->lr_sful_table); > + hmapx_destroy(&data->trk_data.crupdated); > +} > + > +void > +en_lr_stateful_clear_tracked_data(void *data_) > +{ > + struct ed_type_lr_stateful *data = (struct ed_type_lr_stateful *) data_; > + > + hmapx_clear(&data->trk_data.crupdated); > +} > + > +void > +en_lr_stateful_run(struct engine_node *node, void *data_) > +{ > + struct lr_stateful_input input_data = lr_stateful_get_input_data(node); > + struct ed_type_lr_stateful *data = data_; > + > + stopwatch_start(LR_STATEFUL_RUN_STOPWATCH_NAME, time_msec()); > + > + lr_stateful_table_clear(&data->lr_sful_table); > + lr_stateful_table_build(&data->lr_sful_table, input_data.lr_nats, > + input_data.lr_datapaths, > + input_data.lb_datapaths_map, > + input_data.lbgrp_datapaths_map); > + > + stopwatch_stop(LR_STATEFUL_RUN_STOPWATCH_NAME, time_msec()); > + engine_set_node_state(node, EN_UPDATED); > +} > + > +bool > +lr_stateful_northd_handler(struct engine_node *node, void *data OVS_UNUSED) > +{ > + struct northd_data *northd_data = engine_get_input_data("northd", node); > + if (!northd_has_tracked_data(&northd_data->trk_data)) { > + return false; > + } > + > + /* lr_stateful node needs inputs for any changes to NAT and load balancers. > + * Changes to NAT is provided by the lr_nat tracked data and changes > + * to lbs and lb grps is provided by lb_data's tracked data. > + * So we don't need to do anything here for northd changes. > + * But we do need to access the datapaths and lb_datapaths from the > + * northd engine node and hence its an input. > + * */ > + return true; > +} > + > +bool > +lr_stateful_lb_data_handler(struct engine_node *node, void *data_) > +{ > + struct ed_type_lb_data *lb_data = engine_get_input_data("lb_data", node); > + if (!lb_data->tracked) { > + return false; > + } > + > + struct ed_type_lr_stateful *data = > + (struct ed_type_lr_stateful *) data_; > + struct lr_stateful_input input_data = > + lr_stateful_get_input_data(node); > + struct lr_stateful_record *lr_sful_rec; I'd move this definition (lr_sful_rec) inside of the appropriate loops where this is used. That's to avoid it having unexpected values. > + size_t index; > + > + const struct tracked_lb_data *trk_lb_data = &lb_data->tracked_lb_data; > + const struct ovn_lb_group_datapaths *lbgrp_dps; > + const struct crupdated_lbgrp *crupdated_lbgrp; > + const struct crupdated_od_lb_data *codlb; > + const struct ovn_lb_datapaths *lb_dps; > + const struct crupdated_lb *clb; > + const struct ovn_northd_lb *lb; > + const struct ovn_datapath *od; Same comment about these, let's reduce the scope of the variables if possible. > + > + LIST_FOR_EACH (codlb, list_node, &trk_lb_data->crupdated_lr_lbs) { > + od = ovn_datapath_find(&input_data.lr_datapaths->datapaths, > + &codlb->od_uuid); > + ovs_assert(od); > + > + lr_sful_rec = lr_stateful_table_find_(&data->lr_sful_table, od->nbr); > + if (!lr_sful_rec) { > + const struct lr_nat_record *lrnat_rec = lr_nat_table_find_by_index( > + input_data.lr_nats, od->index); > + ovs_assert(lrnat_rec); > + > + lr_sful_rec = lr_stateful_record_create(&data->lr_sful_table, > + lrnat_rec, > + input_data.lb_datapaths_map, > + input_data.lbgrp_datapaths_map); > + > + /* Add the lr_sful_rec rec to the tracking data. */ > + hmapx_add(&data->trk_data.crupdated, lr_sful_rec); > + continue; > + } > + Nit: it might help if we had a comment here to remind the reader that we're now in the "update" case, e.g.: /* Update. */ > + struct uuidset_node *uuidnode; > + UUIDSET_FOR_EACH (uuidnode, &codlb->assoc_lbs) { > + lb_dps = ovn_lb_datapaths_find( > + input_data.lb_datapaths_map, &uuidnode->uuid); > + ovs_assert(lb_dps); > + > + /* Add the lb_ips of lb_dps to the od. */ > + build_lrouter_lb_ips(lr_sful_rec->lb_ips, lb_dps->lb); > + build_lrouter_lb_reachable_ips(lr_sful_rec, lb_dps->lb); > + } > + > + UUIDSET_FOR_EACH (uuidnode, &codlb->assoc_lbgrps) { > + lbgrp_dps = ovn_lb_group_datapaths_find( > + input_data.lbgrp_datapaths_map, &uuidnode->uuid); > + ovs_assert(lbgrp_dps); > + > + for (size_t j = 0; j < lbgrp_dps->lb_group->n_lbs; j++) { > + const struct uuid *lb_uuid > + = &lbgrp_dps->lb_group->lbs[j]->nlb->header_.uuid; > + lb_dps = ovn_lb_datapaths_find(input_data.lb_datapaths_map, > + lb_uuid); > + ovs_assert(lb_dps); > + > + /* Add the lb_ips of lb_dps to the od. */ > + build_lrouter_lb_ips(lr_sful_rec->lb_ips, lb_dps->lb); > + build_lrouter_lb_reachable_ips(lr_sful_rec, lb_dps->lb); > + } > + } > + > + /* Add the lr_sful_rec rec to the tracking data. */ > + hmapx_add(&data->trk_data.crupdated, lr_sful_rec); > + } > + > + HMAP_FOR_EACH (clb, hmap_node, &trk_lb_data->crupdated_lbs) { > + lb = clb->lb; > + const struct uuid *lb_uuid = &lb->nlb->header_.uuid; > + > + lb_dps = ovn_lb_datapaths_find(input_data.lb_datapaths_map, lb_uuid); > + ovs_assert(lb_dps); > + > + BITMAP_FOR_EACH_1 (index, ods_size(input_data.lr_datapaths), > + lb_dps->nb_lr_map) { > + od = input_data.lr_datapaths->array[index]; > + > + lr_sful_rec = lr_stateful_table_find_(&data->lr_sful_table, > + od->nbr); Nit: indent. > + ovs_assert(lr_sful_rec); > + > + /* Update the od->lb_ips with the deleted and inserted > + * vips (if any). */ > + remove_ips_from_lb_ip_set(lr_sful_rec->lb_ips, lb->routable, > + &clb->deleted_vips_v4, > + &clb->deleted_vips_v6); > + add_ips_to_lb_ip_set(lr_sful_rec->lb_ips, lb->routable, > + &clb->inserted_vips_v4, > + &clb->inserted_vips_v6); > + > + remove_lrouter_lb_reachable_ips(lr_sful_rec, lb->neigh_mode, > + &clb->deleted_vips_v4, > + &clb->deleted_vips_v6); > + add_neigh_ips_to_lrouter(lr_sful_rec, lb->neigh_mode, > + &clb->inserted_vips_v4, > + &clb->inserted_vips_v6); > + > + /* Add the lr_sful_rec rec to the tracking data. */ > + hmapx_add(&data->trk_data.crupdated, lr_sful_rec); > + } > + } > + > + HMAP_FOR_EACH (crupdated_lbgrp, hmap_node, > + &trk_lb_data->crupdated_lbgrps) { > + const struct uuid *lb_uuid = &crupdated_lbgrp->lbgrp->uuid; > + > + lbgrp_dps = ovn_lb_group_datapaths_find(input_data.lbgrp_datapaths_map, > + lb_uuid); > + ovs_assert(lbgrp_dps); > + > + struct hmapx_node *hnode; > + HMAPX_FOR_EACH (hnode, &crupdated_lbgrp->assoc_lbs) { > + lb = hnode->data; > + lb_uuid = &lb->nlb->header_.uuid; > + lb_dps = ovn_lb_datapaths_find(input_data.lb_datapaths_map, > + lb_uuid); > + ovs_assert(lb_dps); > + for (size_t i = 0; i < lbgrp_dps->n_lr; i++) { > + od = lbgrp_dps->lr[i]; > + lr_sful_rec = lr_stateful_table_find_(&data->lr_sful_table, > + od->nbr); > + ovs_assert(lr_sful_rec); > + /* Add the lb_ips of lb_dps to the lr lb data. */ > + build_lrouter_lb_ips(lr_sful_rec->lb_ips, lb_dps->lb); > + build_lrouter_lb_reachable_ips(lr_sful_rec, lb_dps->lb); > + > + /* Add the lr_sful_rec rec to the tracking data. */ > + hmapx_add(&data->trk_data.crupdated, lr_sful_rec); > + } > + } > + } > + > + if (lr_stateful_has_tracked_data(&data->trk_data)) { > + struct hmapx_node *hmapx_node; > + /* For all the modified lr_stateful records (re)build the > + * vip nats. */ > + HMAPX_FOR_EACH (hmapx_node, &data->trk_data.crupdated) { > + lr_stateful_build_vip_nats(hmapx_node->data); > + } > + > + engine_set_node_state(node, EN_UPDATED); > + } > + > + return true; > +} > + > +bool > +lr_stateful_lr_nat_handler(struct engine_node *node, void *data_) > +{ > + struct ed_type_lr_nat_data *lr_nat_data = > + engine_get_input_data("lr_nat", node); > + > + if (!lr_nat_has_tracked_data(&lr_nat_data->trk_data)) { > + return false; > + } > + > + struct ed_type_lr_stateful *data = > + (struct ed_type_lr_stateful *) data_; > + struct lr_stateful_input input_data = > + lr_stateful_get_input_data(node); > + const struct lr_nat_record *lrnat_rec; > + struct lr_stateful_record *lr_sful_rec; > + struct hmapx_node *hmapx_node; > + > + HMAPX_FOR_EACH (hmapx_node, &lr_nat_data->trk_data.crupdated) { > + lrnat_rec = hmapx_node->data; > + lr_sful_rec = lr_stateful_table_find_(&data->lr_sful_table, > + lrnat_rec->od->nbr); > + if (!lr_sful_rec) { > + lr_sful_rec = lr_stateful_record_create(&data->lr_sful_table, > + lrnat_rec, > + input_data.lb_datapaths_map, > + input_data.lbgrp_datapaths_map); > + } else { > + lr_stateful_build_vip_nats(lr_sful_rec); > + } > + > + /* Add the lr_sful_rec rec to the tracking data. */ > + hmapx_add(&data->trk_data.crupdated, lr_sful_rec); > + } > + > + if (lr_stateful_has_tracked_data(&data->trk_data)) { > + engine_set_node_state(node, EN_UPDATED); > + } > + > + return true; > +} > + > +const struct lr_stateful_record * > +lr_stateful_table_find_by_index(const struct lr_stateful_table *table, > + size_t od_index) > +{ > + return lr_stateful_table_find_by_index_(table, od_index); > +} > + > +/* static functions. */ > +static void > +lr_stateful_table_init(struct lr_stateful_table *table) > +{ > + *table = (struct lr_stateful_table) { > + .entries = HMAP_INITIALIZER(&table->entries), > + }; > +} > + > +static void > +lr_stateful_table_destroy(struct lr_stateful_table *table) > +{ > + lr_stateful_table_clear(table); > + hmap_destroy(&table->entries); > +} > + > +static void > +lr_stateful_table_clear(struct lr_stateful_table *table) > +{ > + struct lr_stateful_record *lr_sful_rec; > + HMAP_FOR_EACH_POP (lr_sful_rec, key_node, &table->entries) { > + lr_stateful_record_destroy(lr_sful_rec); > + } > + > + free(table->array); > + table->array = NULL; > +} > + > +static void > +lr_stateful_table_build(struct lr_stateful_table *table, > + const struct lr_nat_table *lr_nats, > + const struct ovn_datapaths *lr_datapaths, > + const struct hmap *lb_datapaths_map, > + const struct hmap *lbgrp_datapaths_map) > +{ > + table->array = xrealloc(table->array, > + ods_size(lr_datapaths) * sizeof *table->array); > + const struct lr_nat_record *lrnat_rec; > + LR_NAT_TABLE_FOR_EACH (lrnat_rec, lr_nats) { > + lr_stateful_record_create(table, lrnat_rec, lb_datapaths_map, > + lbgrp_datapaths_map); > + } > +} > + > +static struct lr_stateful_record * > +lr_stateful_table_find_(const struct lr_stateful_table *table, > + const struct nbrec_logical_router *nbr) > +{ > + struct lr_stateful_record *lr_sful_rec; > + > + HMAP_FOR_EACH_WITH_HASH (lr_sful_rec, key_node, > + uuid_hash(&nbr->header_.uuid), &table->entries) { > + if (nbr == lr_sful_rec->od->nbr) { > + return lr_sful_rec; > + } > + } > + return NULL; > +} > + > +static struct lr_stateful_record * > +lr_stateful_table_find_by_index_(const struct lr_stateful_table *table, > + size_t od_index) > +{ > + ovs_assert(od_index <= hmap_count(&table->entries)); > + return table->array[od_index]; > +} > + > +static struct lr_stateful_record * > +lr_stateful_record_create(struct lr_stateful_table *table, > + const struct lr_nat_record *lrnat_rec, > + const struct hmap *lb_datapaths_map, > + const struct hmap *lbgrp_datapaths_map) > +{ > + struct lr_stateful_record *lr_sful_rec = xzalloc(sizeof *lr_sful_rec); > + lr_sful_rec->lrnat_rec = lrnat_rec; > + lr_sful_rec->od = lrnat_rec->od; > + lr_stateful_record_init(lr_sful_rec, lb_datapaths_map, > + lbgrp_datapaths_map); > + > + hmap_insert(&table->entries, &lr_sful_rec->key_node, > + uuid_hash(&lr_sful_rec->od->nbr->header_.uuid)); > + > + table->array[lr_sful_rec->od->index] = lr_sful_rec; > + return lr_sful_rec; > +} > + > +static void > +lr_stateful_record_destroy(struct lr_stateful_record *lr_sful_rec) > +{ > + ovn_lb_ip_set_destroy(lr_sful_rec->lb_ips); > + lr_sful_rec->lb_ips = NULL; > + sset_destroy(&lr_sful_rec->vip_nats); > + free(lr_sful_rec); > +} > + > +static void > +lr_stateful_record_init(struct lr_stateful_record *lr_sful_rec, > + const struct hmap *lb_datapaths_map, > + const struct hmap *lbgrp_datapaths_map) > +{ > + const struct nbrec_load_balancer_group *nbrec_lb_group; > + const struct ovn_lb_group_datapaths *lb_group_dps; > + const struct ovn_lb_datapaths *lb_dps; > + > + /* Checking load balancer groups first, starting from the largest one, > + * to more efficiently copy IP sets. */ > + size_t largest_group = 0; > + > + const struct nbrec_logical_router *nbr = lr_sful_rec->od->nbr; > + for (size_t i = 1; i < nbr->n_load_balancer_group; i++) { > + if (nbr->load_balancer_group[i]->n_load_balancer > > + nbr->load_balancer_group[largest_group]->n_load_balancer) { > + largest_group = i; > + } > + } > + > + for (size_t i = 0; i < nbr->n_load_balancer_group; i++) { > + size_t idx = (i + largest_group) % nbr->n_load_balancer_group; > + > + nbrec_lb_group = nbr->load_balancer_group[idx]; > + const struct uuid *lbgrp_uuid = &nbrec_lb_group->header_.uuid; > + > + lb_group_dps = > + ovn_lb_group_datapaths_find(lbgrp_datapaths_map, > + lbgrp_uuid); > + ovs_assert(lb_group_dps); > + > + if (!lr_sful_rec->lb_ips) { > + lr_sful_rec->lb_ips = > + ovn_lb_ip_set_clone(lb_group_dps->lb_group->lb_ips); > + } else { > + for (size_t j = 0; j < lb_group_dps->lb_group->n_lbs; j++) { > + build_lrouter_lb_ips(lr_sful_rec->lb_ips, > + lb_group_dps->lb_group->lbs[j]); > + } > + } > + > + for (size_t j = 0; j < lb_group_dps->lb_group->n_lbs; j++) { > + build_lrouter_lb_reachable_ips(lr_sful_rec, > + lb_group_dps->lb_group->lbs[j]); > + } > + } > + > + if (!lr_sful_rec->lb_ips) { > + lr_sful_rec->lb_ips = ovn_lb_ip_set_create(); > + } > + > + for (size_t i = 0; i < nbr->n_load_balancer; i++) { > + const struct uuid *lb_uuid = > + &nbr->load_balancer[i]->header_.uuid; > + lb_dps = ovn_lb_datapaths_find(lb_datapaths_map, lb_uuid); > + ovs_assert(lb_dps); > + build_lrouter_lb_ips(lr_sful_rec->lb_ips, lb_dps->lb); > + build_lrouter_lb_reachable_ips(lr_sful_rec, lb_dps->lb); > + } > + > + sset_init(&lr_sful_rec->vip_nats); > + > + if (!nbr->n_nat) { > + lr_stateful_build_vip_nats(lr_sful_rec); > + } > +} > + > +static struct lr_stateful_input > +lr_stateful_get_input_data(struct engine_node *node) > +{ > + struct northd_data *northd_data = engine_get_input_data("northd", node); > + struct ed_type_lr_nat_data *lr_nat_data = > + engine_get_input_data("lr_nat", node); > + > + return (struct lr_stateful_input) { > + .lr_datapaths = &northd_data->lr_datapaths, > + .lb_datapaths_map = &northd_data->lb_datapaths_map, > + .lbgrp_datapaths_map = &northd_data->lb_group_datapaths_map, > + .lr_nats = &lr_nat_data->lr_nats, > + }; > +} > + > +static void > +build_lrouter_lb_reachable_ips(struct lr_stateful_record *lr_sful_rec, > + const struct ovn_northd_lb *lb) > +{ > + add_neigh_ips_to_lrouter(lr_sful_rec, lb->neigh_mode, &lb->ips_v4, > + &lb->ips_v6); > +} > + > +static void > +add_neigh_ips_to_lrouter(struct lr_stateful_record *lr_sful_rec, > + enum lb_neighbor_responder_mode neigh_mode, > + const struct sset *lb_ips_v4, > + const struct sset *lb_ips_v6) > +{ > + /* If configured to not reply to any neighbor requests for all VIPs > + * return early. > + */ > + if (neigh_mode == LB_NEIGH_RESPOND_NONE) { > + return; > + } > + > + const char *ip_address; > + > + /* If configured to reply to neighbor requests for all VIPs force them > + * all to be considered "reachable". > + */ > + if (neigh_mode == LB_NEIGH_RESPOND_ALL) { > + SSET_FOR_EACH (ip_address, lb_ips_v4) { > + sset_add(&lr_sful_rec->lb_ips->ips_v4_reachable, ip_address); > + } > + SSET_FOR_EACH (ip_address, lb_ips_v6) { > + sset_add(&lr_sful_rec->lb_ips->ips_v6_reachable, ip_address); > + } > + > + return; > + } > + > + /* Otherwise, a VIP is reachable if there's at least one router > + * subnet that includes it. > + */ > + ovs_assert(neigh_mode == LB_NEIGH_RESPOND_REACHABLE); > + > + SSET_FOR_EACH (ip_address, lb_ips_v4) { > + struct ovn_port *op; > + ovs_be32 vip_ip4; > + if (ip_parse(ip_address, &vip_ip4)) { > + HMAP_FOR_EACH (op, dp_node, &lr_sful_rec->od->ports) { > + if (lrouter_port_ipv4_reachable(op, vip_ip4)) { > + sset_add(&lr_sful_rec->lb_ips->ips_v4_reachable, > + ip_address); > + break; > + } > + } > + } > + } > + > + SSET_FOR_EACH (ip_address, lb_ips_v6) { > + struct ovn_port *op; > + struct in6_addr vip; > + if (ipv6_parse(ip_address, &vip)) { > + HMAP_FOR_EACH (op, dp_node, &lr_sful_rec->od->ports) { > + if (lrouter_port_ipv6_reachable(op, &vip)) { > + sset_add(&lr_sful_rec->lb_ips->ips_v6_reachable, > + ip_address); > + break; > + } > + } > + } > + } > +} > + > +static void > +remove_lrouter_lb_reachable_ips(struct lr_stateful_record *lr_sful_rec, > + enum lb_neighbor_responder_mode neigh_mode, > + const struct sset *lb_ips_v4, > + const struct sset *lb_ips_v6) > +{ > + if (neigh_mode == LB_NEIGH_RESPOND_NONE) { > + return; > + } > + > + const char *ip_address; > + SSET_FOR_EACH (ip_address, lb_ips_v4) { > + sset_find_and_delete(&lr_sful_rec->lb_ips->ips_v4_reachable, > + ip_address); > + } > + SSET_FOR_EACH (ip_address, lb_ips_v6) { > + sset_find_and_delete(&lr_sful_rec->lb_ips->ips_v6_reachable, > + ip_address); > + } > +} > + > +static void > +lr_stateful_build_vip_nats(struct lr_stateful_record *lr_sful_rec) > +{ > + sset_clear(&lr_sful_rec->vip_nats); > + const char *external_ip; > + SSET_FOR_EACH (external_ip, &lr_sful_rec->lrnat_rec->external_ips) { > + bool is_vip_nat = false; > + if (addr_is_ipv6(external_ip)) { > + is_vip_nat = sset_contains(&lr_sful_rec->lb_ips->ips_v6, > + external_ip); > + } else { > + is_vip_nat = sset_contains(&lr_sful_rec->lb_ips->ips_v4, > + external_ip); > + } > + > + if (is_vip_nat) { > + sset_add(&lr_sful_rec->vip_nats, external_ip); > + } > + } > +} > diff --git a/northd/en-lr-stateful.h b/northd/en-lr-stateful.h > new file mode 100644 > index 0000000000..a72f000d36 > --- /dev/null > +++ b/northd/en-lr-stateful.h > @@ -0,0 +1,104 @@ > +/* > + * Copyright (c) 2023, Red Hat, Inc. > + * > + * Licensed under the Apache License, Version 2.0 (the "License"); > + * you may not use this file except in compliance with the License. > + * You may obtain a copy of the License at: > + * > + * http://www.apache.org/licenses/LICENSE-2.0 > + * > + * Unless required by applicable law or agreed to in writing, software > + * distributed under the License is distributed on an "AS IS" BASIS, > + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. > + * See the License for the specific language governing permissions and > + * limitations under the License. > + */ > +#ifndef EN_LR_STATEFUL_H > +#define EN_LR_STATEFUL_H 1 > + > +#include <stdint.h> > + > +/* OVS includes. */ > +#include "lib/hmapx.h" > +#include "openvswitch/hmap.h" > +#include "sset.h" > + > +/* OVN includes. */ > +#include "lib/inc-proc-eng.h" > +#include "lib/lb.h" > +#include "lib/ovn-nb-idl.h" > +#include "lib/ovn-sb-idl.h" > +#include "lib/ovn-util.h" > + > +struct ovn_datapath; > +struct lr_nat_record; > + > +/* lr_stateful_table: This represents a table of logical routers with > + * stateful related data. > + * stateful related data has two main components > + * - NAT and > + * - Load balancers. > + * > + * lr_stateful_record: It is a record in the lr_stateful_table for each > + * logical router. > + */ > + > +struct lr_stateful_record { > + struct hmap_node key_node; /* Index on 'nbr->header_.uuid'. */ > + > + const struct ovn_datapath *od; > + const struct lr_nat_record *lrnat_rec; > + > + /* Load Balancer vIPs relevant for this datapath. */ > + struct ovn_lb_ip_set *lb_ips; > + > + /* sset of vips which are also part of lr nats. */ > + struct sset vip_nats; > +}; > + > +struct lr_stateful_table { > + struct hmap entries; > + > + /* The array index of each element in 'entries'. */ > + struct lr_stateful_record **array; > +}; > + > +#define LR_STATEFUL_TABLE_FOR_EACH(LR_LB_NAT_REC, TABLE) \ > + HMAP_FOR_EACH (LR_LB_NAT_REC, key_node, &(TABLE)->entries) > + > +struct lr_stateful_tracked_data { > + /* Created or updated logical router with LB and/or NAT data. */ > + struct hmapx crupdated; /* Stores 'struct lr_stateful_record'. */ > +}; > + > +struct ed_type_lr_stateful { > + struct lr_stateful_table lr_sful_table; > + > + struct lr_stateful_tracked_data trk_data; > +}; > + > +struct lr_stateful_input { > + const struct ovn_datapaths *lr_datapaths; > + const struct hmap *lb_datapaths_map; > + const struct hmap *lbgrp_datapaths_map; > + const struct lr_nat_table *lr_nats; > +}; > + > +void *en_lr_stateful_init(struct engine_node *, struct engine_arg *); > +void en_lr_stateful_cleanup(void *data); > +void en_lr_stateful_clear_tracked_data(void *data); > +void en_lr_stateful_run(struct engine_node *, void *data); > + > +bool lr_stateful_northd_handler(struct engine_node *, void *data); > +bool lr_stateful_lr_nat_handler(struct engine_node *, void *data); > +bool lr_stateful_lb_data_handler(struct engine_node *, void *data); > + > +const struct lr_stateful_record *lr_stateful_table_find_by_index( > + const struct lr_stateful_table *, size_t od_index); > + > +static inline bool > +lr_stateful_has_tracked_data(struct lr_stateful_tracked_data *trk_data) { > + return !hmapx_is_empty(&trk_data->crupdated); > +} > + > +#endif /* EN_lr_stateful_H */ > diff --git a/northd/en-sync-sb.c b/northd/en-sync-sb.c > index 11e12428f7..733fb9024e 100644 > --- a/northd/en-sync-sb.c > +++ b/northd/en-sync-sb.c > @@ -22,6 +22,7 @@ > #include "openvswitch/util.h" > > #include "en-lr-nat.h" > +#include "en-lr-stateful.h" > #include "en-sync-sb.h" > #include "lib/inc-proc-eng.h" > #include "lib/lb.h" > @@ -41,7 +42,7 @@ static void sync_addr_sets(struct ovsdb_idl_txn *ovnsb_txn, > const struct nbrec_address_set_table *, > const struct nbrec_port_group_table *, > const struct sbrec_address_set_table *, > - const struct ovn_datapaths *lr_datapaths); > + const struct lr_stateful_table *); > static const struct sbrec_address_set *sb_address_set_lookup_by_name( > struct ovsdb_idl_index *, const char *name); > static void update_sb_addr_set(struct sorted_array *, > @@ -87,11 +88,11 @@ en_sync_to_sb_addr_set_run(struct engine_node *node, void *data OVS_UNUSED) > EN_OVSDB_GET(engine_get_input("SB_address_set", node)); > > const struct engine_context *eng_ctx = engine_get_context(); > - struct northd_data *northd_data = engine_get_input_data("northd", node); > - > + const struct ed_type_lr_stateful *lr_stateful_data = > + engine_get_input_data("lr_stateful", node); > sync_addr_sets(eng_ctx->ovnsb_idl_txn, nb_address_set_table, > nb_port_group_table, sb_address_set_table, > - &northd_data->lr_datapaths); > + &lr_stateful_data->lr_sful_table); > > engine_set_node_state(node, EN_UPDATED); > } > @@ -288,10 +289,12 @@ en_sync_to_sb_pb_run(struct engine_node *node, void *data OVS_UNUSED) > { > const struct engine_context *eng_ctx = engine_get_context(); > struct northd_data *northd_data = engine_get_input_data("northd", node); > - struct ed_type_lr_nat_data *lr_nat_data = > - engine_get_input_data("lr_nat", node); > + struct ed_type_lr_stateful *lr_stateful_data = > + engine_get_input_data("lr_stateful", node); > + > sync_pbs(eng_ctx->ovnsb_idl_txn, &northd_data->ls_ports, > - &northd_data->lr_ports, &lr_nat_data->lr_nats); > + &northd_data->lr_ports, > + &lr_stateful_data->lr_sful_table); > engine_set_node_state(node, EN_UPDATED); > } > > @@ -316,7 +319,11 @@ sync_to_sb_pb_northd_handler(struct engine_node *node, void *data OVS_UNUSED) > return false; > } > > - if (!sync_pbs_for_northd_changed_ovn_ports(&nd->trk_data.trk_lsps)) { > + struct ed_type_lr_stateful *lr_sful_data = > + engine_get_input_data("lr_stateful", node); > + > + if (!sync_pbs_for_northd_changed_ovn_ports(&nd->trk_data.trk_lsps, > + &lr_sful_data->lr_sful_table)) { > return false; > } > > @@ -362,7 +369,7 @@ sync_addr_sets(struct ovsdb_idl_txn *ovnsb_txn, > const struct nbrec_address_set_table *nb_address_set_table, > const struct nbrec_port_group_table *nb_port_group_table, > const struct sbrec_address_set_table *sb_address_set_table, > - const struct ovn_datapaths *lr_datapaths) > + const struct lr_stateful_table *lr_statefuls) > { > struct shash sb_address_sets = SHASH_INITIALIZER(&sb_address_sets); > > @@ -406,16 +413,14 @@ sync_addr_sets(struct ovsdb_idl_txn *ovnsb_txn, > } > > /* Sync router load balancer VIP generated address sets. */ > - struct ovn_datapath *od; > - HMAP_FOR_EACH (od, key_node, &lr_datapaths->datapaths) { > - ovs_assert(od->nbr); > - > - if (sset_count(&od->lb_ips->ips_v4_reachable)) { > - char *ipv4_addrs_name = lr_lb_address_set_name(od->tunnel_key, > - AF_INET); > + const struct lr_stateful_record *lr_sful_rec; > + LR_STATEFUL_TABLE_FOR_EACH (lr_sful_rec, lr_statefuls) { > + if (sset_count(&lr_sful_rec->lb_ips->ips_v4_reachable)) { > + char *ipv4_addrs_name = > + lr_lb_address_set_name(lr_sful_rec->od->tunnel_key, AF_INET); > > struct sorted_array ipv4_addrs_sorted = > - sorted_array_from_sset(&od->lb_ips->ips_v4_reachable); > + sorted_array_from_sset(&lr_sful_rec->lb_ips->ips_v4_reachable); > > sync_addr_set(ovnsb_txn, ipv4_addrs_name, > &ipv4_addrs_sorted, &sb_address_sets); > @@ -423,11 +428,11 @@ sync_addr_sets(struct ovsdb_idl_txn *ovnsb_txn, > free(ipv4_addrs_name); > } > > - if (sset_count(&od->lb_ips->ips_v6_reachable)) { > - char *ipv6_addrs_name = lr_lb_address_set_name(od->tunnel_key, > - AF_INET6); > - struct sorted_array ipv6_addrs_sorted = > - sorted_array_from_sset(&od->lb_ips->ips_v6_reachable); > + if (sset_count(&lr_sful_rec->lb_ips->ips_v6_reachable)) { > + char *ipv6_addrs_name = > + lr_lb_address_set_name(lr_sful_rec->od->tunnel_key, AF_INET6); > + struct sorted_array ipv6_addrs_sorted = sorted_array_from_sset( > + &lr_sful_rec->lb_ips->ips_v6_reachable); > > sync_addr_set(ovnsb_txn, ipv6_addrs_name, > &ipv6_addrs_sorted, &sb_address_sets); > diff --git a/northd/inc-proc-northd.c b/northd/inc-proc-northd.c > index 1f211b278e..97bcce9655 100644 > --- a/northd/inc-proc-northd.c > +++ b/northd/inc-proc-northd.c > @@ -31,6 +31,7 @@ > #include "openvswitch/vlog.h" > #include "inc-proc-northd.h" > #include "en-lb-data.h" > +#include "en-lr-stateful.h" > #include "en-lr-nat.h" > #include "en-northd.h" > #include "en-lflow.h" > @@ -148,6 +149,7 @@ static ENGINE_NODE(sync_to_sb_lb, "sync_to_sb_lb"); > static ENGINE_NODE(sync_to_sb_pb, "sync_to_sb_pb"); > static ENGINE_NODE_WITH_CLEAR_TRACK_DATA(lb_data, "lb_data"); > static ENGINE_NODE_WITH_CLEAR_TRACK_DATA(lr_nat, "lr_nat"); > +static ENGINE_NODE_WITH_CLEAR_TRACK_DATA(lr_stateful, "lr_stateful"); > > void inc_proc_northd_init(struct ovsdb_idl_loop *nb, > struct ovsdb_idl_loop *sb) > @@ -193,6 +195,11 @@ void inc_proc_northd_init(struct ovsdb_idl_loop *nb, > > engine_add_input(&en_lr_nat, &en_northd, lr_nat_northd_handler); > > + engine_add_input(&en_lr_stateful, &en_northd, lr_stateful_northd_handler); > + engine_add_input(&en_lr_stateful, &en_lr_nat, lr_stateful_lr_nat_handler); > + engine_add_input(&en_lr_stateful, &en_lb_data, > + lr_stateful_lb_data_handler); > + > engine_add_input(&en_mac_binding_aging, &en_nb_nb_global, NULL); > engine_add_input(&en_mac_binding_aging, &en_sb_mac_binding, NULL); > engine_add_input(&en_mac_binding_aging, &en_northd, NULL); > @@ -214,15 +221,17 @@ void inc_proc_northd_init(struct ovsdb_idl_loop *nb, > engine_add_input(&en_lflow, &en_sb_logical_flow, NULL); > engine_add_input(&en_lflow, &en_sb_multicast_group, NULL); > engine_add_input(&en_lflow, &en_sb_igmp_group, NULL); > + engine_add_input(&en_lflow, &en_lr_nat, NULL); > + engine_add_input(&en_lflow, &en_lr_stateful, NULL); > engine_add_input(&en_lflow, &en_northd, lflow_northd_handler); > engine_add_input(&en_lflow, &en_port_group, lflow_port_group_handler); > - engine_add_input(&en_lflow, &en_lr_nat, NULL); > > engine_add_input(&en_sync_to_sb_addr_set, &en_nb_address_set, > sync_to_sb_addr_set_nb_address_set_handler); > engine_add_input(&en_sync_to_sb_addr_set, &en_nb_port_group, > sync_to_sb_addr_set_nb_port_group_handler); > engine_add_input(&en_sync_to_sb_addr_set, &en_northd, NULL); > + engine_add_input(&en_sync_to_sb_addr_set, &en_lr_stateful, NULL); > engine_add_input(&en_sync_to_sb_addr_set, &en_sb_address_set, NULL); > > engine_add_input(&en_port_group, &en_nb_port_group, > @@ -240,7 +249,7 @@ void inc_proc_northd_init(struct ovsdb_idl_loop *nb, > > engine_add_input(&en_sync_to_sb_pb, &en_northd, > sync_to_sb_pb_northd_handler); > - engine_add_input(&en_sync_to_sb_pb, &en_lr_nat, NULL); > + engine_add_input(&en_sync_to_sb_pb, &en_lr_stateful, NULL); > > /* en_sync_to_sb engine node syncs the SB database tables from > * the NB database tables. > diff --git a/northd/northd.c b/northd/northd.c > index 7f59415bbd..e9b8332e21 100644 > --- a/northd/northd.c > +++ b/northd/northd.c > @@ -44,6 +44,7 @@ > #include "northd.h" > #include "en-lb-data.h" > #include "en-lr-nat.h" > +#include "en-lr-stateful.h" > #include "lib/ovn-parallel-hmap.h" > #include "ovn/actions.h" > #include "ovn/features.h" > @@ -617,13 +618,6 @@ init_lb_for_datapath(struct ovn_datapath *od) > } > } > > -static void > -destroy_lb_for_datapath(struct ovn_datapath *od) > -{ > - ovn_lb_ip_set_destroy(od->lb_ips); > - od->lb_ips = NULL; > -} > - > /* A group of logical router datapaths which are connected - either > * directly or indirectly. > * Each logical router can belong to only one group. */ > @@ -676,7 +670,6 @@ ovn_datapath_destroy(struct hmap *datapaths, struct ovn_datapath *od) > destroy_ipam_info(&od->ipam_info); > free(od->router_ports); > free(od->ls_peers); > - destroy_lb_for_datapath(od); > free(od->localnet_ports); > free(od->l3dgw_ports); > destroy_mcast_info_for_datapath(od); > @@ -1311,121 +1304,6 @@ struct lflow_ref_node { > struct ovn_lflow *lflow; > }; > > -/* A logical switch port or logical router port. > - * > - * In steady state, an ovn_port points to a northbound Logical_Switch_Port > - * record (via 'nbsp') *or* a Logical_Router_Port record (via 'nbrp'), and to a > - * southbound Port_Binding record (via 'sb'). As the state of the system > - * changes, join_logical_ports() may determine that there is a new LSP or LRP > - * that has no corresponding Port_Binding record (in which case build_ports()) > - * will create the missing Port_Binding) or that a Port_Binding record exists > - * that has no coresponding LSP (in which case build_ports() will delete the > - * spurious Port_Binding). Thus, after build_ports() runs, any given ovn_port > - * will have 'sb' nonnull, and 'nbsp' xor 'nbrp' nonnull. > - * > - * Ordinarily there is only one ovn_port that points to a given LSP or LRP (but > - * distributed gateway ports point a "derived" ovn_port to a duplicate LRP). > - */ > -struct ovn_port { > - /* Port name aka key. > - * > - * This is ordinarily the same as nbsp->name or nbrp->name and > - * sb->logical_port. (A distributed gateway port creates a "derived" > - * ovn_port with key "cr-%s" % nbrp->name.) */ > - struct hmap_node key_node; /* Index on 'key'. */ > - char *key; /* nbsp->name, nbrp->name, sb->logical_port. */ > - char *json_key; /* 'key', quoted for use in JSON. */ > - > - const struct sbrec_port_binding *sb; /* May be NULL. */ > - > - uint32_t tunnel_key; > - > - /* Logical switch port data. */ > - const struct nbrec_logical_switch_port *nbsp; /* May be NULL. */ > - > - struct lport_addresses *lsp_addrs; /* Logical switch port addresses. */ > - unsigned int n_lsp_addrs; /* Total length of lsp_addrs. */ > - unsigned int n_lsp_non_router_addrs; /* Number of elements from the > - * beginning of 'lsp_addrs' extracted > - * directly from LSP 'addresses'. */ > - > - struct lport_addresses *ps_addrs; /* Port security addresses. */ > - unsigned int n_ps_addrs; > - > - bool lsp_can_be_inc_processed; /* If it can be incrementally processed when > - the port changes. */ > - > - /* Logical router port data. */ > - const struct nbrec_logical_router_port *nbrp; /* May be NULL. */ > - > - struct lport_addresses lrp_networks; > - > - struct ovn_port_routable_addresses routables; > - > - /* Logical port multicast data. */ > - struct mcast_port_info mcast_info; > - > - /* At most one of l3dgw_port and cr_port can be not NULL. */ > - > - /* This is set to a distributed gateway port if and only if this ovn_port > - * is "derived" from it. Otherwise this is set to NULL. The derived > - * ovn_port represents the instance of distributed gateway port on the > - * gateway chassis.*/ > - struct ovn_port *l3dgw_port; > - > - /* This is set to the "derived" chassis-redirect port of this port if and > - * only if this port is a distributed gateway port. Otherwise this is set > - * to NULL. */ > - struct ovn_port *cr_port; > - > - bool has_unknown; /* If the addresses have 'unknown' defined. */ > - > - bool has_bfd; > - > - /* The port's peer: > - * > - * - A switch port S of type "router" has a router port R as a peer, > - * and R in turn has S has its peer. > - * > - * - Two connected logical router ports have each other as peer. > - * > - * - Other kinds of ports have no peer. */ > - struct ovn_port *peer; > - > - struct ovn_datapath *od; > - > - struct ovs_list list; /* In list of similar records. */ > - > - struct hmap_node dp_node; /* Node in od->ports. */ > - > - struct lport_addresses proxy_arp_addrs; > - > - /* Temporarily used for traversing a list (or hmap) of ports. */ > - bool visited; > - > - /* List of struct lflow_ref_node that points to the lflows generated by > - * this ovn_port. > - * > - * This data is initialized and destroyed by the en_northd node, but > - * populated and used only by the en_lflow node. Ideally this data should > - * be maintained as part of en_lflow's data (struct lflow_data): a hash > - * index from ovn_port key to lflows. However, it would be less efficient > - * and more complex: > - * > - * 1. It would require an extra search (using the index) to find the > - * lflows. > - * > - * 2. Building the index needs to be thread-safe, using either a global > - * lock which is obviously less efficient, or hash-based lock array which > - * is more complex. > - * > - * Adding the list here is more straightforward. The drawback is that we > - * need to keep in mind that this data belongs to en_lflow node, so never > - * access it from any other nodes. > - */ > - struct ovs_list lflows; > -}; > - > static bool lsp_can_be_inc_processed(const struct nbrec_logical_switch_port *); > > static bool > @@ -1450,16 +1328,21 @@ destroy_routable_addresses(struct ovn_port_routable_addresses *ra) > } > > static char **get_nat_addresses(const struct ovn_port *op, size_t *n, > - bool routable_only, bool include_lb_ips); > + bool routable_only, bool include_lb_ips, > + const struct lr_stateful_record *); > > -static void > -assign_routable_addresses(struct ovn_port *op) > +static struct ovn_port_routable_addresses > +get_op_routable_addresses(struct ovn_port *op, > + const struct lr_stateful_record *lr_sful_rec) > { > size_t n; > - char **nats = get_nat_addresses(op, &n, true, true); > + char **nats = get_nat_addresses(op, &n, true, true, lr_sful_rec); > > if (!nats) { > - return; > + return (struct ovn_port_routable_addresses) { > + .laddrs = NULL, > + .n_addrs = 0, > + }; > } > > struct lport_addresses *laddrs = xcalloc(n, sizeof(*laddrs)); > @@ -1475,9 +1358,15 @@ assign_routable_addresses(struct ovn_port *op) > } > free(nats); > > - /* Everything seems to have worked out */ > - op->routables.laddrs = laddrs; > - op->routables.n_addrs = n_addrs; > + if (!n_addrs) { > + free(laddrs); > + laddrs = NULL; > + } > + > + return (struct ovn_port_routable_addresses) { > + .laddrs = laddrs, > + .n_addrs = n_addrs, > + }; > } > > > @@ -1537,8 +1426,6 @@ ovn_port_destroy_orphan(struct ovn_port *port) > } > free(port->ps_addrs); > > - destroy_routable_addresses(&port->routables); > - > destroy_lport_addresses(&port->lrp_networks); > destroy_lport_addresses(&port->proxy_arp_addrs); > free(port->json_key); > @@ -2580,9 +2467,7 @@ join_logical_ports(const struct sbrec_port_binding_table *sbrec_pb_table, > sizeof *od->l3dgw_ports); > } > od->l3dgw_ports[od->n_l3dgw_ports++] = op; > - > - assign_routable_addresses(op); > - } > + } > } > } > > @@ -2679,7 +2564,8 @@ join_logical_ports(const struct sbrec_port_binding_table *sbrec_pb_table, > * and must free the returned array when it is no longer needed. */ > static char ** > get_nat_addresses(const struct ovn_port *op, size_t *n, bool routable_only, > - bool include_lb_ips) > + bool include_lb_ips, > + const struct lr_stateful_record *lr_sful_rec) > { > size_t n_nats = 0; > struct eth_addr mac; > @@ -2764,23 +2650,25 @@ get_nat_addresses(const struct ovn_port *op, size_t *n, bool routable_only, > } > } > > - if (include_lb_ips) { > + if (include_lb_ips && lr_sful_rec) { > const char *ip_address; > if (routable_only) { > - SSET_FOR_EACH (ip_address, &op->od->lb_ips->ips_v4_routable) { > + SSET_FOR_EACH (ip_address, > + &lr_sful_rec->lb_ips->ips_v4_routable) { > ds_put_format(&c_addresses, " %s", ip_address); > central_ip_address = true; > } > - SSET_FOR_EACH (ip_address, &op->od->lb_ips->ips_v6_routable) { > + SSET_FOR_EACH (ip_address, > + &lr_sful_rec->lb_ips->ips_v6_routable) { > ds_put_format(&c_addresses, " %s", ip_address); > central_ip_address = true; > } > } else { > - SSET_FOR_EACH (ip_address, &op->od->lb_ips->ips_v4) { > + SSET_FOR_EACH (ip_address, &lr_sful_rec->lb_ips->ips_v4) { > ds_put_format(&c_addresses, " %s", ip_address); > central_ip_address = true; > } > - SSET_FOR_EACH (ip_address, &op->od->lb_ips->ips_v6) { > + SSET_FOR_EACH (ip_address, &lr_sful_rec->lb_ips->ips_v6) { > ds_put_format(&c_addresses, " %s", ip_address); > central_ip_address = true; > } > @@ -3851,21 +3739,8 @@ build_lb_datapaths(const struct hmap *lbs, const struct hmap *lb_groups, > HMAP_FOR_EACH (od, key_node, &lr_datapaths->datapaths) { > ovs_assert(od->nbr); > > - /* Checking load balancer groups first, starting from the largest one, > - * to more efficiently copy IP sets. */ > - size_t largest_group = 0; > - > - for (size_t i = 1; i < od->nbr->n_load_balancer_group; i++) { > - if (od->nbr->load_balancer_group[i]->n_load_balancer > > - od->nbr->load_balancer_group[largest_group]->n_load_balancer) { > - largest_group = i; > - } > - } > - > for (size_t i = 0; i < od->nbr->n_load_balancer_group; i++) { > - size_t idx = (i + largest_group) % od->nbr->n_load_balancer_group; > - > - nbrec_lb_group = od->nbr->load_balancer_group[idx]; > + nbrec_lb_group = od->nbr->load_balancer_group[i]; > const struct uuid *lb_group_uuid = &nbrec_lb_group->header_.uuid; > > lb_group_dps = > @@ -3873,20 +3748,6 @@ build_lb_datapaths(const struct hmap *lbs, const struct hmap *lb_groups, > lb_group_uuid); > ovs_assert(lb_group_dps); > ovn_lb_group_datapaths_add_lr(lb_group_dps, od); > - > - if (!od->lb_ips) { > - od->lb_ips = > - ovn_lb_ip_set_clone(lb_group_dps->lb_group->lb_ips); > - } else { > - for (size_t j = 0; j < lb_group_dps->lb_group->n_lbs; j++) { > - build_lrouter_lb_ips(od->lb_ips, > - lb_group_dps->lb_group->lbs[j]); > - } > - } > - } > - > - if (!od->lb_ips) { > - od->lb_ips = ovn_lb_ip_set_create(); > } > > for (size_t i = 0; i < od->nbr->n_load_balancer; i++) { > @@ -3895,7 +3756,6 @@ build_lb_datapaths(const struct hmap *lbs, const struct hmap *lb_groups, > lb_dps = ovn_lb_datapaths_find(lb_datapaths_map, lb_uuid); > ovs_assert(lb_dps); > ovn_lb_datapaths_add_lr(lb_dps, 1, &od); > - build_lrouter_lb_ips(od->lb_ips, lb_dps->lb); > } > } > > @@ -3949,102 +3809,6 @@ build_lb_svcs( > } > } > > -static bool lrouter_port_ipv4_reachable(const struct ovn_port *op, > - ovs_be32 addr); > -static bool lrouter_port_ipv6_reachable(const struct ovn_port *op, > - const struct in6_addr *addr); > - > -static void > -add_neigh_ips_to_lrouter(struct ovn_datapath *od, > - enum lb_neighbor_responder_mode neigh_mode, > - const struct sset *lb_ips_v4, > - const struct sset *lb_ips_v6) > -{ > - /* If configured to not reply to any neighbor requests for all VIPs > - * return early. > - */ > - if (neigh_mode == LB_NEIGH_RESPOND_NONE) { > - return; > - } > - > - const char *ip_address; > - > - /* If configured to reply to neighbor requests for all VIPs force them > - * all to be considered "reachable". > - */ > - if (neigh_mode == LB_NEIGH_RESPOND_ALL) { > - SSET_FOR_EACH (ip_address, lb_ips_v4) { > - sset_add(&od->lb_ips->ips_v4_reachable, ip_address); > - } > - SSET_FOR_EACH (ip_address, lb_ips_v6) { > - sset_add(&od->lb_ips->ips_v6_reachable, ip_address); > - } > - > - return; > - } > - > - /* Otherwise, a VIP is reachable if there's at least one router > - * subnet that includes it. > - */ > - ovs_assert(neigh_mode == LB_NEIGH_RESPOND_REACHABLE); > - > - SSET_FOR_EACH (ip_address, lb_ips_v4) { > - struct ovn_port *op; > - ovs_be32 vip_ip4; > - if (ip_parse(ip_address, &vip_ip4)) { > - HMAP_FOR_EACH (op, dp_node, &od->ports) { > - if (lrouter_port_ipv4_reachable(op, vip_ip4)) { > - sset_add(&od->lb_ips->ips_v4_reachable, > - ip_address); > - break; > - } > - } > - } > - } > - > - SSET_FOR_EACH (ip_address, lb_ips_v6) { > - struct ovn_port *op; > - struct in6_addr vip; > - if (ipv6_parse(ip_address, &vip)) { > - HMAP_FOR_EACH (op, dp_node, &od->ports) { > - if (lrouter_port_ipv6_reachable(op, &vip)) { > - sset_add(&od->lb_ips->ips_v6_reachable, > - ip_address); > - break; > - } > - } > - } > - } > -} > - > -static void > -remove_lrouter_lb_reachable_ips(struct ovn_datapath *od, > - enum lb_neighbor_responder_mode neigh_mode, > - const struct sset *lb_ips_v4, > - const struct sset *lb_ips_v6) > -{ > - if (neigh_mode == LB_NEIGH_RESPOND_NONE) { > - return; > - } > - > - const char *ip_address; > - SSET_FOR_EACH (ip_address, lb_ips_v4) { > - sset_find_and_delete(&od->lb_ips->ips_v4_reachable, ip_address); > - } > - SSET_FOR_EACH (ip_address, lb_ips_v6) { > - sset_find_and_delete(&od->lb_ips->ips_v6_reachable, ip_address); > - } > -} > - > -static void > -build_lrouter_lb_reachable_ips(struct ovn_datapath *od, > - const struct ovn_northd_lb *lb) > -{ > - add_neigh_ips_to_lrouter(od, lb->neigh_mode, &lb->ips_v4, > - &lb->ips_v6); > -} > - > - > static void > build_lrouter_lbs_check(const struct ovn_datapaths *lr_datapaths) > { > @@ -4066,43 +3830,6 @@ build_lrouter_lbs_check(const struct ovn_datapaths *lr_datapaths) > } > } > > -static void > -build_lrouter_lbs_reachable_ips(struct ovn_datapaths *lr_datapaths, > - struct hmap *lb_dps_map, > - struct hmap *lb_group_dps_map) > -{ > - struct ovn_datapath *od; > - > - HMAP_FOR_EACH (od, key_node, &lr_datapaths->datapaths) { > - if (!od->nbr) { > - continue; > - } > - > - for (size_t i = 0; i < od->nbr->n_load_balancer; i++) { > - struct ovn_lb_datapaths *lb_dps = > - ovn_lb_datapaths_find(lb_dps_map, > - &od->nbr->load_balancer[i]->header_.uuid); > - ovs_assert(lb_dps); > - build_lrouter_lb_reachable_ips(od, lb_dps->lb); > - } > - > - for (size_t i = 0; i < od->nbr->n_load_balancer_group; i++) { > - const struct nbrec_load_balancer_group *nbrec_lb_group = > - od->nbr->load_balancer_group[i]; > - struct ovn_lb_group_datapaths *lb_group_dps; > - > - lb_group_dps = > - ovn_lb_group_datapaths_find(lb_group_dps_map, > - &nbrec_lb_group->header_.uuid); > - ovs_assert(lb_group_dps); > - for (size_t j = 0; j < lb_group_dps->lb_group->n_lbs; j++) { > - build_lrouter_lb_reachable_ips(od, > - lb_group_dps->lb_group->lbs[j]); > - } > - } > - } > -} > - > static void > build_lswitch_lbs_from_lrouter(struct ovn_datapaths *lr_datapaths, > struct hmap *lb_dps_map, > @@ -4166,8 +3893,6 @@ build_lb_port_related_data( > struct hmap *svc_monitor_map) > { > build_lrouter_lbs_check(lr_datapaths); > - build_lrouter_lbs_reachable_ips(lr_datapaths, lb_dps_map, > - lb_group_dps_map); > build_lb_svcs(ovnsb_txn, sbrec_service_monitor_table, ls_ports, lb_dps_map, > svc_monitor_lsps, svc_monitor_map); > build_lswitch_lbs_from_lrouter(lr_datapaths, lb_dps_map, lb_group_dps_map); > @@ -4533,7 +4258,8 @@ check_sb_lb_duplicates(const struct sbrec_load_balancer_table *table) > * Caller should make sure that the OVN SB IDL txn is not NULL. Presently it > * only syncs the nat column of port binding corresponding to the 'op->nbsp' */ > static void > -sync_pb_for_lsp(struct ovn_port *op) > +sync_pb_for_lsp(struct ovn_port *op, > + const struct lr_stateful_table *lr_sful_table) > { > ovs_assert(op->nbsp); > > @@ -4552,10 +4278,17 @@ sync_pb_for_lsp(struct ovn_port *op) > if (nat_addresses && !strcmp(nat_addresses, "router")) { > if (op->peer && op->peer->od > && (chassis || op->peer->od->n_l3dgw_ports)) { > - bool exclude_lb_vips = smap_get_bool(&op->nbsp->options, > + bool include_lb_vips = !smap_get_bool(&op->nbsp->options, > "exclude-lb-vips-from-garp", false); > + > + const struct lr_stateful_record *lr_sful_rec = NULL; > + > + if (include_lb_vips) { > + lr_sful_rec = lr_stateful_table_find_by_index( > + lr_sful_table, op->peer->od->index); > + } > nats = get_nat_addresses(op->peer, &n_nats, false, > - !exclude_lb_vips); > + include_lb_vips, lr_sful_rec); > } > } else if (nat_addresses && (chassis || l3dgw_ports)) { > struct lport_addresses laddrs; > @@ -4662,7 +4395,8 @@ sync_pb_for_lsp(struct ovn_port *op) > * Caller should make sure that the OVN SB IDL txn is not NULL. Presently it > * only sets the port binding options column for the router ports */ > static void > -sync_pb_for_lrp(struct ovn_port *op, const struct lr_nat_table *lr_nats) > +sync_pb_for_lrp(struct ovn_port *op, > + const struct lr_stateful_table *lr_sful_table) > { > ovs_assert(op->nbrp); > > @@ -4671,14 +4405,14 @@ sync_pb_for_lrp(struct ovn_port *op, const struct lr_nat_table *lr_nats) > > const char *chassis_name = smap_get(&op->od->nbr->options, "chassis"); > if (is_cr_port(op)) { > - const struct lr_nat_record *lrnat_rec = > - lr_nat_table_find_by_index(lr_nats, op->od->index); > - ovs_assert(lrnat_rec); > + const struct lr_stateful_record *lr_sful_rec = > + lr_stateful_table_find_by_index(lr_sful_table, op->od->index); > + ovs_assert(lr_sful_rec); > > smap_add(&new, "distributed-port", op->nbrp->name); > > bool always_redirect = > - !lrnat_rec->has_distributed_nat && > + !lr_sful_rec->lrnat_rec->has_distributed_nat && > !l3dgw_port_has_associated_vtep_lports(op->l3dgw_port); > > const char *redirect_type = smap_get(&op->nbrp->options, > @@ -4728,17 +4462,18 @@ static void ovn_update_ipv6_opt_for_op(struct ovn_port *op); > * the logical switch ports. */ > void > sync_pbs(struct ovsdb_idl_txn *ovnsb_idl_txn, struct hmap *ls_ports, > - struct hmap *lr_ports, const struct lr_nat_table *lr_nats) > + struct hmap *lr_ports, > + const struct lr_stateful_table *lr_sful_table) > { > ovs_assert(ovnsb_idl_txn); > > struct ovn_port *op; > HMAP_FOR_EACH (op, key_node, ls_ports) { > - sync_pb_for_lsp(op); > + sync_pb_for_lsp(op, lr_sful_table); > } > > HMAP_FOR_EACH (op, key_node, lr_ports) { > - sync_pb_for_lrp(op, lr_nats); > + sync_pb_for_lrp(op, lr_sful_table); > } > > ovn_update_ipv6_options(lr_ports); > @@ -4747,20 +4482,22 @@ sync_pbs(struct ovsdb_idl_txn *ovnsb_idl_txn, struct hmap *ls_ports, > /* Sync the SB Port bindings for the added and updated logical switch ports > * of the tracked northd engine data. */ > bool > -sync_pbs_for_northd_changed_ovn_ports(struct tracked_ovn_ports *trk_ovn_ports) > +sync_pbs_for_northd_changed_ovn_ports( > + struct tracked_ovn_ports *trk_ovn_ports, > + const struct lr_stateful_table *lr_sful_table) > { > struct hmapx_node *hmapx_node; > struct ovn_port *op; > HMAPX_FOR_EACH (hmapx_node, &trk_ovn_ports->created) { > op = hmapx_node->data; > ovs_assert(op->nbsp); > - sync_pb_for_lsp(op); > + sync_pb_for_lsp(op, lr_sful_table); > } > > HMAPX_FOR_EACH (hmapx_node, &trk_ovn_ports->updated) { > op = hmapx_node->data; > ovs_assert(op->nbsp); > - sync_pb_for_lsp(op); > + sync_pb_for_lsp(op, lr_sful_table); > } > > return true; > @@ -5425,14 +5162,13 @@ fail: > } > > /* Returns true if the logical router has changes which can be > - * incrementally handled. > + * incrementally handled or the changes can be ignored. > * Presently supports i-p for the below changes: > * - load balancers and load balancer groups. > * - NAT changes > */ > static bool > -lr_changes_can_be_handled( > - const struct nbrec_logical_router *lr) > +lr_changes_can_be_handled(const struct nbrec_logical_router *lr) > { > /* Check if the columns are changed in this row. */ > enum nbrec_logical_router_column_id col; > @@ -5496,17 +5232,6 @@ is_lr_nats_changed(const struct nbrec_logical_router *nbr) { > || is_lr_nats_seqno_changed(nbr)); > } > > -static bool > -lr_has_routable_nats(const struct nbrec_logical_router *nbr) { > - for (size_t i = 0; i < nbr->n_nat; i++) { > - if (smap_get_bool(&nbr->nat[i]->options, "add_route", false)) { > - return true; > - } > - } > - > - return false; > -} > - > /* Return true if changes are handled incrementally, false otherwise. > * > * Note: Changes to load balancer and load balancer groups associated with > @@ -5533,12 +5258,6 @@ northd_handle_lr_changes(const struct northd_input *ni, > } > > if (is_lr_nats_changed(changed_lr)) { > - if (lr_has_routable_nats(changed_lr)) { > - /* router has routable NATs. We can't handle these changes > - * incrementally yet. Fall back to recompute. */ > - goto fail; > - } > - > struct ovn_datapath *od = ovn_datapath_find_( > &nd->lr_datapaths.datapaths, > &changed_lr->header_.uuid); > @@ -5820,10 +5539,6 @@ northd_handle_lb_data_changes(struct tracked_lb_data *trk_lb_data, > ovs_assert(lb_dps); > ovn_lb_datapaths_add_lr(lb_dps, 1, &od); > > - /* Add the lb_ips of lb_dps to the od. */ > - build_lrouter_lb_ips(od->lb_ips, lb_dps->lb); > - build_lrouter_lb_reachable_ips(od, lb_dps->lb); > - > /* Add the lb to the northd tracked data. */ > hmapx_add(&nd_changes->trk_lbs.crupdated, lb_dps); > } > @@ -5842,10 +5557,6 @@ northd_handle_lb_data_changes(struct tracked_lb_data *trk_lb_data, > ovs_assert(lb_dps); > ovn_lb_datapaths_add_lr(lb_dps, 1, &od); > > - /* Add the lb_ips of lb_dps to the od. */ > - build_lrouter_lb_ips(od->lb_ips, lb_dps->lb); > - build_lrouter_lb_reachable_ips(od, lb_dps->lb); > - > /* Add the lb to the northd tracked data. */ > hmapx_add(&nd_changes->trk_lbs.crupdated, lb_dps); > } > @@ -5874,22 +5585,6 @@ northd_handle_lb_data_changes(struct tracked_lb_data *trk_lb_data, > od = lr_datapaths->array[index]; > /* Re-evaluate 'od->has_lb_vip' */ > init_lb_for_datapath(od); > - > - /* Update the od->lb_ips with the deleted and inserted > - * vips (if any). */ > - remove_ips_from_lb_ip_set(od->lb_ips, lb->routable, > - &clb->deleted_vips_v4, > - &clb->deleted_vips_v6); > - add_ips_to_lb_ip_set(od->lb_ips, lb->routable, > - &clb->inserted_vips_v4, > - &clb->inserted_vips_v6); > - > - remove_lrouter_lb_reachable_ips(od, lb->neigh_mode, > - &clb->deleted_vips_v4, > - &clb->deleted_vips_v6); > - add_neigh_ips_to_lrouter(od, lb->neigh_mode, > - &clb->inserted_vips_v4, > - &clb->inserted_vips_v6); > } > } > > @@ -5914,9 +5609,6 @@ northd_handle_lb_data_changes(struct tracked_lb_data *trk_lb_data, > > /* Re-evaluate 'od->has_lb_vip' */ > init_lb_for_datapath(od); > - > - /* Add the lb_ips of lb_dps to the od. */ > - build_lrouter_lb_ips(od->lb_ips, lb_dps->lb); > } > > for (size_t i = 0; i < lbgrp_dps->n_ls; i++) { > @@ -9210,7 +8902,7 @@ arp_nd_ns_match(const char *ips, int addr_family, struct ds *match) > /* Returns 'true' if the IPv4 'addr' is on the same subnet with one of the > * IPs configured on the router port. > */ > -static bool > +bool > lrouter_port_ipv4_reachable(const struct ovn_port *op, ovs_be32 addr) > { > for (size_t i = 0; i < op->lrp_networks.n_ipv4_addrs; i++) { > @@ -9226,7 +8918,7 @@ lrouter_port_ipv4_reachable(const struct ovn_port *op, ovs_be32 addr) > /* Returns 'true' if the IPv6 'addr' is on the same subnet with one of the > * IPs configured on the router port. > */ > -static bool > +bool > lrouter_port_ipv6_reachable(const struct ovn_port *op, > const struct in6_addr *addr) > { > @@ -9289,11 +8981,12 @@ build_lswitch_rport_arp_req_flow(const char *ips, > */ > static void > build_lswitch_rport_arp_req_flows(struct ovn_port *op, > - struct ovn_datapath *sw_od, > - struct ovn_port *sw_op, > - const struct lr_nat_table *lr_nats, > - struct hmap *lflows, > - const struct ovsdb_idl_row *stage_hint) > + struct ovn_datapath *sw_od, > + struct ovn_port *sw_op, > + const struct lr_nat_table *lr_nats, > + const struct lr_stateful_table *lr_sful_table, > + struct hmap *lflows, > + const struct ovsdb_idl_row *stage_hint) > { > if (!op || !op->nbrp) { > return; > @@ -9307,32 +9000,38 @@ build_lswitch_rport_arp_req_flows(struct ovn_port *op, > * router port. > * Priority: 80. > */ > - > - const char *ip_addr; > - SSET_FOR_EACH (ip_addr, &op->od->lb_ips->ips_v4_reachable) { > - ovs_be32 ipv4_addr; > - > - /* Check if the ovn port has a network configured on which we could > - * expect ARP requests for the LB VIP. > - */ > - if (ip_parse(ip_addr, &ipv4_addr) && > - lrouter_port_ipv4_reachable(op, ipv4_addr)) { > - build_lswitch_rport_arp_req_flow( > - ip_addr, AF_INET, sw_op, sw_od, 80, lflows, > - stage_hint); > + const struct lr_stateful_record *lr_sful_rec = NULL; > + if (op->od->nbr->n_load_balancer || op->od->nbr->n_load_balancer_group) { > + lr_sful_rec = lr_stateful_table_find_by_index(lr_sful_table, > + op->od->index); > + ovs_assert(lr_sful_rec); > + > + const char *ip_addr; > + SSET_FOR_EACH (ip_addr, &lr_sful_rec->lb_ips->ips_v4_reachable) { > + ovs_be32 ipv4_addr; > + > + /* Check if the ovn port has a network configured on which we could > + * expect ARP requests for the LB VIP. > + */ > + if (ip_parse(ip_addr, &ipv4_addr) && > + lrouter_port_ipv4_reachable(op, ipv4_addr)) { > + build_lswitch_rport_arp_req_flow( > + ip_addr, AF_INET, sw_op, sw_od, 80, lflows, > + stage_hint); > + } > } > - } > - SSET_FOR_EACH (ip_addr, &op->od->lb_ips->ips_v6_reachable) { > - struct in6_addr ipv6_addr; > + SSET_FOR_EACH (ip_addr, &lr_sful_rec->lb_ips->ips_v6_reachable) { > + struct in6_addr ipv6_addr; > > - /* Check if the ovn port has a network configured on which we could > - * expect NS requests for the LB VIP. > - */ > - if (ipv6_parse(ip_addr, &ipv6_addr) && > - lrouter_port_ipv6_reachable(op, &ipv6_addr)) { > - build_lswitch_rport_arp_req_flow( > - ip_addr, AF_INET6, sw_op, sw_od, 80, lflows, > - stage_hint); > + /* Check if the ovn port has a network configured on which we could > + * expect NS requests for the LB VIP. > + */ > + if (ipv6_parse(ip_addr, &ipv6_addr) && > + lrouter_port_ipv6_reachable(op, &ipv6_addr)) { > + build_lswitch_rport_arp_req_flow( > + ip_addr, AF_INET6, sw_op, sw_od, 80, lflows, > + stage_hint); > + } > } > } > > @@ -9382,13 +9081,15 @@ build_lswitch_rport_arp_req_flows(struct ovn_port *op, > * expect ARP requests/NS for the DNAT external_ip. > */ > if (nat_entry_is_v6(nat_entry)) { > - if (!sset_contains(&op->od->lb_ips->ips_v6, nat->external_ip)) { > + if (!lr_sful_rec || !sset_contains(&lr_sful_rec->lb_ips->ips_v6, > + nat->external_ip)) { > build_lswitch_rport_arp_req_flow( > nat->external_ip, AF_INET6, sw_op, sw_od, 80, lflows, > stage_hint); > } > } else { > - if (!sset_contains(&op->od->lb_ips->ips_v4, nat->external_ip)) { > + if (!lr_sful_rec || !sset_contains(&lr_sful_rec->lb_ips->ips_v4, > + nat->external_ip)) { > build_lswitch_rport_arp_req_flow( > nat->external_ip, AF_INET, sw_op, sw_od, 80, lflows, > stage_hint); > @@ -10429,6 +10130,7 @@ build_lswitch_ip_mcast_igmp_mld(struct ovn_igmp_group *igmp_group, > static void > build_lswitch_ip_unicast_lookup(struct ovn_port *op, > const struct lr_nat_table *lr_nats, > + const struct lr_stateful_table *lr_sful_table, > struct hmap *lflows, > struct ds *actions, > struct ds *match) > @@ -10444,7 +10146,8 @@ build_lswitch_ip_unicast_lookup(struct ovn_port *op, > */ > if (lsp_is_router(op->nbsp)) { > build_lswitch_rport_arp_req_flows(op->peer, op->od, op, lr_nats, > - lflows, &op->nbsp->header_); > + lr_sful_table, lflows, > + &op->nbsp->header_); > } > > for (size_t i = 0; i < op->nbsp->n_addresses; i++) { > @@ -12634,6 +12337,7 @@ build_lrouter_port_nat_arp_nd_flow(struct ovn_port *op, > static void > build_lrouter_drop_own_dest(struct ovn_port *op, > const struct lr_nat_record *lrnat_rec, > + const struct lr_stateful_record *lr_sful_rec, > enum ovn_stage stage, > uint16_t priority, bool drop_snat_ip, > struct hmap *lflows) > @@ -12646,8 +12350,9 @@ build_lrouter_drop_own_dest(struct ovn_port *op, > > bool router_ip_in_snat_ips = !!shash_find(&lrnat_rec->snat_ips, > ip); > - bool router_ip_in_lb_ips = > - !!sset_find(&op->od->lb_ips->ips_v4, ip); > + bool router_ip_in_lb_ips = (lr_sful_rec && > + !!sset_find(&lr_sful_rec->lb_ips->ips_v4, > + ip)); > bool drop_router_ip = (drop_snat_ip == (router_ip_in_snat_ips || > router_ip_in_lb_ips)); > > @@ -12676,8 +12381,9 @@ build_lrouter_drop_own_dest(struct ovn_port *op, > > bool router_ip_in_snat_ips = !!shash_find(&lrnat_rec->snat_ips, > ip); > - bool router_ip_in_lb_ips = > - !!sset_find(&op->od->lb_ips->ips_v6, ip); > + bool router_ip_in_lb_ips = (lr_sful_rec && > + !!sset_find(&lr_sful_rec->lb_ips->ips_v6, > + ip)); > bool drop_router_ip = (drop_snat_ip == (router_ip_in_snat_ips || > router_ip_in_lb_ips)); > > @@ -13389,7 +13095,8 @@ build_ip_routing_flows_for_lrp( > */ > static void > build_ip_routing_flows_for_router_type_lsp( > - struct ovn_port *op, const struct hmap *lr_ports, struct hmap *lflows) > + struct ovn_port *op, const struct lr_stateful_table *lr_sful_table, > + const struct hmap *lr_ports, struct hmap *lflows) > { > ovs_assert(op->nbsp); > if (!lsp_is_router(op->nbsp)) { > @@ -13397,7 +13104,8 @@ build_ip_routing_flows_for_router_type_lsp( > } > > struct ovn_port *peer = ovn_port_get_peer(lr_ports, op); > - if (!peer || !peer->nbrp || !peer->lrp_networks.n_ipv4_addrs) { > + if (!peer || !peer->nbrp || !peer->lrp_networks.n_ipv4_addrs > + || !op->od->n_router_ports) { > return; > } > > @@ -13408,19 +13116,29 @@ build_ip_routing_flows_for_router_type_lsp( > continue; > } > > - struct ovn_port_routable_addresses *ra = &router_port->routables; > - for (size_t j = 0; j < ra->n_addrs; j++) { > - struct lport_addresses *laddrs = &ra->laddrs[j]; > - for (size_t k = 0; k < laddrs->n_ipv4_addrs; k++) { > - add_route(lflows, peer->od, peer, > - peer->lrp_networks.ipv4_addrs[0].addr_s, > - laddrs->ipv4_addrs[k].network_s, > - laddrs->ipv4_addrs[k].plen, NULL, false, 0, > - &peer->nbrp->header_, false, > - ROUTE_PRIO_OFFSET_CONNECTED); > + const struct lr_stateful_record *lr_sful_rec = > + lr_stateful_table_find_by_index(lr_sful_table, > + router_port->od->index); > + > + if (router_port->nbrp->ha_chassis_group || > + router_port->nbrp->n_gateway_chassis) { > + struct ovn_port_routable_addresses ra = > + get_op_routable_addresses(router_port, lr_sful_rec); > + for (size_t j = 0; j < ra.n_addrs; j++) { > + struct lport_addresses *laddrs = &ra.laddrs[j]; > + for (size_t k = 0; k < laddrs->n_ipv4_addrs; k++) { > + add_route(lflows, peer->od, peer, > + peer->lrp_networks.ipv4_addrs[0].addr_s, > + laddrs->ipv4_addrs[k].network_s, > + laddrs->ipv4_addrs[k].plen, NULL, false, 0, > + &peer->nbrp->header_, false, > + ROUTE_PRIO_OFFSET_CONNECTED); > + } > } > + destroy_routable_addresses(&ra); > } > } > + > } > > static void > @@ -13644,33 +13362,36 @@ build_arp_resolve_flows_for_lrouter( > > static void > routable_addresses_to_lflows(struct hmap *lflows, struct ovn_port *router_port, > - struct ovn_port *peer, struct ds *match, > - struct ds *actions) > + struct ovn_port *peer, > + const struct lr_stateful_record *lr_sful_rec, > + struct ds *match, struct ds *actions) > { > - struct ovn_port_routable_addresses *ra = &router_port->routables; > - if (!ra->n_addrs) { > + struct ovn_port_routable_addresses ra = > + get_op_routable_addresses(router_port, lr_sful_rec); > + if (!ra.n_addrs) { > return; > } > > - for (size_t i = 0; i < ra->n_addrs; i++) { > + for (size_t i = 0; i < ra.n_addrs; i++) { > ds_clear(match); > ds_put_format(match, "outport == %s && "REG_NEXT_HOP_IPV4" == {", > peer->json_key); > bool first = true; > - for (size_t j = 0; j < ra->laddrs[i].n_ipv4_addrs; j++) { > + for (size_t j = 0; j < ra.laddrs[i].n_ipv4_addrs; j++) { > if (!first) { > ds_put_cstr(match, ", "); > } > - ds_put_cstr(match, ra->laddrs[i].ipv4_addrs[j].addr_s); > + ds_put_cstr(match, ra.laddrs[i].ipv4_addrs[j].addr_s); > first = false; > } > ds_put_cstr(match, "}"); > > ds_clear(actions); > - ds_put_format(actions, "eth.dst = %s; next;", ra->laddrs[i].ea_s); > + ds_put_format(actions, "eth.dst = %s; next;", ra.laddrs[i].ea_s); > ovn_lflow_add(lflows, peer->od, S_ROUTER_IN_ARP_RESOLVE, 100, > ds_cstr(match), ds_cstr(actions)); > } > + destroy_routable_addresses(&ra); > } > > /* Local router ingress table ARP_RESOLVE: ARP Resolution. > @@ -13687,6 +13408,7 @@ routable_addresses_to_lflows(struct hmap *lflows, struct ovn_port *router_port, > static void > build_arp_resolve_flows_for_lrp( > struct ovn_port *op, const struct lr_nat_record *lrnat_rec, > + const struct lr_stateful_record *lr_sful_rec, > struct hmap *lflows, struct ds *match, struct ds *actions) > { > ovs_assert(op->nbrp); > @@ -13763,8 +13485,8 @@ build_arp_resolve_flows_for_lrp( > * > * Priority 2. > */ > - build_lrouter_drop_own_dest(op, lrnat_rec, S_ROUTER_IN_ARP_RESOLVE, 2, > - true, lflows); > + build_lrouter_drop_own_dest(op, lrnat_rec, lr_sful_rec, > + S_ROUTER_IN_ARP_RESOLVE, 2, true, lflows); > } > > /* This function adds ARP resolve flows related to a LSP. */ > @@ -13772,6 +13494,7 @@ static void > build_arp_resolve_flows_for_lsp( > struct ovn_port *op, struct hmap *lflows, > const struct hmap *lr_ports, > + const struct lr_stateful_table *lr_sful_table, > struct ds *match, struct ds *actions) > { > ovs_assert(op->nbsp); > @@ -13915,8 +13638,11 @@ build_arp_resolve_flows_for_lsp( > > if (smap_get(&peer->od->nbr->options, "chassis") > || peer->cr_port) { > + const struct lr_stateful_record *lr_sful_rec; > + lr_sful_rec = lr_stateful_table_find_by_index(lr_sful_table, > + router_port->od->index); > routable_addresses_to_lflows(lflows, router_port, peer, > - match, actions); > + lr_sful_rec, match, actions); > } > } > } > @@ -14636,6 +14362,7 @@ static void > build_lrouter_ipv4_ip_input(struct ovn_port *op, > struct hmap *lflows, > const struct lr_nat_record *lrnat_rec, > + const struct lr_stateful_record *lr_sful_rec, > struct ds *match, struct ds *actions, > const struct shash *meter_groups) > { > @@ -14760,7 +14487,7 @@ build_lrouter_ipv4_ip_input(struct ovn_port *op, > &op->nbrp->header_, lflows); > } > > - if (sset_count(&op->od->lb_ips->ips_v4_reachable)) { > + if (lr_sful_rec && sset_count(&lr_sful_rec->lb_ips->ips_v4_reachable)) { > ds_clear(match); > if (is_l3dgw_port(op)) { > ds_put_format(match, "is_chassis_resident(%s)", > @@ -14776,7 +14503,7 @@ build_lrouter_ipv4_ip_input(struct ovn_port *op, > free(lb_ips_v4_as); > } > > - if (sset_count(&op->od->lb_ips->ips_v6_reachable)) { > + if (lr_sful_rec && sset_count(&lr_sful_rec->lb_ips->ips_v6_reachable)) { > ds_clear(match); > > if (is_l3dgw_port(op)) { > @@ -14878,8 +14605,8 @@ build_lrouter_ipv4_ip_input(struct ovn_port *op, > * Priority 60. > */ > if (!lrnat_rec->lb_force_snat_router_ip) { > - build_lrouter_drop_own_dest(op, lrnat_rec, S_ROUTER_IN_IP_INPUT, 60, > - false, lflows); > + build_lrouter_drop_own_dest(op, lrnat_rec, lr_sful_rec, > + S_ROUTER_IN_IP_INPUT, 60, false, lflows); > } > /* ARP / ND handling for external IP addresses. > * > @@ -16018,6 +15745,7 @@ struct lswitch_flow_build_info { > const struct hmap *lr_ports; > const struct ls_port_group_table *ls_port_groups; > const struct lr_nat_table *lr_nats; > + const struct lr_stateful_table *lr_sful_table; > struct hmap *lflows; > struct hmap *igmp_groups; > const struct shash *meter_groups; > @@ -16101,14 +15829,15 @@ build_lswitch_and_lrouter_iterate_by_lr(struct ovn_datapath *od, > * switch port. > */ > static void > -build_lswitch_and_lrouter_iterate_by_lsp(struct ovn_port *op, > - const struct hmap *ls_ports, > - const struct hmap *lr_ports, > - const struct lr_nat_table *lr_nats, > - const struct shash *meter_groups, > - struct ds *match, > - struct ds *actions, > - struct hmap *lflows) > +build_lswitch_and_lrouter_iterate_by_lsp( > + struct ovn_port *op, const struct hmap *ls_ports, > + const struct hmap *lr_ports, > + const struct lr_nat_table *lr_nats, > + const struct lr_stateful_table *lr_sful_table, > + const struct shash *meter_groups, > + struct ds *match, > + struct ds *actions, > + struct hmap *lflows) > { > ovs_assert(op->nbsp); > start_collecting_lflows(); > @@ -16121,11 +15850,14 @@ build_lswitch_and_lrouter_iterate_by_lsp(struct ovn_port *op, > meter_groups, actions, match); > build_lswitch_dhcp_options_and_response(op, lflows, meter_groups); > build_lswitch_external_port(op, lflows); > - build_lswitch_ip_unicast_lookup(op, lr_nats, lflows, actions, match); > + build_lswitch_ip_unicast_lookup(op, lr_nats, lr_sful_table, lflows, > + actions, match); > > /* Build Logical Router Flows. */ > - build_ip_routing_flows_for_router_type_lsp(op, lr_ports, lflows); > - build_arp_resolve_flows_for_lsp(op, lflows, lr_ports, match, actions); > + build_ip_routing_flows_for_router_type_lsp(op, lr_sful_table, lr_ports, > + lflows); > + build_arp_resolve_flows_for_lsp(op, lflows, lr_ports, lr_sful_table, > + match, actions); > > link_ovn_port_to_lflows(op, &collected_lflows); > end_collecting_lflows(); > @@ -16144,6 +15876,8 @@ build_lswitch_and_lrouter_iterate_by_lrp(struct ovn_port *op, > lsi->lr_nats, op->od->index); > ovs_assert(lrnet_rec); > > + const struct lr_stateful_record *lr_sful_rec = > + lr_stateful_table_find_by_index(lsi->lr_sful_table, op->od->index); > build_adm_ctrl_flows_for_lrouter_port(op, lsi->lflows, &lsi->match, > &lsi->actions); > build_neigh_learning_flows_for_lrouter_port(op, lsi->lflows, &lsi->match, > @@ -16151,15 +15885,15 @@ build_lswitch_and_lrouter_iterate_by_lrp(struct ovn_port *op, > build_ip_routing_flows_for_lrp(op, lsi->lflows); > build_ND_RA_flows_for_lrouter_port(op, lsi->lflows, &lsi->match, > &lsi->actions, lsi->meter_groups); > - build_arp_resolve_flows_for_lrp(op, lrnet_rec, lsi->lflows, &lsi->match, > - &lsi->actions); > + build_arp_resolve_flows_for_lrp(op, lrnet_rec, lr_sful_rec, lsi->lflows, > + &lsi->match, &lsi->actions); > build_egress_delivery_flows_for_lrouter_port(op, lsi->lflows, &lsi->match, > &lsi->actions); > build_dhcpv6_reply_flows_for_lrouter_port(op, lsi->lflows, &lsi->match); > build_ipv6_input_flows_for_lrouter_port(op, lsi->lflows, > &lsi->match, &lsi->actions, > lsi->meter_groups); > - build_lrouter_ipv4_ip_input(op, lsi->lflows, lrnet_rec, > + build_lrouter_ipv4_ip_input(op, lsi->lflows, lrnet_rec, lr_sful_rec, > &lsi->match, &lsi->actions, lsi->meter_groups); > build_lrouter_force_snat_flows_op(op, lrnet_rec, lsi->lflows, &lsi->match, > &lsi->actions); > @@ -16220,12 +15954,13 @@ build_lflows_thread(void *arg) > return NULL; > } > build_lswitch_and_lrouter_iterate_by_lsp(op, lsi->ls_ports, > - lsi->lr_ports, > - lsi->lr_nats, > - lsi->meter_groups, > - &lsi->match, > - &lsi->actions, > - lsi->lflows); > + lsi->lr_ports, > + lsi->lr_nats, > + lsi->lr_sful_table, > + lsi->meter_groups, > + &lsi->match, > + &lsi->actions, > + lsi->lflows); > } > } > for (bnum = control->id; > @@ -16332,6 +16067,7 @@ build_lswitch_and_lrouter_flows(const struct ovn_datapaths *ls_datapaths, > const struct hmap *lr_ports, > const struct ls_port_group_table *ls_pgs, > const struct lr_nat_table *lr_nats, > + const struct lr_stateful_table *lr_sful_table, > struct hmap *lflows, > struct hmap *igmp_groups, > const struct shash *meter_groups, > @@ -16362,6 +16098,7 @@ build_lswitch_and_lrouter_flows(const struct ovn_datapaths *ls_datapaths, > lsiv[index].lr_ports = lr_ports; > lsiv[index].ls_port_groups = ls_pgs; > lsiv[index].lr_nats = lr_nats; > + lsiv[index].lr_sful_table = lr_sful_table; > lsiv[index].igmp_groups = igmp_groups; > lsiv[index].meter_groups = meter_groups; > lsiv[index].lb_dps_map = lb_dps_map; > @@ -16397,6 +16134,7 @@ build_lswitch_and_lrouter_flows(const struct ovn_datapaths *ls_datapaths, > .lr_ports = lr_ports, > .ls_port_groups = ls_pgs, > .lr_nats = lr_nats, > + .lr_sful_table = lr_sful_table, > .lflows = lflows, > .igmp_groups = igmp_groups, > .meter_groups = meter_groups, > @@ -16425,6 +16163,7 @@ build_lswitch_and_lrouter_flows(const struct ovn_datapaths *ls_datapaths, > build_lswitch_and_lrouter_iterate_by_lsp(op, lsi.ls_ports, > lsi.lr_ports, > lsi.lr_nats, > + lsi.lr_sful_table, > lsi.meter_groups, > &lsi.match, &lsi.actions, > lsi.lflows); > @@ -16546,6 +16285,7 @@ void build_lflows(struct ovsdb_idl_txn *ovnsb_txn, > input_data->lr_ports, > input_data->ls_port_groups, > input_data->lr_nats, > + input_data->lr_sful_table, > lflows, > &igmp_groups, > input_data->meter_groups, > @@ -17024,6 +16764,7 @@ lflow_handle_northd_port_changes(struct ovsdb_idl_txn *ovnsb_txn, > build_lswitch_and_lrouter_iterate_by_lsp(op, lflow_input->ls_ports, > lflow_input->lr_ports, > lflow_input->lr_nats, > + lflow_input->lr_sful_table, > lflow_input->meter_groups, > &match, &actions, > lflows); > @@ -17059,11 +16800,11 @@ lflow_handle_northd_port_changes(struct ovsdb_idl_txn *ovnsb_txn, > struct ds match = DS_EMPTY_INITIALIZER; > struct ds actions = DS_EMPTY_INITIALIZER; > build_lswitch_and_lrouter_iterate_by_lsp(op, lflow_input->ls_ports, > - lflow_input->lr_ports, > - lflow_input->lr_nats, > - lflow_input->meter_groups, > - &match, &actions, > - lflows); > + lflow_input->lr_ports, > + lflow_input->lr_nats, > + lflow_input->lr_sful_table, > + lflow_input->meter_groups, > + &match, &actions, lflows); > ds_destroy(&match); > ds_destroy(&actions); > > diff --git a/northd/northd.h b/northd/northd.h > index 4dd260761e..6d64fff626 100644 > --- a/northd/northd.h > +++ b/northd/northd.h > @@ -180,6 +180,7 @@ struct lflow_input { > const struct hmap *lr_ports; > const struct ls_port_group_table *ls_port_groups; > const struct lr_nat_table *lr_nats; > + const struct lr_stateful_table *lr_sful_table; > const struct shash *meter_groups; > const struct hmap *lb_datapaths_map; > const struct hmap *bfd_connections; > @@ -319,9 +320,6 @@ struct ovn_datapath { > /* router datapath has a logical port with redirect-type set to bridged. */ > bool redirect_bridged; > > - /* Load Balancer vIPs relevant for this datapath. */ > - struct ovn_lb_ip_set *lb_ips; > - > struct ovn_port **localnet_ports; > size_t n_localnet_ports; > > @@ -338,6 +336,119 @@ struct ovn_datapath { > const struct ovn_datapath *ovn_datapath_find(const struct hmap *datapaths, > const struct uuid *uuid); > > +/* A logical switch port or logical router port. > + * > + * In steady state, an ovn_port points to a northbound Logical_Switch_Port > + * record (via 'nbsp') *or* a Logical_Router_Port record (via 'nbrp'), and to a > + * southbound Port_Binding record (via 'sb'). As the state of the system > + * changes, join_logical_ports() may determine that there is a new LSP or LRP > + * that has no corresponding Port_Binding record (in which case build_ports()) > + * will create the missing Port_Binding) or that a Port_Binding record exists > + * that has no coresponding LSP (in which case build_ports() will delete the > + * spurious Port_Binding). Thus, after build_ports() runs, any given ovn_port > + * will have 'sb' nonnull, and 'nbsp' xor 'nbrp' nonnull. > + * > + * Ordinarily there is only one ovn_port that points to a given LSP or LRP (but > + * distributed gateway ports point a "derived" ovn_port to a duplicate LRP). > + */ > +struct ovn_port { > + /* Port name aka key. > + * > + * This is ordinarily the same as nbsp->name or nbrp->name and > + * sb->logical_port. (A distributed gateway port creates a "derived" > + * ovn_port with key "cr-%s" % nbrp->name.) */ > + struct hmap_node key_node; /* Index on 'key'. */ > + char *key; /* nbsp->name, nbrp->name, sb->logical_port. */ > + char *json_key; /* 'key', quoted for use in JSON. */ > + > + const struct sbrec_port_binding *sb; /* May be NULL. */ > + > + uint32_t tunnel_key; > + > + /* Logical switch port data. */ > + const struct nbrec_logical_switch_port *nbsp; /* May be NULL. */ > + > + struct lport_addresses *lsp_addrs; /* Logical switch port addresses. */ > + unsigned int n_lsp_addrs; /* Total length of lsp_addrs. */ > + unsigned int n_lsp_non_router_addrs; /* Number of elements from the > + * beginning of 'lsp_addrs' extracted > + * directly from LSP 'addresses'. */ > + > + struct lport_addresses *ps_addrs; /* Port security addresses. */ > + unsigned int n_ps_addrs; > + > + bool lsp_can_be_inc_processed; /* If it can be incrementally processed when > + the port changes. */ > + > + /* Logical router port data. */ > + const struct nbrec_logical_router_port *nbrp; /* May be NULL. */ > + > + struct lport_addresses lrp_networks; > + > + /* Logical port multicast data. */ > + struct mcast_port_info mcast_info; > + > + /* At most one of l3dgw_port and cr_port can be not NULL. */ > + > + /* This is set to a distributed gateway port if and only if this ovn_port > + * is "derived" from it. Otherwise this is set to NULL. The derived > + * ovn_port represents the instance of distributed gateway port on the > + * gateway chassis.*/ > + struct ovn_port *l3dgw_port; > + > + /* This is set to the "derived" chassis-redirect port of this port if and > + * only if this port is a distributed gateway port. Otherwise this is set > + * to NULL. */ > + struct ovn_port *cr_port; > + > + bool has_unknown; /* If the addresses have 'unknown' defined. */ > + > + bool has_bfd; > + > + /* The port's peer: > + * > + * - A switch port S of type "router" has a router port R as a peer, > + * and R in turn has S has its peer. > + * > + * - Two connected logical router ports have each other as peer. > + * > + * - Other kinds of ports have no peer. */ > + struct ovn_port *peer; > + > + struct ovn_datapath *od; > + > + struct ovs_list list; /* In list of similar records. */ > + > + struct hmap_node dp_node; /* Node in od->ports. */ > + > + struct lport_addresses proxy_arp_addrs; > + > + /* Temporarily used for traversing a list (or hmap) of ports. */ > + bool visited; > + > + /* List of struct lflow_ref_node that points to the lflows generated by > + * this ovn_port. > + * > + * This data is initialized and destroyed by the en_northd node, but > + * populated and used only by the en_lflow node. Ideally this data should > + * be maintained as part of en_lflow's data (struct lflow_data): a hash > + * index from ovn_port key to lflows. However, it would be less efficient > + * and more complex: > + * > + * 1. It would require an extra search (using the index) to find the > + * lflows. > + * > + * 2. Building the index needs to be thread-safe, using either a global > + * lock which is obviously less efficient, or hash-based lock array which > + * is more complex. > + * > + * Adding the list here is more straightforward. The drawback is that we > + * need to keep in mind that this data belongs to en_lflow node, so never > + * access it from any other nodes. > + */ > + struct ovs_list lflows; > +}; > + > void ovnnb_db_run(struct northd_input *input_data, > struct northd_data *data, > struct ovsdb_idl_txn *ovnnb_txn, > @@ -397,9 +508,13 @@ void sync_lbs(struct ovsdb_idl_txn *, const struct sbrec_load_balancer_table *, > struct chassis_features *chassis_features); > bool check_sb_lb_duplicates(const struct sbrec_load_balancer_table *); > > +struct lr_stateful_table; > void sync_pbs(struct ovsdb_idl_txn *, struct hmap *ls_ports, > - struct hmap *lr_ports, const struct lr_nat_table *); > -bool sync_pbs_for_northd_changed_ovn_ports(struct tracked_ovn_ports *); > + struct hmap *lr_ports, > + const struct lr_stateful_table *); > +bool sync_pbs_for_northd_changed_ovn_ports( > + struct tracked_ovn_ports *, > + const struct lr_stateful_table *); > > static inline bool > northd_has_tracked_data(struct northd_tracked_data *trk_nd_changes) { > @@ -424,4 +539,15 @@ northd_has_lr_nats_in_tracked_data(struct northd_tracked_data *trk_nd_changes) > return (trk_nd_changes->type & NORTHD_TRACKED_LR_NATS); > } > > +/* Returns 'true' if the IPv4 'addr' is on the same subnet with one of the > + * IPs configured on the router port. > + */ > +bool lrouter_port_ipv4_reachable(const struct ovn_port *, ovs_be32 addr); > + > +/* Returns 'true' if the IPv6 'addr' is on the same subnet with one of the > + * IPs configured on the router port. > + */ > +bool lrouter_port_ipv6_reachable(const struct ovn_port *, > + const struct in6_addr *); > + > #endif /* NORTHD_H */ > diff --git a/northd/ovn-northd.c b/northd/ovn-northd.c > index 40f9764b3a..e85e00cc9b 100644 > --- a/northd/ovn-northd.c > +++ b/northd/ovn-northd.c > @@ -871,6 +871,7 @@ main(int argc, char *argv[]) > stopwatch_create(PORT_GROUP_RUN_STOPWATCH_NAME, SW_MS); > stopwatch_create(SYNC_METERS_RUN_STOPWATCH_NAME, SW_MS); > stopwatch_create(LR_NAT_RUN_STOPWATCH_NAME, SW_MS); > + stopwatch_create(LR_STATEFUL_RUN_STOPWATCH_NAME, SW_MS); > > /* Initialize incremental processing engine for ovn-northd */ > inc_proc_northd_init(&ovnnb_idl_loop, &ovnsb_idl_loop); > diff --git a/tests/ovn-northd.at b/tests/ovn-northd.at > index dc01e43de0..03d62695db 100644 > --- a/tests/ovn-northd.at > +++ b/tests/ovn-northd.at > @@ -10516,18 +10516,21 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb set load_balancer . ip_port_mappings:10.0.0.3=sw0-p1:10.0.0.2 > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute nocompute > > check ovn-nbctl --wait=sb set load_balancer . options:foo=bar > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute nocompute > > check ovn-nbctl --wait=sb -- lb-add lb2 20.0.0.10:80 20.0.0.20:80 -- lb-add lb3 30.0.0.10:80 30.0.0.20:80 > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute nocompute > > @@ -10537,6 +10540,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb -- lb-del lb2 -- lb-del lb3 > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute nocompute > > @@ -10550,6 +10554,7 @@ AT_CHECK([ovn-nbctl --wait=sb \ > ]) > check_engine_stats lb_data norecompute compute > check_engine_stats northd recompute nocompute > +check_engine_stats lr_stateful recompute nocompute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute nocompute > > @@ -10567,6 +10572,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb clear Load_Balancer . health_check > check_engine_stats lb_data norecompute compute > check_engine_stats northd recompute nocompute > +check_engine_stats lr_stateful recompute nocompute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute nocompute > > @@ -10581,6 +10587,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > ovn-nbctl --wait=sb lsp-set-options sw0-lr0 router-port=lr0-sw0 > check_engine_stats lb_data norecompute compute > check_engine_stats northd recompute compute > +check_engine_stats lr_stateful recompute nocompute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute nocompute > > @@ -10589,6 +10596,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb ls-lb-add sw0 lb1 > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > # A LB applied to a switch/router triggers: > # - a recompute in the first iteration (handling northd change) > @@ -10601,6 +10609,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb set load_balancer lb1 vips:'"10.0.0.10:80"'='"10.0.0.100:80"' > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute compute > CHECK_NO_CHANGE_AFTER_RECOMPUTE(1) > @@ -10610,6 +10619,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb clear load_Balancer lb1 vips > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute compute > CHECK_NO_CHANGE_AFTER_RECOMPUTE(1) > @@ -10619,6 +10629,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb lb-add lb1 10.0.0.10:80 10.0.0.3:80 > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute compute > CHECK_NO_CHANGE_AFTER_RECOMPUTE(1) > @@ -10628,6 +10639,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb lb-add lb1 10.0.0.20:80 10.0.0.30:8080 > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute compute > CHECK_NO_CHANGE_AFTER_RECOMPUTE(1) > @@ -10637,6 +10649,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb ls-lb-del sw0 lb1 > check_engine_stats lb_data norecompute compute > check_engine_stats northd recompute nocompute > +check_engine_stats lr_stateful recompute nocompute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute compute > CHECK_NO_CHANGE_AFTER_RECOMPUTE(1) > @@ -10647,6 +10660,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb ls-lb-add sw0 lb1 -- lsp-add sw0 sw0p1 > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute compute > > @@ -10667,6 +10681,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb lr-lb-add lr0 lb1 > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute compute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -10676,6 +10691,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb set load_balancer lb1 vips:'"10.0.0.10:80"'='"10.0.0.100:80"' > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute compute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -10685,6 +10701,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb clear load_Balancer lb1 vips > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute compute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -10694,6 +10711,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb lb-add lb1 10.0.0.10:80 10.0.0.3:80 > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute compute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -10703,6 +10721,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb lb-add lb1 10.0.0.20:80 10.0.0.30:8080 > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute compute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -10733,6 +10752,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb add load_balancer_group . load_Balancer $lb1_uuid > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute nocompute > > @@ -10740,6 +10760,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb clear load_balancer_group . load_Balancer > check_engine_stats lb_data norecompute compute > check_engine_stats northd recompute nocompute > +check_engine_stats lr_stateful recompute nocompute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute nocompute > > @@ -10756,6 +10777,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb add logical_switch sw0 load_balancer_group $lbg1_uuid > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute compute > > @@ -10772,6 +10794,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb set load_balancer lb1 vips:'"10.0.0.10:80"'='"10.0.0.100:80"' > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute compute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -10781,6 +10804,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb clear load_Balancer lb1 vips > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute compute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -10790,6 +10814,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb lb-add lb1 10.0.0.10:80 10.0.0.3:80 > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute compute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -10799,6 +10824,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb lb-add lb1 10.0.0.20:80 10.0.0.30:8080 > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute compute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -10814,6 +10840,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl add logical_router lr0 load_balancer_group $lbg1_uuid > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute compute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -10823,6 +10850,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb set load_balancer lb1 vips:'"10.0.0.10:80"'='"10.0.0.100:80"' > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute compute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -10832,6 +10860,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb clear load_Balancer lb1 vips > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute compute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -10841,6 +10870,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb lb-add lb1 10.0.0.10:80 10.0.0.3:80 > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute compute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -10850,6 +10880,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb lb-add lb1 10.0.0.20:80 10.0.0.30:8080 > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute compute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -10858,6 +10889,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb clear logical_router lr0 load_balancer_group > check_engine_stats lb_data norecompute compute > check_engine_stats northd recompute nocompute > +check_engine_stats lr_stateful recompute nocompute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute compute > > @@ -10866,6 +10898,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb add logical_switch sw0 load_balancer_group $lbg1_uuid > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute compute > > @@ -10874,6 +10907,7 @@ check ovn-nbctl --wait=sb clear logical_switch sw0 load_balancer_group -- \ > destroy load_balancer_group $lbg1_uuid > check_engine_stats lb_data norecompute compute > check_engine_stats northd recompute nocompute > +check_engine_stats lr_stateful recompute nocompute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb compute compute > > @@ -10897,6 +10931,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > lbg1_uuid=$(ovn-nbctl --wait=sb create load_balancer_group name=lbg1) > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow norecompute nocompute > check_engine_stats sync_to_sb_lb norecompute nocompute > > @@ -10904,6 +10939,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb set load_balancer_group . load_balancer="$lb2_uuid,$lb3_uuid,$lb4_uuid" > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute nocompute > > @@ -10911,6 +10947,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb set logical_switch sw0 load_balancer_group=$lbg1_uuid > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute compute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -10919,6 +10956,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb set logical_router lr1 load_balancer_group=$lbg1_uuid > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute compute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -10927,6 +10965,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb ls-lb-add sw0 lb2 > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute nocompute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -10935,6 +10974,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb ls-lb-add sw0 lb3 > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute nocompute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -10944,6 +10984,7 @@ check ovn-nbctl --wait=sb lr-lb-add lr1 lb1 > check ovn-nbctl --wait=sb lr-lb-add lr1 lb2 > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute compute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -10952,6 +10993,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb ls-lb-del sw0 lb2 > check_engine_stats lb_data norecompute compute > check_engine_stats northd recompute nocompute > +check_engine_stats lr_stateful recompute nocompute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute nocompute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -10960,6 +11002,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb lr-lb-del lr1 lb2 > check_engine_stats lb_data norecompute compute > check_engine_stats northd recompute nocompute > +check_engine_stats lr_stateful recompute nocompute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute nocompute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -10970,6 +11013,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb lb-del lb4 > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute compute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -10980,6 +11024,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb lb-del lb2 > check_engine_stats lb_data norecompute compute > check_engine_stats northd norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute compute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -10988,6 +11033,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb remove load_balancer_group . load_balancer $lb3_uuid > check_engine_stats lb_data norecompute compute > check_engine_stats northd recompute nocompute > +check_engine_stats lr_stateful recompute nocompute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_lb recompute compute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -11054,6 +11100,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb lr-add lr0 > check_engine_stats northd recompute nocompute > check_engine_stats lr_nat recompute nocompute > +check_engine_stats lr_stateful recompute nocompute > check_engine_stats sync_to_sb_pb recompute nocompute > check_engine_stats lflow recompute nocompute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -11066,6 +11113,7 @@ check ovn-nbctl lrp-add lr0 lr0-sw0 00:00:00:00:ff:01 10.0.0.1/24 > # for the SB port binding change. > check_engine_stats northd recompute compute > check_engine_stats lr_nat recompute nocompute > +check_engine_stats lr_stateful recompute nocompute > check_engine_stats sync_to_sb_pb recompute nocompute > check_engine_stats lflow recompute nocompute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -11077,6 +11125,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb lsp-set-options sw0-lr0 router-port=lr0-sw0 > check_engine_stats northd recompute compute > check_engine_stats lr_nat recompute nocompute > +check_engine_stats lr_stateful recompute nocompute > check_engine_stats sync_to_sb_pb recompute nocompute > check_engine_stats lflow recompute nocompute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -11102,6 +11151,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl set logical_router_port lr0-sw0 options:foo=bar > check_engine_stats northd recompute nocompute > check_engine_stats lr_nat recompute nocompute > +check_engine_stats lr_stateful recompute nocompute > check_engine_stats sync_to_sb_pb recompute nocompute > check_engine_stats lflow recompute nocompute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -11131,6 +11181,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb set NAT . external_ip=172.168.0.120 > check_engine_stats northd norecompute compute > check_engine_stats lr_nat norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_pb recompute nocompute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -11140,6 +11191,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb set NAT . logical_ip=10.0.0.10 > check_engine_stats northd norecompute compute > check_engine_stats lr_nat norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_pb recompute nocompute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -11149,6 +11201,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb set NAT . type=snat > check_engine_stats northd norecompute compute > check_engine_stats lr_nat norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_pb recompute nocompute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -11158,6 +11211,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb lr-nat-add lr0 dnat_and_snat 172.168.0.110 10.0.0.4 sw0p1 30:54:00:00:00:03 > check_engine_stats northd norecompute compute > check_engine_stats lr_nat norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_pb recompute nocompute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -11168,6 +11222,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb set NAT $nat2_uuid external_mac='"30:54:00:00:00:04"' > check_engine_stats northd norecompute compute > check_engine_stats lr_nat norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_pb recompute nocompute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -11184,6 +11239,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb lr-nat-add lr0 dnat_and_snat 172.168.0.140 10.0.0.20 > check_engine_stats northd norecompute compute > check_engine_stats lr_nat norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats sync_to_sb_pb recompute nocompute > check_engine_stats lflow recompute nocompute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -11192,6 +11248,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb lr-nat-add lr0 dnat_and_snat 172.168.0.150 10.0.0.41 > check_engine_stats northd norecompute compute > check_engine_stats lr_nat norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats sync_to_sb_pb recompute nocompute > check_engine_stats lflow recompute nocompute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -11200,6 +11257,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb lr-nat-del lr0 dnat_and_snat 172.168.0.150 > check_engine_stats northd norecompute compute > check_engine_stats lr_nat norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats sync_to_sb_pb recompute nocompute > check_engine_stats lflow recompute nocompute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -11208,6 +11266,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb lr-nat-del lr0 dnat_and_snat 172.168.0.140 > check_engine_stats northd norecompute compute > check_engine_stats lr_nat norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats sync_to_sb_pb recompute nocompute > check_engine_stats lflow recompute nocompute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -11217,6 +11276,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb clear logical_router lr0 nat > check_engine_stats northd norecompute compute > check_engine_stats lr_nat norecompute compute > +check_engine_stats lr_stateful norecompute compute > check_engine_stats lflow recompute nocompute > check_engine_stats sync_to_sb_pb recompute nocompute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -11226,6 +11286,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb lr-policy-add lr0 10 "ip4.src == 10.0.0.3" reroute 172.168.0.101,172.168.0.102 > check_engine_stats northd recompute nocompute > check_engine_stats lr_nat recompute nocompute > +check_engine_stats lr_stateful recompute nocompute > check_engine_stats sync_to_sb_pb recompute nocompute > check_engine_stats lflow recompute nocompute > CHECK_NO_CHANGE_AFTER_RECOMPUTE > @@ -11234,6 +11295,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > check ovn-nbctl --wait=sb lr-policy-del lr0 10 "ip4.src == 10.0.0.3" > check_engine_stats northd recompute nocompute > check_engine_stats lr_nat recompute nocompute > +check_engine_stats lr_stateful recompute nocompute > check_engine_stats sync_to_sb_pb recompute nocompute > check_engine_stats lflow recompute nocompute > CHECK_NO_CHANGE_AFTER_RECOMPUTE
On Wed, Dec 13, 2023 at 8:55 AM Dumitru Ceara <dceara@redhat.com> wrote: > > On 11/28/23 03:35, numans@ovn.org wrote: > > From: Numan Siddique <numans@ovn.org> > > > > This new engine now maintains the load balancer and NAT data of a > > logical router which was earlier part of northd engine node data. > > The main inputs to this engine are: > > - northd node > > - lr_nat node > > - lb_data node > > > > A record for each logical router is maintained in the 'lr_stateful_table' > > hmap table and this record > > - stores the lb related data > > - embeds the 'lr_nat' record. > > > > This engine node becomes an input to 'lflow' node. > > > > Signed-off-by: Numan Siddique <numans@ovn.org> > > --- > > lib/stopwatch-names.h | 1 + > > northd/automake.mk | 2 + > > northd/en-lflow.c | 4 + > > northd/en-lr-nat.h | 3 + > > northd/en-lr-stateful.c | 647 ++++++++++++++++++++++++++++++++++++++ > > northd/en-lr-stateful.h | 104 +++++++ > > northd/en-sync-sb.c | 49 +-- > > northd/inc-proc-northd.c | 13 +- > > northd/northd.c | 655 ++++++++++++--------------------------- > > northd/northd.h | 136 +++++++- > > northd/ovn-northd.c | 1 + > > tests/ovn-northd.at | 62 ++++ > > 12 files changed, 1191 insertions(+), 486 deletions(-) > > create mode 100644 northd/en-lr-stateful.c > > create mode 100644 northd/en-lr-stateful.h > > > > diff --git a/lib/stopwatch-names.h b/lib/stopwatch-names.h > > index 0a16da211e..63a04db9ca 100644 > > --- a/lib/stopwatch-names.h > > +++ b/lib/stopwatch-names.h > > @@ -33,5 +33,6 @@ > > #define PORT_GROUP_RUN_STOPWATCH_NAME "port_group_run" > > #define SYNC_METERS_RUN_STOPWATCH_NAME "sync_meters_run" > > #define LR_NAT_RUN_STOPWATCH_NAME "lr_nat_run" > > +#define LR_STATEFUL_RUN_STOPWATCH_NAME "lr_stateful" > > > > #endif > > diff --git a/northd/automake.mk b/northd/automake.mk > > index ae367a2a8b..3666cd452c 100644 > > --- a/northd/automake.mk > > +++ b/northd/automake.mk > > @@ -26,6 +26,8 @@ northd_ovn_northd_SOURCES = \ > > northd/en-lb-data.h \ > > northd/en-lr-nat.c \ > > northd/en-lr-nat.h \ > > + northd/en-lr-stateful.c \ > > + northd/en-lr-stateful.h \ > > northd/inc-proc-northd.c \ > > northd/inc-proc-northd.h \ > > northd/ipam.c \ > > diff --git a/northd/en-lflow.c b/northd/en-lflow.c > > index e4f875ef7c..0aee073520 100644 > > --- a/northd/en-lflow.c > > +++ b/northd/en-lflow.c > > @@ -20,6 +20,7 @@ > > > > #include "en-lflow.h" > > #include "en-lr-nat.h" > > +#include "en-lr-stateful.h" > > #include "en-northd.h" > > #include "en-meters.h" > > > > @@ -43,6 +44,8 @@ lflow_get_input_data(struct engine_node *node, > > engine_get_input_data("sync_meters", node); > > struct ed_type_lr_nat_data *lr_nat_data = > > engine_get_input_data("lr_nat", node); > > + struct ed_type_lr_stateful *lr_sful_data = > > + engine_get_input_data("lr_stateful", node); > > > > lflow_input->nbrec_bfd_table = > > EN_OVSDB_GET(engine_get_input("NB_bfd", node)); > > @@ -66,6 +69,7 @@ lflow_get_input_data(struct engine_node *node, > > lflow_input->lr_ports = &northd_data->lr_ports; > > lflow_input->ls_port_groups = &pg_data->ls_port_groups; > > lflow_input->lr_nats = &lr_nat_data->lr_nats; > > + lflow_input->lr_sful_table = &lr_sful_data->lr_sful_table; > > lflow_input->meter_groups = &sync_meters_data->meter_groups; > > lflow_input->lb_datapaths_map = &northd_data->lb_datapaths_map; > > lflow_input->svc_monitor_map = &northd_data->svc_monitor_map; > > diff --git a/northd/en-lr-nat.h b/northd/en-lr-nat.h > > index 3ec4c7b506..4d8e142ae6 100644 > > --- a/northd/en-lr-nat.h > > +++ b/northd/en-lr-nat.h > > @@ -86,6 +86,9 @@ struct lr_nat_table { > > const struct lr_nat_record * lr_nat_table_find_by_index( > > const struct lr_nat_table *, size_t od_index); > > > > +#define LR_NAT_TABLE_FOR_EACH(LR_NAT_REC, TABLE) \ > > + HMAP_FOR_EACH (LR_NAT_REC, key_node, &(TABLE)->entries) > > + > > struct ed_type_lr_nat_data { > > struct lr_nat_table lr_nats; > > > > diff --git a/northd/en-lr-stateful.c b/northd/en-lr-stateful.c > > new file mode 100644 > > index 0000000000..bd03c586da > > --- /dev/null > > +++ b/northd/en-lr-stateful.c > > @@ -0,0 +1,647 @@ > > +/* > > + * Copyright (c) 2023, Red Hat, Inc. > > + * > > + * Licensed under the Apache License, Version 2.0 (the "License"); > > + * you may not use this file except in compliance with the License. > > + * You may obtain a copy of the License at: > > + * > > + * http://www.apache.org/licenses/LICENSE-2.0 > > + * > > + * Unless required by applicable law or agreed to in writing, software > > + * distributed under the License is distributed on an "AS IS" BASIS, > > + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. > > + * See the License for the specific language governing permissions and > > + * limitations under the License. > > + */ > > + > > +#include <config.h> > > + > > +#include <getopt.h> > > +#include <stdlib.h> > > +#include <stdio.h> > > + > > +/* OVS includes */ > > +#include "include/openvswitch/hmap.h" > > +#include "lib/bitmap.h" > > +#include "lib/socket-util.h" > > +#include "lib/uuidset.h" > > +#include "openvswitch/util.h" > > +#include "openvswitch/vlog.h" > > +#include "stopwatch.h" > > + > > +/* OVN includes */ > > +#include "en-lb-data.h" > > +#include "en-lr-nat.h" > > +#include "en-lr-stateful.h" > > +#include "lib/inc-proc-eng.h" > > +#include "lib/lb.h" > > +#include "lib/ovn-nb-idl.h" > > +#include "lib/ovn-sb-idl.h" > > +#include "lib/ovn-util.h" > > +#include "lib/stopwatch-names.h" > > +#include "northd.h" > > + > > +VLOG_DEFINE_THIS_MODULE(en_lr_stateful); > > + > > +/* Static function declarations. */ > > +static void lr_stateful_table_init(struct lr_stateful_table *); > > +static void lr_stateful_table_clear(struct lr_stateful_table *); > > +static void lr_stateful_table_destroy(struct lr_stateful_table *); > > +static struct lr_stateful_record *lr_stateful_table_find_( > > + const struct lr_stateful_table *, const struct nbrec_logical_router *); > > +static struct lr_stateful_record *lr_stateful_table_find_by_index_( > > + const struct lr_stateful_table *table, size_t od_index); > > + > > +static void lr_stateful_table_build(struct lr_stateful_table *, > > + const struct lr_nat_table *, > > + const struct ovn_datapaths *lr_datapaths, > > + const struct hmap *lb_datapaths_map, > > + const struct hmap *lbgrp_datapaths_map); > > + > > +static struct lr_stateful_input lr_stateful_get_input_data( > > + struct engine_node *); > > + > > +static struct lr_stateful_record *lr_stateful_record_create( > > + struct lr_stateful_table *, const struct lr_nat_record *, > > + const struct hmap *lb_datapaths_map, > > + const struct hmap *lbgrp_datapaths_map); > > +static void lr_stateful_record_destroy(struct lr_stateful_record *); > > +static void lr_stateful_record_init( > > + struct lr_stateful_record *, > > + const struct hmap *lb_datapaths_map, > > + const struct hmap *lbgrp_datapaths_map); > > + > > +static void build_lrouter_lb_reachable_ips(struct lr_stateful_record *, > > + const struct ovn_northd_lb *); > > +static void add_neigh_ips_to_lrouter(struct lr_stateful_record *, > > + enum lb_neighbor_responder_mode, > > + const struct sset *lb_ips_v4, > > + const struct sset *lb_ips_v6); > > +static void remove_lrouter_lb_reachable_ips(struct lr_stateful_record *, > > + enum lb_neighbor_responder_mode, > > + const struct sset *lb_ips_v4, > > + const struct sset *lb_ips_v6); > > +static void lr_stateful_build_vip_nats(struct lr_stateful_record *); > > + > > +/* 'lr_stateful' engine node manages the NB logical router LB data. > > + */ > > +void * > > +en_lr_stateful_init(struct engine_node *node OVS_UNUSED, > > + struct engine_arg *arg OVS_UNUSED) > > +{ > > + struct ed_type_lr_stateful *data = xzalloc(sizeof *data); > > + lr_stateful_table_init(&data->lr_sful_table); > > + hmapx_init(&data->trk_data.crupdated); > > + return data; > > +} > > + > > +void > > +en_lr_stateful_cleanup(void *data_) > > +{ > > + struct ed_type_lr_stateful *data = > > + (struct ed_type_lr_stateful *) data_; > > + lr_stateful_table_destroy(&data->lr_sful_table); > > + hmapx_destroy(&data->trk_data.crupdated); > > +} > > + > > +void > > +en_lr_stateful_clear_tracked_data(void *data_) > > +{ > > + struct ed_type_lr_stateful *data = (struct ed_type_lr_stateful *) data_; > > + > > + hmapx_clear(&data->trk_data.crupdated); > > +} > > + > > +void > > +en_lr_stateful_run(struct engine_node *node, void *data_) > > +{ > > + struct lr_stateful_input input_data = lr_stateful_get_input_data(node); > > + struct ed_type_lr_stateful *data = data_; > > + > > + stopwatch_start(LR_STATEFUL_RUN_STOPWATCH_NAME, time_msec()); > > + > > + lr_stateful_table_clear(&data->lr_sful_table); > > + lr_stateful_table_build(&data->lr_sful_table, input_data.lr_nats, > > + input_data.lr_datapaths, > > + input_data.lb_datapaths_map, > > + input_data.lbgrp_datapaths_map); > > + > > + stopwatch_stop(LR_STATEFUL_RUN_STOPWATCH_NAME, time_msec()); > > + engine_set_node_state(node, EN_UPDATED); > > +} > > + > > +bool > > +lr_stateful_northd_handler(struct engine_node *node, void *data OVS_UNUSED) > > +{ > > + struct northd_data *northd_data = engine_get_input_data("northd", node); > > + if (!northd_has_tracked_data(&northd_data->trk_data)) { > > + return false; > > + } > > + > > + /* lr_stateful node needs inputs for any changes to NAT and load balancers. > > + * Changes to NAT is provided by the lr_nat tracked data and changes > > + * to lbs and lb grps is provided by lb_data's tracked data. > > + * So we don't need to do anything here for northd changes. > > + * But we do need to access the datapaths and lb_datapaths from the > > + * northd engine node and hence its an input. > > + * */ > > + return true; > > +} > > + > > +bool > > +lr_stateful_lb_data_handler(struct engine_node *node, void *data_) > > +{ > > + struct ed_type_lb_data *lb_data = engine_get_input_data("lb_data", node); > > + if (!lb_data->tracked) { > > + return false; > > + } > > + > > + struct ed_type_lr_stateful *data = > > + (struct ed_type_lr_stateful *) data_; > > + struct lr_stateful_input input_data = > > + lr_stateful_get_input_data(node); > > + struct lr_stateful_record *lr_sful_rec; > > I'd move this definition (lr_sful_rec) inside of the appropriate loops > where this is used. That's to avoid it having unexpected values. Ack. Done in v4 > > > + size_t index; > > + > > + const struct tracked_lb_data *trk_lb_data = &lb_data->tracked_lb_data; > > + const struct ovn_lb_group_datapaths *lbgrp_dps; > > + const struct crupdated_lbgrp *crupdated_lbgrp; > > + const struct crupdated_od_lb_data *codlb; > > + const struct ovn_lb_datapaths *lb_dps; > > + const struct crupdated_lb *clb; > > + const struct ovn_northd_lb *lb; > > + const struct ovn_datapath *od; > > Same comment about these, let's reduce the scope of the variables if > possible. Ack. Done in v4 > > > + > > + LIST_FOR_EACH (codlb, list_node, &trk_lb_data->crupdated_lr_lbs) { > > + od = ovn_datapath_find(&input_data.lr_datapaths->datapaths, > > + &codlb->od_uuid); > > + ovs_assert(od); > > + > > + lr_sful_rec = lr_stateful_table_find_(&data->lr_sful_table, od->nbr); > > + if (!lr_sful_rec) { > > + const struct lr_nat_record *lrnat_rec = lr_nat_table_find_by_index( > > + input_data.lr_nats, od->index); > > + ovs_assert(lrnat_rec); > > + > > + lr_sful_rec = lr_stateful_record_create(&data->lr_sful_table, > > + lrnat_rec, > > + input_data.lb_datapaths_map, > > + input_data.lbgrp_datapaths_map); > > + > > + /* Add the lr_sful_rec rec to the tracking data. */ > > + hmapx_add(&data->trk_data.crupdated, lr_sful_rec); > > + continue; > > + } > > + > > Nit: it might help if we had a comment here to remind the reader that > we're now in the "update" case, e.g.: > > /* Update. */ Ack. Done in v4 > > > + struct uuidset_node *uuidnode; > > + UUIDSET_FOR_EACH (uuidnode, &codlb->assoc_lbs) { > > + lb_dps = ovn_lb_datapaths_find( > > + input_data.lb_datapaths_map, &uuidnode->uuid); > > + ovs_assert(lb_dps); > > + > > + /* Add the lb_ips of lb_dps to the od. */ > > + build_lrouter_lb_ips(lr_sful_rec->lb_ips, lb_dps->lb); > > + build_lrouter_lb_reachable_ips(lr_sful_rec, lb_dps->lb); > > + } > > + > > + UUIDSET_FOR_EACH (uuidnode, &codlb->assoc_lbgrps) { > > + lbgrp_dps = ovn_lb_group_datapaths_find( > > + input_data.lbgrp_datapaths_map, &uuidnode->uuid); > > + ovs_assert(lbgrp_dps); > > + > > + for (size_t j = 0; j < lbgrp_dps->lb_group->n_lbs; j++) { > > + const struct uuid *lb_uuid > > + = &lbgrp_dps->lb_group->lbs[j]->nlb->header_.uuid; > > + lb_dps = ovn_lb_datapaths_find(input_data.lb_datapaths_map, > > + lb_uuid); > > + ovs_assert(lb_dps); > > + > > + /* Add the lb_ips of lb_dps to the od. */ > > + build_lrouter_lb_ips(lr_sful_rec->lb_ips, lb_dps->lb); > > + build_lrouter_lb_reachable_ips(lr_sful_rec, lb_dps->lb); > > + } > > + } > > + > > + /* Add the lr_sful_rec rec to the tracking data. */ > > + hmapx_add(&data->trk_data.crupdated, lr_sful_rec); > > + } > > + > > + HMAP_FOR_EACH (clb, hmap_node, &trk_lb_data->crupdated_lbs) { > > + lb = clb->lb; > > + const struct uuid *lb_uuid = &lb->nlb->header_.uuid; > > + > > + lb_dps = ovn_lb_datapaths_find(input_data.lb_datapaths_map, lb_uuid); > > + ovs_assert(lb_dps); > > + > > + BITMAP_FOR_EACH_1 (index, ods_size(input_data.lr_datapaths), > > + lb_dps->nb_lr_map) { > > + od = input_data.lr_datapaths->array[index]; > > + > > + lr_sful_rec = lr_stateful_table_find_(&data->lr_sful_table, > > + od->nbr); > > Nit: indent. Ack. Done in v4 Thanks Numan > > > + ovs_assert(lr_sful_rec); > > + > > + /* Update the od->lb_ips with the deleted and inserted > > + * vips (if any). */ > > + remove_ips_from_lb_ip_set(lr_sful_rec->lb_ips, lb->routable, > > + &clb->deleted_vips_v4, > > + &clb->deleted_vips_v6); > > + add_ips_to_lb_ip_set(lr_sful_rec->lb_ips, lb->routable, > > + &clb->inserted_vips_v4, > > + &clb->inserted_vips_v6); > > + > > + remove_lrouter_lb_reachable_ips(lr_sful_rec, lb->neigh_mode, > > + &clb->deleted_vips_v4, > > + &clb->deleted_vips_v6); > > + add_neigh_ips_to_lrouter(lr_sful_rec, lb->neigh_mode, > > + &clb->inserted_vips_v4, > > + &clb->inserted_vips_v6); > > + > > + /* Add the lr_sful_rec rec to the tracking data. */ > > + hmapx_add(&data->trk_data.crupdated, lr_sful_rec); > > + } > > + } > > + > > + HMAP_FOR_EACH (crupdated_lbgrp, hmap_node, > > + &trk_lb_data->crupdated_lbgrps) { > > + const struct uuid *lb_uuid = &crupdated_lbgrp->lbgrp->uuid; > > + > > + lbgrp_dps = ovn_lb_group_datapaths_find(input_data.lbgrp_datapaths_map, > > + lb_uuid); > > + ovs_assert(lbgrp_dps); > > + > > + struct hmapx_node *hnode; > > + HMAPX_FOR_EACH (hnode, &crupdated_lbgrp->assoc_lbs) { > > + lb = hnode->data; > > + lb_uuid = &lb->nlb->header_.uuid; > > + lb_dps = ovn_lb_datapaths_find(input_data.lb_datapaths_map, > > + lb_uuid); > > + ovs_assert(lb_dps); > > + for (size_t i = 0; i < lbgrp_dps->n_lr; i++) { > > + od = lbgrp_dps->lr[i]; > > + lr_sful_rec = lr_stateful_table_find_(&data->lr_sful_table, > > + od->nbr); > > + ovs_assert(lr_sful_rec); > > + /* Add the lb_ips of lb_dps to the lr lb data. */ > > + build_lrouter_lb_ips(lr_sful_rec->lb_ips, lb_dps->lb); > > + build_lrouter_lb_reachable_ips(lr_sful_rec, lb_dps->lb); > > + > > + /* Add the lr_sful_rec rec to the tracking data. */ > > + hmapx_add(&data->trk_data.crupdated, lr_sful_rec); > > + } > > + } > > + } > > + > > + if (lr_stateful_has_tracked_data(&data->trk_data)) { > > + struct hmapx_node *hmapx_node; > > + /* For all the modified lr_stateful records (re)build the > > + * vip nats. */ > > + HMAPX_FOR_EACH (hmapx_node, &data->trk_data.crupdated) { > > + lr_stateful_build_vip_nats(hmapx_node->data); > > + } > > + > > + engine_set_node_state(node, EN_UPDATED); > > + } > > + > > + return true; > > +} > > + > > +bool > > +lr_stateful_lr_nat_handler(struct engine_node *node, void *data_) > > +{ > > + struct ed_type_lr_nat_data *lr_nat_data = > > + engine_get_input_data("lr_nat", node); > > + > > + if (!lr_nat_has_tracked_data(&lr_nat_data->trk_data)) { > > + return false; > > + } > > + > > + struct ed_type_lr_stateful *data = > > + (struct ed_type_lr_stateful *) data_; > > + struct lr_stateful_input input_data = > > + lr_stateful_get_input_data(node); > > + const struct lr_nat_record *lrnat_rec; > > + struct lr_stateful_record *lr_sful_rec; > > + struct hmapx_node *hmapx_node; > > + > > + HMAPX_FOR_EACH (hmapx_node, &lr_nat_data->trk_data.crupdated) { > > + lrnat_rec = hmapx_node->data; > > + lr_sful_rec = lr_stateful_table_find_(&data->lr_sful_table, > > + lrnat_rec->od->nbr); > > + if (!lr_sful_rec) { > > + lr_sful_rec = lr_stateful_record_create(&data->lr_sful_table, > > + lrnat_rec, > > + input_data.lb_datapaths_map, > > + input_data.lbgrp_datapaths_map); > > + } else { > > + lr_stateful_build_vip_nats(lr_sful_rec); > > + } > > + > > + /* Add the lr_sful_rec rec to the tracking data. */ > > + hmapx_add(&data->trk_data.crupdated, lr_sful_rec); > > + } > > + > > + if (lr_stateful_has_tracked_data(&data->trk_data)) { > > + engine_set_node_state(node, EN_UPDATED); > > + } > > + > > + return true; > > +} > > + > > +const struct lr_stateful_record * > > +lr_stateful_table_find_by_index(const struct lr_stateful_table *table, > > + size_t od_index) > > +{ > > + return lr_stateful_table_find_by_index_(table, od_index); > > +} > > + > > +/* static functions. */ > > +static void > > +lr_stateful_table_init(struct lr_stateful_table *table) > > +{ > > + *table = (struct lr_stateful_table) { > > + .entries = HMAP_INITIALIZER(&table->entries), > > + }; > > +} > > + > > +static void > > +lr_stateful_table_destroy(struct lr_stateful_table *table) > > +{ > > + lr_stateful_table_clear(table); > > + hmap_destroy(&table->entries); > > +} > > + > > +static void > > +lr_stateful_table_clear(struct lr_stateful_table *table) > > +{ > > + struct lr_stateful_record *lr_sful_rec; > > + HMAP_FOR_EACH_POP (lr_sful_rec, key_node, &table->entries) { > > + lr_stateful_record_destroy(lr_sful_rec); > > + } > > + > > + free(table->array); > > + table->array = NULL; > > +} > > + > > +static void > > +lr_stateful_table_build(struct lr_stateful_table *table, > > + const struct lr_nat_table *lr_nats, > > + const struct ovn_datapaths *lr_datapaths, > > + const struct hmap *lb_datapaths_map, > > + const struct hmap *lbgrp_datapaths_map) > > +{ > > + table->array = xrealloc(table->array, > > + ods_size(lr_datapaths) * sizeof *table->array); > > + const struct lr_nat_record *lrnat_rec; > > + LR_NAT_TABLE_FOR_EACH (lrnat_rec, lr_nats) { > > + lr_stateful_record_create(table, lrnat_rec, lb_datapaths_map, > > + lbgrp_datapaths_map); > > + } > > +} > > + > > +static struct lr_stateful_record * > > +lr_stateful_table_find_(const struct lr_stateful_table *table, > > + const struct nbrec_logical_router *nbr) > > +{ > > + struct lr_stateful_record *lr_sful_rec; > > + > > + HMAP_FOR_EACH_WITH_HASH (lr_sful_rec, key_node, > > + uuid_hash(&nbr->header_.uuid), &table->entries) { > > + if (nbr == lr_sful_rec->od->nbr) { > > + return lr_sful_rec; > > + } > > + } > > + return NULL; > > +} > > + > > +static struct lr_stateful_record * > > +lr_stateful_table_find_by_index_(const struct lr_stateful_table *table, > > + size_t od_index) > > +{ > > + ovs_assert(od_index <= hmap_count(&table->entries)); > > + return table->array[od_index]; > > +} > > + > > +static struct lr_stateful_record * > > +lr_stateful_record_create(struct lr_stateful_table *table, > > + const struct lr_nat_record *lrnat_rec, > > + const struct hmap *lb_datapaths_map, > > + const struct hmap *lbgrp_datapaths_map) > > +{ > > + struct lr_stateful_record *lr_sful_rec = xzalloc(sizeof *lr_sful_rec); > > + lr_sful_rec->lrnat_rec = lrnat_rec; > > + lr_sful_rec->od = lrnat_rec->od; > > + lr_stateful_record_init(lr_sful_rec, lb_datapaths_map, > > + lbgrp_datapaths_map); > > + > > + hmap_insert(&table->entries, &lr_sful_rec->key_node, > > + uuid_hash(&lr_sful_rec->od->nbr->header_.uuid)); > > + > > + table->array[lr_sful_rec->od->index] = lr_sful_rec; > > + return lr_sful_rec; > > +} > > + > > +static void > > +lr_stateful_record_destroy(struct lr_stateful_record *lr_sful_rec) > > +{ > > + ovn_lb_ip_set_destroy(lr_sful_rec->lb_ips); > > + lr_sful_rec->lb_ips = NULL; > > + sset_destroy(&lr_sful_rec->vip_nats); > > + free(lr_sful_rec); > > +} > > + > > +static void > > +lr_stateful_record_init(struct lr_stateful_record *lr_sful_rec, > > + const struct hmap *lb_datapaths_map, > > + const struct hmap *lbgrp_datapaths_map) > > +{ > > + const struct nbrec_load_balancer_group *nbrec_lb_group; > > + const struct ovn_lb_group_datapaths *lb_group_dps; > > + const struct ovn_lb_datapaths *lb_dps; > > + > > + /* Checking load balancer groups first, starting from the largest one, > > + * to more efficiently copy IP sets. */ > > + size_t largest_group = 0; > > + > > + const struct nbrec_logical_router *nbr = lr_sful_rec->od->nbr; > > + for (size_t i = 1; i < nbr->n_load_balancer_group; i++) { > > + if (nbr->load_balancer_group[i]->n_load_balancer > > > + nbr->load_balancer_group[largest_group]->n_load_balancer) { > > + largest_group = i; > > + } > > + } > > + > > + for (size_t i = 0; i < nbr->n_load_balancer_group; i++) { > > + size_t idx = (i + largest_group) % nbr->n_load_balancer_group; > > + > > + nbrec_lb_group = nbr->load_balancer_group[idx]; > > + const struct uuid *lbgrp_uuid = &nbrec_lb_group->header_.uuid; > > + > > + lb_group_dps = > > + ovn_lb_group_datapaths_find(lbgrp_datapaths_map, > > + lbgrp_uuid); > > + ovs_assert(lb_group_dps); > > + > > + if (!lr_sful_rec->lb_ips) { > > + lr_sful_rec->lb_ips = > > + ovn_lb_ip_set_clone(lb_group_dps->lb_group->lb_ips); > > + } else { > > + for (size_t j = 0; j < lb_group_dps->lb_group->n_lbs; j++) { > > + build_lrouter_lb_ips(lr_sful_rec->lb_ips, > > + lb_group_dps->lb_group->lbs[j]); > > + } > > + } > > + > > + for (size_t j = 0; j < lb_group_dps->lb_group->n_lbs; j++) { > > + build_lrouter_lb_reachable_ips(lr_sful_rec, > > + lb_group_dps->lb_group->lbs[j]); > > + } > > + } > > + > > + if (!lr_sful_rec->lb_ips) { > > + lr_sful_rec->lb_ips = ovn_lb_ip_set_create(); > > + } > > + > > + for (size_t i = 0; i < nbr->n_load_balancer; i++) { > > + const struct uuid *lb_uuid = > > + &nbr->load_balancer[i]->header_.uuid; > > + lb_dps = ovn_lb_datapaths_find(lb_datapaths_map, lb_uuid); > > + ovs_assert(lb_dps); > > + build_lrouter_lb_ips(lr_sful_rec->lb_ips, lb_dps->lb); > > + build_lrouter_lb_reachable_ips(lr_sful_rec, lb_dps->lb); > > + } > > + > > + sset_init(&lr_sful_rec->vip_nats); > > + > > + if (!nbr->n_nat) { > > + lr_stateful_build_vip_nats(lr_sful_rec); > > + } > > +} > > + > > +static struct lr_stateful_input > > +lr_stateful_get_input_data(struct engine_node *node) > > +{ > > + struct northd_data *northd_data = engine_get_input_data("northd", node); > > + struct ed_type_lr_nat_data *lr_nat_data = > > + engine_get_input_data("lr_nat", node); > > + > > + return (struct lr_stateful_input) { > > + .lr_datapaths = &northd_data->lr_datapaths, > > + .lb_datapaths_map = &northd_data->lb_datapaths_map, > > + .lbgrp_datapaths_map = &northd_data->lb_group_datapaths_map, > > + .lr_nats = &lr_nat_data->lr_nats, > > + }; > > +} > > + > > +static void > > +build_lrouter_lb_reachable_ips(struct lr_stateful_record *lr_sful_rec, > > + const struct ovn_northd_lb *lb) > > +{ > > + add_neigh_ips_to_lrouter(lr_sful_rec, lb->neigh_mode, &lb->ips_v4, > > + &lb->ips_v6); > > +} > > + > > +static void > > +add_neigh_ips_to_lrouter(struct lr_stateful_record *lr_sful_rec, > > + enum lb_neighbor_responder_mode neigh_mode, > > + const struct sset *lb_ips_v4, > > + const struct sset *lb_ips_v6) > > +{ > > + /* If configured to not reply to any neighbor requests for all VIPs > > + * return early. > > + */ > > + if (neigh_mode == LB_NEIGH_RESPOND_NONE) { > > + return; > > + } > > + > > + const char *ip_address; > > + > > + /* If configured to reply to neighbor requests for all VIPs force them > > + * all to be considered "reachable". > > + */ > > + if (neigh_mode == LB_NEIGH_RESPOND_ALL) { > > + SSET_FOR_EACH (ip_address, lb_ips_v4) { > > + sset_add(&lr_sful_rec->lb_ips->ips_v4_reachable, ip_address); > > + } > > + SSET_FOR_EACH (ip_address, lb_ips_v6) { > > + sset_add(&lr_sful_rec->lb_ips->ips_v6_reachable, ip_address); > > + } > > + > > + return; > > + } > > + > > + /* Otherwise, a VIP is reachable if there's at least one router > > + * subnet that includes it. > > + */ > > + ovs_assert(neigh_mode == LB_NEIGH_RESPOND_REACHABLE); > > + > > + SSET_FOR_EACH (ip_address, lb_ips_v4) { > > + struct ovn_port *op; > > + ovs_be32 vip_ip4; > > + if (ip_parse(ip_address, &vip_ip4)) { > > + HMAP_FOR_EACH (op, dp_node, &lr_sful_rec->od->ports) { > > + if (lrouter_port_ipv4_reachable(op, vip_ip4)) { > > + sset_add(&lr_sful_rec->lb_ips->ips_v4_reachable, > > + ip_address); > > + break; > > + } > > + } > > + } > > + } > > + > > + SSET_FOR_EACH (ip_address, lb_ips_v6) { > > + struct ovn_port *op; > > + struct in6_addr vip; > > + if (ipv6_parse(ip_address, &vip)) { > > + HMAP_FOR_EACH (op, dp_node, &lr_sful_rec->od->ports) { > > + if (lrouter_port_ipv6_reachable(op, &vip)) { > > + sset_add(&lr_sful_rec->lb_ips->ips_v6_reachable, > > + ip_address); > > + break; > > + } > > + } > > + } > > + } > > +} > > + > > +static void > > +remove_lrouter_lb_reachable_ips(struct lr_stateful_record *lr_sful_rec, > > + enum lb_neighbor_responder_mode neigh_mode, > > + const struct sset *lb_ips_v4, > > + const struct sset *lb_ips_v6) > > +{ > > + if (neigh_mode == LB_NEIGH_RESPOND_NONE) { > > + return; > > + } > > + > > + const char *ip_address; > > + SSET_FOR_EACH (ip_address, lb_ips_v4) { > > + sset_find_and_delete(&lr_sful_rec->lb_ips->ips_v4_reachable, > > + ip_address); > > + } > > + SSET_FOR_EACH (ip_address, lb_ips_v6) { > > + sset_find_and_delete(&lr_sful_rec->lb_ips->ips_v6_reachable, > > + ip_address); > > + } > > +} > > + > > +static void > > +lr_stateful_build_vip_nats(struct lr_stateful_record *lr_sful_rec) > > +{ > > + sset_clear(&lr_sful_rec->vip_nats); > > + const char *external_ip; > > + SSET_FOR_EACH (external_ip, &lr_sful_rec->lrnat_rec->external_ips) { > > + bool is_vip_nat = false; > > + if (addr_is_ipv6(external_ip)) { > > + is_vip_nat = sset_contains(&lr_sful_rec->lb_ips->ips_v6, > > + external_ip); > > + } else { > > + is_vip_nat = sset_contains(&lr_sful_rec->lb_ips->ips_v4, > > + external_ip); > > + } > > + > > + if (is_vip_nat) { > > + sset_add(&lr_sful_rec->vip_nats, external_ip); > > + } > > + } > > +} > > diff --git a/northd/en-lr-stateful.h b/northd/en-lr-stateful.h > > new file mode 100644 > > index 0000000000..a72f000d36 > > --- /dev/null > > +++ b/northd/en-lr-stateful.h > > @@ -0,0 +1,104 @@ > > +/* > > + * Copyright (c) 2023, Red Hat, Inc. > > + * > > + * Licensed under the Apache License, Version 2.0 (the "License"); > > + * you may not use this file except in compliance with the License. > > + * You may obtain a copy of the License at: > > + * > > + * http://www.apache.org/licenses/LICENSE-2.0 > > + * > > + * Unless required by applicable law or agreed to in writing, software > > + * distributed under the License is distributed on an "AS IS" BASIS, > > + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. > > + * See the License for the specific language governing permissions and > > + * limitations under the License. > > + */ > > +#ifndef EN_LR_STATEFUL_H > > +#define EN_LR_STATEFUL_H 1 > > + > > +#include <stdint.h> > > + > > +/* OVS includes. */ > > +#include "lib/hmapx.h" > > +#include "openvswitch/hmap.h" > > +#include "sset.h" > > + > > +/* OVN includes. */ > > +#include "lib/inc-proc-eng.h" > > +#include "lib/lb.h" > > +#include "lib/ovn-nb-idl.h" > > +#include "lib/ovn-sb-idl.h" > > +#include "lib/ovn-util.h" > > + > > +struct ovn_datapath; > > +struct lr_nat_record; > > + > > +/* lr_stateful_table: This represents a table of logical routers with > > + * stateful related data. > > + * stateful related data has two main components > > + * - NAT and > > + * - Load balancers. > > + * > > + * lr_stateful_record: It is a record in the lr_stateful_table for each > > + * logical router. > > + */ > > + > > +struct lr_stateful_record { > > + struct hmap_node key_node; /* Index on 'nbr->header_.uuid'. */ > > + > > + const struct ovn_datapath *od; > > + const struct lr_nat_record *lrnat_rec; > > + > > + /* Load Balancer vIPs relevant for this datapath. */ > > + struct ovn_lb_ip_set *lb_ips; > > + > > + /* sset of vips which are also part of lr nats. */ > > + struct sset vip_nats; > > +}; > > + > > +struct lr_stateful_table { > > + struct hmap entries; > > + > > + /* The array index of each element in 'entries'. */ > > + struct lr_stateful_record **array; > > +}; > > + > > +#define LR_STATEFUL_TABLE_FOR_EACH(LR_LB_NAT_REC, TABLE) \ > > + HMAP_FOR_EACH (LR_LB_NAT_REC, key_node, &(TABLE)->entries) > > + > > +struct lr_stateful_tracked_data { > > + /* Created or updated logical router with LB and/or NAT data. */ > > + struct hmapx crupdated; /* Stores 'struct lr_stateful_record'. */ > > +}; > > + > > +struct ed_type_lr_stateful { > > + struct lr_stateful_table lr_sful_table; > > + > > + struct lr_stateful_tracked_data trk_data; > > +}; > > + > > +struct lr_stateful_input { > > + const struct ovn_datapaths *lr_datapaths; > > + const struct hmap *lb_datapaths_map; > > + const struct hmap *lbgrp_datapaths_map; > > + const struct lr_nat_table *lr_nats; > > +}; > > + > > +void *en_lr_stateful_init(struct engine_node *, struct engine_arg *); > > +void en_lr_stateful_cleanup(void *data); > > +void en_lr_stateful_clear_tracked_data(void *data); > > +void en_lr_stateful_run(struct engine_node *, void *data); > > + > > +bool lr_stateful_northd_handler(struct engine_node *, void *data); > > +bool lr_stateful_lr_nat_handler(struct engine_node *, void *data); > > +bool lr_stateful_lb_data_handler(struct engine_node *, void *data); > > + > > +const struct lr_stateful_record *lr_stateful_table_find_by_index( > > + const struct lr_stateful_table *, size_t od_index); > > + > > +static inline bool > > +lr_stateful_has_tracked_data(struct lr_stateful_tracked_data *trk_data) { > > + return !hmapx_is_empty(&trk_data->crupdated); > > +} > > + > > +#endif /* EN_lr_stateful_H */ > > diff --git a/northd/en-sync-sb.c b/northd/en-sync-sb.c > > index 11e12428f7..733fb9024e 100644 > > --- a/northd/en-sync-sb.c > > +++ b/northd/en-sync-sb.c > > @@ -22,6 +22,7 @@ > > #include "openvswitch/util.h" > > > > #include "en-lr-nat.h" > > +#include "en-lr-stateful.h" > > #include "en-sync-sb.h" > > #include "lib/inc-proc-eng.h" > > #include "lib/lb.h" > > @@ -41,7 +42,7 @@ static void sync_addr_sets(struct ovsdb_idl_txn *ovnsb_txn, > > const struct nbrec_address_set_table *, > > const struct nbrec_port_group_table *, > > const struct sbrec_address_set_table *, > > - const struct ovn_datapaths *lr_datapaths); > > + const struct lr_stateful_table *); > > static const struct sbrec_address_set *sb_address_set_lookup_by_name( > > struct ovsdb_idl_index *, const char *name); > > static void update_sb_addr_set(struct sorted_array *, > > @@ -87,11 +88,11 @@ en_sync_to_sb_addr_set_run(struct engine_node *node, void *data OVS_UNUSED) > > EN_OVSDB_GET(engine_get_input("SB_address_set", node)); > > > > const struct engine_context *eng_ctx = engine_get_context(); > > - struct northd_data *northd_data = engine_get_input_data("northd", node); > > - > > + const struct ed_type_lr_stateful *lr_stateful_data = > > + engine_get_input_data("lr_stateful", node); > > sync_addr_sets(eng_ctx->ovnsb_idl_txn, nb_address_set_table, > > nb_port_group_table, sb_address_set_table, > > - &northd_data->lr_datapaths); > > + &lr_stateful_data->lr_sful_table); > > > > engine_set_node_state(node, EN_UPDATED); > > } > > @@ -288,10 +289,12 @@ en_sync_to_sb_pb_run(struct engine_node *node, void *data OVS_UNUSED) > > { > > const struct engine_context *eng_ctx = engine_get_context(); > > struct northd_data *northd_data = engine_get_input_data("northd", node); > > - struct ed_type_lr_nat_data *lr_nat_data = > > - engine_get_input_data("lr_nat", node); > > + struct ed_type_lr_stateful *lr_stateful_data = > > + engine_get_input_data("lr_stateful", node); > > + > > sync_pbs(eng_ctx->ovnsb_idl_txn, &northd_data->ls_ports, > > - &northd_data->lr_ports, &lr_nat_data->lr_nats); > > + &northd_data->lr_ports, > > + &lr_stateful_data->lr_sful_table); > > engine_set_node_state(node, EN_UPDATED); > > } > > > > @@ -316,7 +319,11 @@ sync_to_sb_pb_northd_handler(struct engine_node *node, void *data OVS_UNUSED) > > return false; > > } > > > > - if (!sync_pbs_for_northd_changed_ovn_ports(&nd->trk_data.trk_lsps)) { > > + struct ed_type_lr_stateful *lr_sful_data = > > + engine_get_input_data("lr_stateful", node); > > + > > + if (!sync_pbs_for_northd_changed_ovn_ports(&nd->trk_data.trk_lsps, > > + &lr_sful_data->lr_sful_table)) { > > return false; > > } > > > > @@ -362,7 +369,7 @@ sync_addr_sets(struct ovsdb_idl_txn *ovnsb_txn, > > const struct nbrec_address_set_table *nb_address_set_table, > > const struct nbrec_port_group_table *nb_port_group_table, > > const struct sbrec_address_set_table *sb_address_set_table, > > - const struct ovn_datapaths *lr_datapaths) > > + const struct lr_stateful_table *lr_statefuls) > > { > > struct shash sb_address_sets = SHASH_INITIALIZER(&sb_address_sets); > > > > @@ -406,16 +413,14 @@ sync_addr_sets(struct ovsdb_idl_txn *ovnsb_txn, > > } > > > > /* Sync router load balancer VIP generated address sets. */ > > - struct ovn_datapath *od; > > - HMAP_FOR_EACH (od, key_node, &lr_datapaths->datapaths) { > > - ovs_assert(od->nbr); > > - > > - if (sset_count(&od->lb_ips->ips_v4_reachable)) { > > - char *ipv4_addrs_name = lr_lb_address_set_name(od->tunnel_key, > > - AF_INET); > > + const struct lr_stateful_record *lr_sful_rec; > > + LR_STATEFUL_TABLE_FOR_EACH (lr_sful_rec, lr_statefuls) { > > + if (sset_count(&lr_sful_rec->lb_ips->ips_v4_reachable)) { > > + char *ipv4_addrs_name = > > + lr_lb_address_set_name(lr_sful_rec->od->tunnel_key, AF_INET); > > > > struct sorted_array ipv4_addrs_sorted = > > - sorted_array_from_sset(&od->lb_ips->ips_v4_reachable); > > + sorted_array_from_sset(&lr_sful_rec->lb_ips->ips_v4_reachable); > > > > sync_addr_set(ovnsb_txn, ipv4_addrs_name, > > &ipv4_addrs_sorted, &sb_address_sets); > > @@ -423,11 +428,11 @@ sync_addr_sets(struct ovsdb_idl_txn *ovnsb_txn, > > free(ipv4_addrs_name); > > } > > > > - if (sset_count(&od->lb_ips->ips_v6_reachable)) { > > - char *ipv6_addrs_name = lr_lb_address_set_name(od->tunnel_key, > > - AF_INET6); > > - struct sorted_array ipv6_addrs_sorted = > > - sorted_array_from_sset(&od->lb_ips->ips_v6_reachable); > > + if (sset_count(&lr_sful_rec->lb_ips->ips_v6_reachable)) { > > + char *ipv6_addrs_name = > > + lr_lb_address_set_name(lr_sful_rec->od->tunnel_key, AF_INET6); > > + struct sorted_array ipv6_addrs_sorted = sorted_array_from_sset( > > + &lr_sful_rec->lb_ips->ips_v6_reachable); > > > > sync_addr_set(ovnsb_txn, ipv6_addrs_name, > > &ipv6_addrs_sorted, &sb_address_sets); > > diff --git a/northd/inc-proc-northd.c b/northd/inc-proc-northd.c > > index 1f211b278e..97bcce9655 100644 > > --- a/northd/inc-proc-northd.c > > +++ b/northd/inc-proc-northd.c > > @@ -31,6 +31,7 @@ > > #include "openvswitch/vlog.h" > > #include "inc-proc-northd.h" > > #include "en-lb-data.h" > > +#include "en-lr-stateful.h" > > #include "en-lr-nat.h" > > #include "en-northd.h" > > #include "en-lflow.h" > > @@ -148,6 +149,7 @@ static ENGINE_NODE(sync_to_sb_lb, "sync_to_sb_lb"); > > static ENGINE_NODE(sync_to_sb_pb, "sync_to_sb_pb"); > > static ENGINE_NODE_WITH_CLEAR_TRACK_DATA(lb_data, "lb_data"); > > static ENGINE_NODE_WITH_CLEAR_TRACK_DATA(lr_nat, "lr_nat"); > > +static ENGINE_NODE_WITH_CLEAR_TRACK_DATA(lr_stateful, "lr_stateful"); > > > > void inc_proc_northd_init(struct ovsdb_idl_loop *nb, > > struct ovsdb_idl_loop *sb) > > @@ -193,6 +195,11 @@ void inc_proc_northd_init(struct ovsdb_idl_loop *nb, > > > > engine_add_input(&en_lr_nat, &en_northd, lr_nat_northd_handler); > > > > + engine_add_input(&en_lr_stateful, &en_northd, lr_stateful_northd_handler); > > + engine_add_input(&en_lr_stateful, &en_lr_nat, lr_stateful_lr_nat_handler); > > + engine_add_input(&en_lr_stateful, &en_lb_data, > > + lr_stateful_lb_data_handler); > > + > > engine_add_input(&en_mac_binding_aging, &en_nb_nb_global, NULL); > > engine_add_input(&en_mac_binding_aging, &en_sb_mac_binding, NULL); > > engine_add_input(&en_mac_binding_aging, &en_northd, NULL); > > @@ -214,15 +221,17 @@ void inc_proc_northd_init(struct ovsdb_idl_loop *nb, > > engine_add_input(&en_lflow, &en_sb_logical_flow, NULL); > > engine_add_input(&en_lflow, &en_sb_multicast_group, NULL); > > engine_add_input(&en_lflow, &en_sb_igmp_group, NULL); > > + engine_add_input(&en_lflow, &en_lr_nat, NULL); > > + engine_add_input(&en_lflow, &en_lr_stateful, NULL); > > engine_add_input(&en_lflow, &en_northd, lflow_northd_handler); > > engine_add_input(&en_lflow, &en_port_group, lflow_port_group_handler); > > - engine_add_input(&en_lflow, &en_lr_nat, NULL); > > > > engine_add_input(&en_sync_to_sb_addr_set, &en_nb_address_set, > > sync_to_sb_addr_set_nb_address_set_handler); > > engine_add_input(&en_sync_to_sb_addr_set, &en_nb_port_group, > > sync_to_sb_addr_set_nb_port_group_handler); > > engine_add_input(&en_sync_to_sb_addr_set, &en_northd, NULL); > > + engine_add_input(&en_sync_to_sb_addr_set, &en_lr_stateful, NULL); > > engine_add_input(&en_sync_to_sb_addr_set, &en_sb_address_set, NULL); > > > > engine_add_input(&en_port_group, &en_nb_port_group, > > @@ -240,7 +249,7 @@ void inc_proc_northd_init(struct ovsdb_idl_loop *nb, > > > > engine_add_input(&en_sync_to_sb_pb, &en_northd, > > sync_to_sb_pb_northd_handler); > > - engine_add_input(&en_sync_to_sb_pb, &en_lr_nat, NULL); > > + engine_add_input(&en_sync_to_sb_pb, &en_lr_stateful, NULL); > > > > /* en_sync_to_sb engine node syncs the SB database tables from > > * the NB database tables. > > diff --git a/northd/northd.c b/northd/northd.c > > index 7f59415bbd..e9b8332e21 100644 > > --- a/northd/northd.c > > +++ b/northd/northd.c > > @@ -44,6 +44,7 @@ > > #include "northd.h" > > #include "en-lb-data.h" > > #include "en-lr-nat.h" > > +#include "en-lr-stateful.h" > > #include "lib/ovn-parallel-hmap.h" > > #include "ovn/actions.h" > > #include "ovn/features.h" > > @@ -617,13 +618,6 @@ init_lb_for_datapath(struct ovn_datapath *od) > > } > > } > > > > -static void > > -destroy_lb_for_datapath(struct ovn_datapath *od) > > -{ > > - ovn_lb_ip_set_destroy(od->lb_ips); > > - od->lb_ips = NULL; > > -} > > - > > /* A group of logical router datapaths which are connected - either > > * directly or indirectly. > > * Each logical router can belong to only one group. */ > > @@ -676,7 +670,6 @@ ovn_datapath_destroy(struct hmap *datapaths, struct ovn_datapath *od) > > destroy_ipam_info(&od->ipam_info); > > free(od->router_ports); > > free(od->ls_peers); > > - destroy_lb_for_datapath(od); > > free(od->localnet_ports); > > free(od->l3dgw_ports); > > destroy_mcast_info_for_datapath(od); > > @@ -1311,121 +1304,6 @@ struct lflow_ref_node { > > struct ovn_lflow *lflow; > > }; > > > > -/* A logical switch port or logical router port. > > - * > > - * In steady state, an ovn_port points to a northbound Logical_Switch_Port > > - * record (via 'nbsp') *or* a Logical_Router_Port record (via 'nbrp'), and to a > > - * southbound Port_Binding record (via 'sb'). As the state of the system > > - * changes, join_logical_ports() may determine that there is a new LSP or LRP > > - * that has no corresponding Port_Binding record (in which case build_ports()) > > - * will create the missing Port_Binding) or that a Port_Binding record exists > > - * that has no coresponding LSP (in which case build_ports() will delete the > > - * spurious Port_Binding). Thus, after build_ports() runs, any given ovn_port > > - * will have 'sb' nonnull, and 'nbsp' xor 'nbrp' nonnull. > > - * > > - * Ordinarily there is only one ovn_port that points to a given LSP or LRP (but > > - * distributed gateway ports point a "derived" ovn_port to a duplicate LRP). > > - */ > > -struct ovn_port { > > - /* Port name aka key. > > - * > > - * This is ordinarily the same as nbsp->name or nbrp->name and > > - * sb->logical_port. (A distributed gateway port creates a "derived" > > - * ovn_port with key "cr-%s" % nbrp->name.) */ > > - struct hmap_node key_node; /* Index on 'key'. */ > > - char *key; /* nbsp->name, nbrp->name, sb->logical_port. */ > > - char *json_key; /* 'key', quoted for use in JSON. */ > > - > > - const struct sbrec_port_binding *sb; /* May be NULL. */ > > - > > - uint32_t tunnel_key; > > - > > - /* Logical switch port data. */ > > - const struct nbrec_logical_switch_port *nbsp; /* May be NULL. */ > > - > > - struct lport_addresses *lsp_addrs; /* Logical switch port addresses. */ > > - unsigned int n_lsp_addrs; /* Total length of lsp_addrs. */ > > - unsigned int n_lsp_non_router_addrs; /* Number of elements from the > > - * beginning of 'lsp_addrs' extracted > > - * directly from LSP 'addresses'. */ > > - > > - struct lport_addresses *ps_addrs; /* Port security addresses. */ > > - unsigned int n_ps_addrs; > > - > > - bool lsp_can_be_inc_processed; /* If it can be incrementally processed when > > - the port changes. */ > > - > > - /* Logical router port data. */ > > - const struct nbrec_logical_router_port *nbrp; /* May be NULL. */ > > - > > - struct lport_addresses lrp_networks; > > - > > - struct ovn_port_routable_addresses routables; > > - > > - /* Logical port multicast data. */ > > - struct mcast_port_info mcast_info; > > - > > - /* At most one of l3dgw_port and cr_port can be not NULL. */ > > - > > - /* This is set to a distributed gateway port if and only if this ovn_port > > - * is "derived" from it. Otherwise this is set to NULL. The derived > > - * ovn_port represents the instance of distributed gateway port on the > > - * gateway chassis.*/ > > - struct ovn_port *l3dgw_port; > > - > > - /* This is set to the "derived" chassis-redirect port of this port if and > > - * only if this port is a distributed gateway port. Otherwise this is set > > - * to NULL. */ > > - struct ovn_port *cr_port; > > - > > - bool has_unknown; /* If the addresses have 'unknown' defined. */ > > - > > - bool has_bfd; > > - > > - /* The port's peer: > > - * > > - * - A switch port S of type "router" has a router port R as a peer, > > - * and R in turn has S has its peer. > > - * > > - * - Two connected logical router ports have each other as peer. > > - * > > - * - Other kinds of ports have no peer. */ > > - struct ovn_port *peer; > > - > > - struct ovn_datapath *od; > > - > > - struct ovs_list list; /* In list of similar records. */ > > - > > - struct hmap_node dp_node; /* Node in od->ports. */ > > - > > - struct lport_addresses proxy_arp_addrs; > > - > > - /* Temporarily used for traversing a list (or hmap) of ports. */ > > - bool visited; > > - > > - /* List of struct lflow_ref_node that points to the lflows generated by > > - * this ovn_port. > > - * > > - * This data is initialized and destroyed by the en_northd node, but > > - * populated and used only by the en_lflow node. Ideally this data should > > - * be maintained as part of en_lflow's data (struct lflow_data): a hash > > - * index from ovn_port key to lflows. However, it would be less efficient > > - * and more complex: > > - * > > - * 1. It would require an extra search (using the index) to find the > > - * lflows. > > - * > > - * 2. Building the index needs to be thread-safe, using either a global > > - * lock which is obviously less efficient, or hash-based lock array which > > - * is more complex. > > - * > > - * Adding the list here is more straightforward. The drawback is that we > > - * need to keep in mind that this data belongs to en_lflow node, so never > > - * access it from any other nodes. > > - */ > > - struct ovs_list lflows; > > -}; > > - > > static bool lsp_can_be_inc_processed(const struct nbrec_logical_switch_port *); > > > > static bool > > @@ -1450,16 +1328,21 @@ destroy_routable_addresses(struct ovn_port_routable_addresses *ra) > > } > > > > static char **get_nat_addresses(const struct ovn_port *op, size_t *n, > > - bool routable_only, bool include_lb_ips); > > + bool routable_only, bool include_lb_ips, > > + const struct lr_stateful_record *); > > > > -static void > > -assign_routable_addresses(struct ovn_port *op) > > +static struct ovn_port_routable_addresses > > +get_op_routable_addresses(struct ovn_port *op, > > + const struct lr_stateful_record *lr_sful_rec) > > { > > size_t n; > > - char **nats = get_nat_addresses(op, &n, true, true); > > + char **nats = get_nat_addresses(op, &n, true, true, lr_sful_rec); > > > > if (!nats) { > > - return; > > + return (struct ovn_port_routable_addresses) { > > + .laddrs = NULL, > > + .n_addrs = 0, > > + }; > > } > > > > struct lport_addresses *laddrs = xcalloc(n, sizeof(*laddrs)); > > @@ -1475,9 +1358,15 @@ assign_routable_addresses(struct ovn_port *op) > > } > > free(nats); > > > > - /* Everything seems to have worked out */ > > - op->routables.laddrs = laddrs; > > - op->routables.n_addrs = n_addrs; > > + if (!n_addrs) { > > + free(laddrs); > > + laddrs = NULL; > > + } > > + > > + return (struct ovn_port_routable_addresses) { > > + .laddrs = laddrs, > > + .n_addrs = n_addrs, > > + }; > > } > > > > > > @@ -1537,8 +1426,6 @@ ovn_port_destroy_orphan(struct ovn_port *port) > > } > > free(port->ps_addrs); > > > > - destroy_routable_addresses(&port->routables); > > - > > destroy_lport_addresses(&port->lrp_networks); > > destroy_lport_addresses(&port->proxy_arp_addrs); > > free(port->json_key); > > @@ -2580,9 +2467,7 @@ join_logical_ports(const struct sbrec_port_binding_table *sbrec_pb_table, > > sizeof *od->l3dgw_ports); > > } > > od->l3dgw_ports[od->n_l3dgw_ports++] = op; > > - > > - assign_routable_addresses(op); > > - } > > + } > > } > > } > > > > @@ -2679,7 +2564,8 @@ join_logical_ports(const struct sbrec_port_binding_table *sbrec_pb_table, > > * and must free the returned array when it is no longer needed. */ > > static char ** > > get_nat_addresses(const struct ovn_port *op, size_t *n, bool routable_only, > > - bool include_lb_ips) > > + bool include_lb_ips, > > + const struct lr_stateful_record *lr_sful_rec) > > { > > size_t n_nats = 0; > > struct eth_addr mac; > > @@ -2764,23 +2650,25 @@ get_nat_addresses(const struct ovn_port *op, size_t *n, bool routable_only, > > } > > } > > > > - if (include_lb_ips) { > > + if (include_lb_ips && lr_sful_rec) { > > const char *ip_address; > > if (routable_only) { > > - SSET_FOR_EACH (ip_address, &op->od->lb_ips->ips_v4_routable) { > > + SSET_FOR_EACH (ip_address, > > + &lr_sful_rec->lb_ips->ips_v4_routable) { > > ds_put_format(&c_addresses, " %s", ip_address); > > central_ip_address = true; > > } > > - SSET_FOR_EACH (ip_address, &op->od->lb_ips->ips_v6_routable) { > > + SSET_FOR_EACH (ip_address, > > + &lr_sful_rec->lb_ips->ips_v6_routable) { > > ds_put_format(&c_addresses, " %s", ip_address); > > central_ip_address = true; > > } > > } else { > > - SSET_FOR_EACH (ip_address, &op->od->lb_ips->ips_v4) { > > + SSET_FOR_EACH (ip_address, &lr_sful_rec->lb_ips->ips_v4) { > > ds_put_format(&c_addresses, " %s", ip_address); > > central_ip_address = true; > > } > > - SSET_FOR_EACH (ip_address, &op->od->lb_ips->ips_v6) { > > + SSET_FOR_EACH (ip_address, &lr_sful_rec->lb_ips->ips_v6) { > > ds_put_format(&c_addresses, " %s", ip_address); > > central_ip_address = true; > > } > > @@ -3851,21 +3739,8 @@ build_lb_datapaths(const struct hmap *lbs, const struct hmap *lb_groups, > > HMAP_FOR_EACH (od, key_node, &lr_datapaths->datapaths) { > > ovs_assert(od->nbr); > > > > - /* Checking load balancer groups first, starting from the largest one, > > - * to more efficiently copy IP sets. */ > > - size_t largest_group = 0; > > - > > - for (size_t i = 1; i < od->nbr->n_load_balancer_group; i++) { > > - if (od->nbr->load_balancer_group[i]->n_load_balancer > > > - od->nbr->load_balancer_group[largest_group]->n_load_balancer) { > > - largest_group = i; > > - } > > - } > > - > > for (size_t i = 0; i < od->nbr->n_load_balancer_group; i++) { > > - size_t idx = (i + largest_group) % od->nbr->n_load_balancer_group; > > - > > - nbrec_lb_group = od->nbr->load_balancer_group[idx]; > > + nbrec_lb_group = od->nbr->load_balancer_group[i]; > > const struct uuid *lb_group_uuid = &nbrec_lb_group->header_.uuid; > > > > lb_group_dps = > > @@ -3873,20 +3748,6 @@ build_lb_datapaths(const struct hmap *lbs, const struct hmap *lb_groups, > > lb_group_uuid); > > ovs_assert(lb_group_dps); > > ovn_lb_group_datapaths_add_lr(lb_group_dps, od); > > - > > - if (!od->lb_ips) { > > - od->lb_ips = > > - ovn_lb_ip_set_clone(lb_group_dps->lb_group->lb_ips); > > - } else { > > - for (size_t j = 0; j < lb_group_dps->lb_group->n_lbs; j++) { > > - build_lrouter_lb_ips(od->lb_ips, > > - lb_group_dps->lb_group->lbs[j]); > > - } > > - } > > - } > > - > > - if (!od->lb_ips) { > > - od->lb_ips = ovn_lb_ip_set_create(); > > } > > > > for (size_t i = 0; i < od->nbr->n_load_balancer; i++) { > > @@ -3895,7 +3756,6 @@ build_lb_datapaths(const struct hmap *lbs, const struct hmap *lb_groups, > > lb_dps = ovn_lb_datapaths_find(lb_datapaths_map, lb_uuid); > > ovs_assert(lb_dps); > > ovn_lb_datapaths_add_lr(lb_dps, 1, &od); > > - build_lrouter_lb_ips(od->lb_ips, lb_dps->lb); > > } > > } > > > > @@ -3949,102 +3809,6 @@ build_lb_svcs( > > } > > } > > > > -static bool lrouter_port_ipv4_reachable(const struct ovn_port *op, > > - ovs_be32 addr); > > -static bool lrouter_port_ipv6_reachable(const struct ovn_port *op, > > - const struct in6_addr *addr); > > - > > -static void > > -add_neigh_ips_to_lrouter(struct ovn_datapath *od, > > - enum lb_neighbor_responder_mode neigh_mode, > > - const struct sset *lb_ips_v4, > > - const struct sset *lb_ips_v6) > > -{ > > - /* If configured to not reply to any neighbor requests for all VIPs > > - * return early. > > - */ > > - if (neigh_mode == LB_NEIGH_RESPOND_NONE) { > > - return; > > - } > > - > > - const char *ip_address; > > - > > - /* If configured to reply to neighbor requests for all VIPs force them > > - * all to be considered "reachable". > > - */ > > - if (neigh_mode == LB_NEIGH_RESPOND_ALL) { > > - SSET_FOR_EACH (ip_address, lb_ips_v4) { > > - sset_add(&od->lb_ips->ips_v4_reachable, ip_address); > > - } > > - SSET_FOR_EACH (ip_address, lb_ips_v6) { > > - sset_add(&od->lb_ips->ips_v6_reachable, ip_address); > > - } > > - > > - return; > > - } > > - > > - /* Otherwise, a VIP is reachable if there's at least one router > > - * subnet that includes it. > > - */ > > - ovs_assert(neigh_mode == LB_NEIGH_RESPOND_REACHABLE); > > - > > - SSET_FOR_EACH (ip_address, lb_ips_v4) { > > - struct ovn_port *op; > > - ovs_be32 vip_ip4; > > - if (ip_parse(ip_address, &vip_ip4)) { > > - HMAP_FOR_EACH (op, dp_node, &od->ports) { > > - if (lrouter_port_ipv4_reachable(op, vip_ip4)) { > > - sset_add(&od->lb_ips->ips_v4_reachable, > > - ip_address); > > - break; > > - } > > - } > > - } > > - } > > - > > - SSET_FOR_EACH (ip_address, lb_ips_v6) { > > - struct ovn_port *op; > > - struct in6_addr vip; > > - if (ipv6_parse(ip_address, &vip)) { > > - HMAP_FOR_EACH (op, dp_node, &od->ports) { > > - if (lrouter_port_ipv6_reachable(op, &vip)) { > > - sset_add(&od->lb_ips->ips_v6_reachable, > > - ip_address); > > - break; > > - } > > - } > > - } > > - } > > -} > > - > > -static void > > -remove_lrouter_lb_reachable_ips(struct ovn_datapath *od, > > - enum lb_neighbor_responder_mode neigh_mode, > > - const struct sset *lb_ips_v4, > > - const struct sset *lb_ips_v6) > > -{ > > - if (neigh_mode == LB_NEIGH_RESPOND_NONE) { > > - return; > > - } > > - > > - const char *ip_address; > > - SSET_FOR_EACH (ip_address, lb_ips_v4) { > > - sset_find_and_delete(&od->lb_ips->ips_v4_reachable, ip_address); > > - } > > - SSET_FOR_EACH (ip_address, lb_ips_v6) { > > - sset_find_and_delete(&od->lb_ips->ips_v6_reachable, ip_address); > > - } > > -} > > - > > -static void > > -build_lrouter_lb_reachable_ips(struct ovn_datapath *od, > > - const struct ovn_northd_lb *lb) > > -{ > > - add_neigh_ips_to_lrouter(od, lb->neigh_mode, &lb->ips_v4, > > - &lb->ips_v6); > > -} > > - > > - > > static void > > build_lrouter_lbs_check(const struct ovn_datapaths *lr_datapaths) > > { > > @@ -4066,43 +3830,6 @@ build_lrouter_lbs_check(const struct ovn_datapaths *lr_datapaths) > > } > > } > > > > -static void > > -build_lrouter_lbs_reachable_ips(struct ovn_datapaths *lr_datapaths, > > - struct hmap *lb_dps_map, > > - struct hmap *lb_group_dps_map) > > -{ > > - struct ovn_datapath *od; > > - > > - HMAP_FOR_EACH (od, key_node, &lr_datapaths->datapaths) { > > - if (!od->nbr) { > > - continue; > > - } > > - > > - for (size_t i = 0; i < od->nbr->n_load_balancer; i++) { > > - struct ovn_lb_datapaths *lb_dps = > > - ovn_lb_datapaths_find(lb_dps_map, > > - &od->nbr->load_balancer[i]->header_.uuid); > > - ovs_assert(lb_dps); > > - build_lrouter_lb_reachable_ips(od, lb_dps->lb); > > - } > > - > > - for (size_t i = 0; i < od->nbr->n_load_balancer_group; i++) { > > - const struct nbrec_load_balancer_group *nbrec_lb_group = > > - od->nbr->load_balancer_group[i]; > > - struct ovn_lb_group_datapaths *lb_group_dps; > > - > > - lb_group_dps = > > - ovn_lb_group_datapaths_find(lb_group_dps_map, > > - &nbrec_lb_group->header_.uuid); > > - ovs_assert(lb_group_dps); > > - for (size_t j = 0; j < lb_group_dps->lb_group->n_lbs; j++) { > > - build_lrouter_lb_reachable_ips(od, > > - lb_group_dps->lb_group->lbs[j]); > > - } > > - } > > - } > > -} > > - > > static void > > build_lswitch_lbs_from_lrouter(struct ovn_datapaths *lr_datapaths, > > struct hmap *lb_dps_map, > > @@ -4166,8 +3893,6 @@ build_lb_port_related_data( > > struct hmap *svc_monitor_map) > > { > > build_lrouter_lbs_check(lr_datapaths); > > - build_lrouter_lbs_reachable_ips(lr_datapaths, lb_dps_map, > > - lb_group_dps_map); > > build_lb_svcs(ovnsb_txn, sbrec_service_monitor_table, ls_ports, lb_dps_map, > > svc_monitor_lsps, svc_monitor_map); > > build_lswitch_lbs_from_lrouter(lr_datapaths, lb_dps_map, lb_group_dps_map); > > @@ -4533,7 +4258,8 @@ check_sb_lb_duplicates(const struct sbrec_load_balancer_table *table) > > * Caller should make sure that the OVN SB IDL txn is not NULL. Presently it > > * only syncs the nat column of port binding corresponding to the 'op->nbsp' */ > > static void > > -sync_pb_for_lsp(struct ovn_port *op) > > +sync_pb_for_lsp(struct ovn_port *op, > > + const struct lr_stateful_table *lr_sful_table) > > { > > ovs_assert(op->nbsp); > > > > @@ -4552,10 +4278,17 @@ sync_pb_for_lsp(struct ovn_port *op) > > if (nat_addresses && !strcmp(nat_addresses, "router")) { > > if (op->peer && op->peer->od > > && (chassis || op->peer->od->n_l3dgw_ports)) { > > - bool exclude_lb_vips = smap_get_bool(&op->nbsp->options, > > + bool include_lb_vips = !smap_get_bool(&op->nbsp->options, > > "exclude-lb-vips-from-garp", false); > > + > > + const struct lr_stateful_record *lr_sful_rec = NULL; > > + > > + if (include_lb_vips) { > > + lr_sful_rec = lr_stateful_table_find_by_index( > > + lr_sful_table, op->peer->od->index); > > + } > > nats = get_nat_addresses(op->peer, &n_nats, false, > > - !exclude_lb_vips); > > + include_lb_vips, lr_sful_rec); > > } > > } else if (nat_addresses && (chassis || l3dgw_ports)) { > > struct lport_addresses laddrs; > > @@ -4662,7 +4395,8 @@ sync_pb_for_lsp(struct ovn_port *op) > > * Caller should make sure that the OVN SB IDL txn is not NULL. Presently it > > * only sets the port binding options column for the router ports */ > > static void > > -sync_pb_for_lrp(struct ovn_port *op, const struct lr_nat_table *lr_nats) > > +sync_pb_for_lrp(struct ovn_port *op, > > + const struct lr_stateful_table *lr_sful_table) > > { > > ovs_assert(op->nbrp); > > > > @@ -4671,14 +4405,14 @@ sync_pb_for_lrp(struct ovn_port *op, const struct lr_nat_table *lr_nats) > > > > const char *chassis_name = smap_get(&op->od->nbr->options, "chassis"); > > if (is_cr_port(op)) { > > - const struct lr_nat_record *lrnat_rec = > > - lr_nat_table_find_by_index(lr_nats, op->od->index); > > - ovs_assert(lrnat_rec); > > + const struct lr_stateful_record *lr_sful_rec = > > + lr_stateful_table_find_by_index(lr_sful_table, op->od->index); > > + ovs_assert(lr_sful_rec); > > > > smap_add(&new, "distributed-port", op->nbrp->name); > > > > bool always_redirect = > > - !lrnat_rec->has_distributed_nat && > > + !lr_sful_rec->lrnat_rec->has_distributed_nat && > > !l3dgw_port_has_associated_vtep_lports(op->l3dgw_port); > > > > const char *redirect_type = smap_get(&op->nbrp->options, > > @@ -4728,17 +4462,18 @@ static void ovn_update_ipv6_opt_for_op(struct ovn_port *op); > > * the logical switch ports. */ > > void > > sync_pbs(struct ovsdb_idl_txn *ovnsb_idl_txn, struct hmap *ls_ports, > > - struct hmap *lr_ports, const struct lr_nat_table *lr_nats) > > + struct hmap *lr_ports, > > + const struct lr_stateful_table *lr_sful_table) > > { > > ovs_assert(ovnsb_idl_txn); > > > > struct ovn_port *op; > > HMAP_FOR_EACH (op, key_node, ls_ports) { > > - sync_pb_for_lsp(op); > > + sync_pb_for_lsp(op, lr_sful_table); > > } > > > > HMAP_FOR_EACH (op, key_node, lr_ports) { > > - sync_pb_for_lrp(op, lr_nats); > > + sync_pb_for_lrp(op, lr_sful_table); > > } > > > > ovn_update_ipv6_options(lr_ports); > > @@ -4747,20 +4482,22 @@ sync_pbs(struct ovsdb_idl_txn *ovnsb_idl_txn, struct hmap *ls_ports, > > /* Sync the SB Port bindings for the added and updated logical switch ports > > * of the tracked northd engine data. */ > > bool > > -sync_pbs_for_northd_changed_ovn_ports(struct tracked_ovn_ports *trk_ovn_ports) > > +sync_pbs_for_northd_changed_ovn_ports( > > + struct tracked_ovn_ports *trk_ovn_ports, > > + const struct lr_stateful_table *lr_sful_table) > > { > > struct hmapx_node *hmapx_node; > > struct ovn_port *op; > > HMAPX_FOR_EACH (hmapx_node, &trk_ovn_ports->created) { > > op = hmapx_node->data; > > ovs_assert(op->nbsp); > > - sync_pb_for_lsp(op); > > + sync_pb_for_lsp(op, lr_sful_table); > > } > > > > HMAPX_FOR_EACH (hmapx_node, &trk_ovn_ports->updated) { > > op = hmapx_node->data; > > ovs_assert(op->nbsp); > > - sync_pb_for_lsp(op); > > + sync_pb_for_lsp(op, lr_sful_table); > > } > > > > return true; > > @@ -5425,14 +5162,13 @@ fail: > > } > > > > /* Returns true if the logical router has changes which can be > > - * incrementally handled. > > + * incrementally handled or the changes can be ignored. > > * Presently supports i-p for the below changes: > > * - load balancers and load balancer groups. > > * - NAT changes > > */ > > static bool > > -lr_changes_can_be_handled( > > - const struct nbrec_logical_router *lr) > > +lr_changes_can_be_handled(const struct nbrec_logical_router *lr) > > { > > /* Check if the columns are changed in this row. */ > > enum nbrec_logical_router_column_id col; > > @@ -5496,17 +5232,6 @@ is_lr_nats_changed(const struct nbrec_logical_router *nbr) { > > || is_lr_nats_seqno_changed(nbr)); > > } > > > > -static bool > > -lr_has_routable_nats(const struct nbrec_logical_router *nbr) { > > - for (size_t i = 0; i < nbr->n_nat; i++) { > > - if (smap_get_bool(&nbr->nat[i]->options, "add_route", false)) { > > - return true; > > - } > > - } > > - > > - return false; > > -} > > - > > /* Return true if changes are handled incrementally, false otherwise. > > * > > * Note: Changes to load balancer and load balancer groups associated with > > @@ -5533,12 +5258,6 @@ northd_handle_lr_changes(const struct northd_input *ni, > > } > > > > if (is_lr_nats_changed(changed_lr)) { > > - if (lr_has_routable_nats(changed_lr)) { > > - /* router has routable NATs. We can't handle these changes > > - * incrementally yet. Fall back to recompute. */ > > - goto fail; > > - } > > - > > struct ovn_datapath *od = ovn_datapath_find_( > > &nd->lr_datapaths.datapaths, > > &changed_lr->header_.uuid); > > @@ -5820,10 +5539,6 @@ northd_handle_lb_data_changes(struct tracked_lb_data *trk_lb_data, > > ovs_assert(lb_dps); > > ovn_lb_datapaths_add_lr(lb_dps, 1, &od); > > > > - /* Add the lb_ips of lb_dps to the od. */ > > - build_lrouter_lb_ips(od->lb_ips, lb_dps->lb); > > - build_lrouter_lb_reachable_ips(od, lb_dps->lb); > > - > > /* Add the lb to the northd tracked data. */ > > hmapx_add(&nd_changes->trk_lbs.crupdated, lb_dps); > > } > > @@ -5842,10 +5557,6 @@ northd_handle_lb_data_changes(struct tracked_lb_data *trk_lb_data, > > ovs_assert(lb_dps); > > ovn_lb_datapaths_add_lr(lb_dps, 1, &od); > > > > - /* Add the lb_ips of lb_dps to the od. */ > > - build_lrouter_lb_ips(od->lb_ips, lb_dps->lb); > > - build_lrouter_lb_reachable_ips(od, lb_dps->lb); > > - > > /* Add the lb to the northd tracked data. */ > > hmapx_add(&nd_changes->trk_lbs.crupdated, lb_dps); > > } > > @@ -5874,22 +5585,6 @@ northd_handle_lb_data_changes(struct tracked_lb_data *trk_lb_data, > > od = lr_datapaths->array[index]; > > /* Re-evaluate 'od->has_lb_vip' */ > > init_lb_for_datapath(od); > > - > > - /* Update the od->lb_ips with the deleted and inserted > > - * vips (if any). */ > > - remove_ips_from_lb_ip_set(od->lb_ips, lb->routable, > > - &clb->deleted_vips_v4, > > - &clb->deleted_vips_v6); > > - add_ips_to_lb_ip_set(od->lb_ips, lb->routable, > > - &clb->inserted_vips_v4, > > - &clb->inserted_vips_v6); > > - > > - remove_lrouter_lb_reachable_ips(od, lb->neigh_mode, > > - &clb->deleted_vips_v4, > > - &clb->deleted_vips_v6); > > - add_neigh_ips_to_lrouter(od, lb->neigh_mode, > > - &clb->inserted_vips_v4, > > - &clb->inserted_vips_v6); > > } > > } > > > > @@ -5914,9 +5609,6 @@ northd_handle_lb_data_changes(struct tracked_lb_data *trk_lb_data, > > > > /* Re-evaluate 'od->has_lb_vip' */ > > init_lb_for_datapath(od); > > - > > - /* Add the lb_ips of lb_dps to the od. */ > > - build_lrouter_lb_ips(od->lb_ips, lb_dps->lb); > > } > > > > for (size_t i = 0; i < lbgrp_dps->n_ls; i++) { > > @@ -9210,7 +8902,7 @@ arp_nd_ns_match(const char *ips, int addr_family, struct ds *match) > > /* Returns 'true' if the IPv4 'addr' is on the same subnet with one of the > > * IPs configured on the router port. > > */ > > -static bool > > +bool > > lrouter_port_ipv4_reachable(const struct ovn_port *op, ovs_be32 addr) > > { > > for (size_t i = 0; i < op->lrp_networks.n_ipv4_addrs; i++) { > > @@ -9226,7 +8918,7 @@ lrouter_port_ipv4_reachable(const struct ovn_port *op, ovs_be32 addr) > > /* Returns 'true' if the IPv6 'addr' is on the same subnet with one of the > > * IPs configured on the router port. > > */ > > -static bool > > +bool > > lrouter_port_ipv6_reachable(const struct ovn_port *op, > > const struct in6_addr *addr) > > { > > @@ -9289,11 +8981,12 @@ build_lswitch_rport_arp_req_flow(const char *ips, > > */ > > static void > > build_lswitch_rport_arp_req_flows(struct ovn_port *op, > > - struct ovn_datapath *sw_od, > > - struct ovn_port *sw_op, > > - const struct lr_nat_table *lr_nats, > > - struct hmap *lflows, > > - const struct ovsdb_idl_row *stage_hint) > > + struct ovn_datapath *sw_od, > > + struct ovn_port *sw_op, > > + const struct lr_nat_table *lr_nats, > > + const struct lr_stateful_table *lr_sful_table, > > + struct hmap *lflows, > > + const struct ovsdb_idl_row *stage_hint) > > { > > if (!op || !op->nbrp) { > > return; > > @@ -9307,32 +9000,38 @@ build_lswitch_rport_arp_req_flows(struct ovn_port *op, > > * router port. > > * Priority: 80. > > */ > > - > > - const char *ip_addr; > > - SSET_FOR_EACH (ip_addr, &op->od->lb_ips->ips_v4_reachable) { > > - ovs_be32 ipv4_addr; > > - > > - /* Check if the ovn port has a network configured on which we could > > - * expect ARP requests for the LB VIP. > > - */ > > - if (ip_parse(ip_addr, &ipv4_addr) && > > - lrouter_port_ipv4_reachable(op, ipv4_addr)) { > > - build_lswitch_rport_arp_req_flow( > > - ip_addr, AF_INET, sw_op, sw_od, 80, lflows, > > - stage_hint); > > + const struct lr_stateful_record *lr_sful_rec = NULL; > > + if (op->od->nbr->n_load_balancer || op->od->nbr->n_load_balancer_group) { > > + lr_sful_rec = lr_stateful_table_find_by_index(lr_sful_table, > > + op->od->index); > > + ovs_assert(lr_sful_rec); > > + > > + const char *ip_addr; > > + SSET_FOR_EACH (ip_addr, &lr_sful_rec->lb_ips->ips_v4_reachable) { > > + ovs_be32 ipv4_addr; > > + > > + /* Check if the ovn port has a network configured on which we could > > + * expect ARP requests for the LB VIP. > > + */ > > + if (ip_parse(ip_addr, &ipv4_addr) && > > + lrouter_port_ipv4_reachable(op, ipv4_addr)) { > > + build_lswitch_rport_arp_req_flow( > > + ip_addr, AF_INET, sw_op, sw_od, 80, lflows, > > + stage_hint); > > + } > > } > > - } > > - SSET_FOR_EACH (ip_addr, &op->od->lb_ips->ips_v6_reachable) { > > - struct in6_addr ipv6_addr; > > + SSET_FOR_EACH (ip_addr, &lr_sful_rec->lb_ips->ips_v6_reachable) { > > + struct in6_addr ipv6_addr; > > > > - /* Check if the ovn port has a network configured on which we could > > - * expect NS requests for the LB VIP. > > - */ > > - if (ipv6_parse(ip_addr, &ipv6_addr) && > > - lrouter_port_ipv6_reachable(op, &ipv6_addr)) { > > - build_lswitch_rport_arp_req_flow( > > - ip_addr, AF_INET6, sw_op, sw_od, 80, lflows, > > - stage_hint); > > + /* Check if the ovn port has a network configured on which we could > > + * expect NS requests for the LB VIP. > > + */ > > + if (ipv6_parse(ip_addr, &ipv6_addr) && > > + lrouter_port_ipv6_reachable(op, &ipv6_addr)) { > > + build_lswitch_rport_arp_req_flow( > > + ip_addr, AF_INET6, sw_op, sw_od, 80, lflows, > > + stage_hint); > > + } > > } > > } > > > > @@ -9382,13 +9081,15 @@ build_lswitch_rport_arp_req_flows(struct ovn_port *op, > > * expect ARP requests/NS for the DNAT external_ip. > > */ > > if (nat_entry_is_v6(nat_entry)) { > > - if (!sset_contains(&op->od->lb_ips->ips_v6, nat->external_ip)) { > > + if (!lr_sful_rec || !sset_contains(&lr_sful_rec->lb_ips->ips_v6, > > + nat->external_ip)) { > > build_lswitch_rport_arp_req_flow( > > nat->external_ip, AF_INET6, sw_op, sw_od, 80, lflows, > > stage_hint); > > } > > } else { > > - if (!sset_contains(&op->od->lb_ips->ips_v4, nat->external_ip)) { > > + if (!lr_sful_rec || !sset_contains(&lr_sful_rec->lb_ips->ips_v4, > > + nat->external_ip)) { > > build_lswitch_rport_arp_req_flow( > > nat->external_ip, AF_INET, sw_op, sw_od, 80, lflows, > > stage_hint); > > @@ -10429,6 +10130,7 @@ build_lswitch_ip_mcast_igmp_mld(struct ovn_igmp_group *igmp_group, > > static void > > build_lswitch_ip_unicast_lookup(struct ovn_port *op, > > const struct lr_nat_table *lr_nats, > > + const struct lr_stateful_table *lr_sful_table, > > struct hmap *lflows, > > struct ds *actions, > > struct ds *match) > > @@ -10444,7 +10146,8 @@ build_lswitch_ip_unicast_lookup(struct ovn_port *op, > > */ > > if (lsp_is_router(op->nbsp)) { > > build_lswitch_rport_arp_req_flows(op->peer, op->od, op, lr_nats, > > - lflows, &op->nbsp->header_); > > + lr_sful_table, lflows, > > + &op->nbsp->header_); > > } > > > > for (size_t i = 0; i < op->nbsp->n_addresses; i++) { > > @@ -12634,6 +12337,7 @@ build_lrouter_port_nat_arp_nd_flow(struct ovn_port *op, > > static void > > build_lrouter_drop_own_dest(struct ovn_port *op, > > const struct lr_nat_record *lrnat_rec, > > + const struct lr_stateful_record *lr_sful_rec, > > enum ovn_stage stage, > > uint16_t priority, bool drop_snat_ip, > > struct hmap *lflows) > > @@ -12646,8 +12350,9 @@ build_lrouter_drop_own_dest(struct ovn_port *op, > > > > bool router_ip_in_snat_ips = !!shash_find(&lrnat_rec->snat_ips, > > ip); > > - bool router_ip_in_lb_ips = > > - !!sset_find(&op->od->lb_ips->ips_v4, ip); > > + bool router_ip_in_lb_ips = (lr_sful_rec && > > + !!sset_find(&lr_sful_rec->lb_ips->ips_v4, > > + ip)); > > bool drop_router_ip = (drop_snat_ip == (router_ip_in_snat_ips || > > router_ip_in_lb_ips)); > > > > @@ -12676,8 +12381,9 @@ build_lrouter_drop_own_dest(struct ovn_port *op, > > > > bool router_ip_in_snat_ips = !!shash_find(&lrnat_rec->snat_ips, > > ip); > > - bool router_ip_in_lb_ips = > > - !!sset_find(&op->od->lb_ips->ips_v6, ip); > > + bool router_ip_in_lb_ips = (lr_sful_rec && > > + !!sset_find(&lr_sful_rec->lb_ips->ips_v6, > > + ip)); > > bool drop_router_ip = (drop_snat_ip == (router_ip_in_snat_ips || > > router_ip_in_lb_ips)); > > > > @@ -13389,7 +13095,8 @@ build_ip_routing_flows_for_lrp( > > */ > > static void > > build_ip_routing_flows_for_router_type_lsp( > > - struct ovn_port *op, const struct hmap *lr_ports, struct hmap *lflows) > > + struct ovn_port *op, const struct lr_stateful_table *lr_sful_table, > > + const struct hmap *lr_ports, struct hmap *lflows) > > { > > ovs_assert(op->nbsp); > > if (!lsp_is_router(op->nbsp)) { > > @@ -13397,7 +13104,8 @@ build_ip_routing_flows_for_router_type_lsp( > > } > > > > struct ovn_port *peer = ovn_port_get_peer(lr_ports, op); > > - if (!peer || !peer->nbrp || !peer->lrp_networks.n_ipv4_addrs) { > > + if (!peer || !peer->nbrp || !peer->lrp_networks.n_ipv4_addrs > > + || !op->od->n_router_ports) { > > return; > > } > > > > @@ -13408,19 +13116,29 @@ build_ip_routing_flows_for_router_type_lsp( > > continue; > > } > > > > - struct ovn_port_routable_addresses *ra = &router_port->routables; > > - for (size_t j = 0; j < ra->n_addrs; j++) { > > - struct lport_addresses *laddrs = &ra->laddrs[j]; > > - for (size_t k = 0; k < laddrs->n_ipv4_addrs; k++) { > > - add_route(lflows, peer->od, peer, > > - peer->lrp_networks.ipv4_addrs[0].addr_s, > > - laddrs->ipv4_addrs[k].network_s, > > - laddrs->ipv4_addrs[k].plen, NULL, false, 0, > > - &peer->nbrp->header_, false, > > - ROUTE_PRIO_OFFSET_CONNECTED); > > + const struct lr_stateful_record *lr_sful_rec = > > + lr_stateful_table_find_by_index(lr_sful_table, > > + router_port->od->index); > > + > > + if (router_port->nbrp->ha_chassis_group || > > + router_port->nbrp->n_gateway_chassis) { > > + struct ovn_port_routable_addresses ra = > > + get_op_routable_addresses(router_port, lr_sful_rec); > > + for (size_t j = 0; j < ra.n_addrs; j++) { > > + struct lport_addresses *laddrs = &ra.laddrs[j]; > > + for (size_t k = 0; k < laddrs->n_ipv4_addrs; k++) { > > + add_route(lflows, peer->od, peer, > > + peer->lrp_networks.ipv4_addrs[0].addr_s, > > + laddrs->ipv4_addrs[k].network_s, > > + laddrs->ipv4_addrs[k].plen, NULL, false, 0, > > + &peer->nbrp->header_, false, > > + ROUTE_PRIO_OFFSET_CONNECTED); > > + } > > } > > + destroy_routable_addresses(&ra); > > } > > } > > + > > } > > > > static void > > @@ -13644,33 +13362,36 @@ build_arp_resolve_flows_for_lrouter( > > > > static void > > routable_addresses_to_lflows(struct hmap *lflows, struct ovn_port *router_port, > > - struct ovn_port *peer, struct ds *match, > > - struct ds *actions) > > + struct ovn_port *peer, > > + const struct lr_stateful_record *lr_sful_rec, > > + struct ds *match, struct ds *actions) > > { > > - struct ovn_port_routable_addresses *ra = &router_port->routables; > > - if (!ra->n_addrs) { > > + struct ovn_port_routable_addresses ra = > > + get_op_routable_addresses(router_port, lr_sful_rec); > > + if (!ra.n_addrs) { > > return; > > } > > > > - for (size_t i = 0; i < ra->n_addrs; i++) { > > + for (size_t i = 0; i < ra.n_addrs; i++) { > > ds_clear(match); > > ds_put_format(match, "outport == %s && "REG_NEXT_HOP_IPV4" == {", > > peer->json_key); > > bool first = true; > > - for (size_t j = 0; j < ra->laddrs[i].n_ipv4_addrs; j++) { > > + for (size_t j = 0; j < ra.laddrs[i].n_ipv4_addrs; j++) { > > if (!first) { > > ds_put_cstr(match, ", "); > > } > > - ds_put_cstr(match, ra->laddrs[i].ipv4_addrs[j].addr_s); > > + ds_put_cstr(match, ra.laddrs[i].ipv4_addrs[j].addr_s); > > first = false; > > } > > ds_put_cstr(match, "}"); > > > > ds_clear(actions); > > - ds_put_format(actions, "eth.dst = %s; next;", ra->laddrs[i].ea_s); > > + ds_put_format(actions, "eth.dst = %s; next;", ra.laddrs[i].ea_s); > > ovn_lflow_add(lflows, peer->od, S_ROUTER_IN_ARP_RESOLVE, 100, > > ds_cstr(match), ds_cstr(actions)); > > } > > + destroy_routable_addresses(&ra); > > } > > > > /* Local router ingress table ARP_RESOLVE: ARP Resolution. > > @@ -13687,6 +13408,7 @@ routable_addresses_to_lflows(struct hmap *lflows, struct ovn_port *router_port, > > static void > > build_arp_resolve_flows_for_lrp( > > struct ovn_port *op, const struct lr_nat_record *lrnat_rec, > > + const struct lr_stateful_record *lr_sful_rec, > > struct hmap *lflows, struct ds *match, struct ds *actions) > > { > > ovs_assert(op->nbrp); > > @@ -13763,8 +13485,8 @@ build_arp_resolve_flows_for_lrp( > > * > > * Priority 2. > > */ > > - build_lrouter_drop_own_dest(op, lrnat_rec, S_ROUTER_IN_ARP_RESOLVE, 2, > > - true, lflows); > > + build_lrouter_drop_own_dest(op, lrnat_rec, lr_sful_rec, > > + S_ROUTER_IN_ARP_RESOLVE, 2, true, lflows); > > } > > > > /* This function adds ARP resolve flows related to a LSP. */ > > @@ -13772,6 +13494,7 @@ static void > > build_arp_resolve_flows_for_lsp( > > struct ovn_port *op, struct hmap *lflows, > > const struct hmap *lr_ports, > > + const struct lr_stateful_table *lr_sful_table, > > struct ds *match, struct ds *actions) > > { > > ovs_assert(op->nbsp); > > @@ -13915,8 +13638,11 @@ build_arp_resolve_flows_for_lsp( > > > > if (smap_get(&peer->od->nbr->options, "chassis") > > || peer->cr_port) { > > + const struct lr_stateful_record *lr_sful_rec; > > + lr_sful_rec = lr_stateful_table_find_by_index(lr_sful_table, > > + router_port->od->index); > > routable_addresses_to_lflows(lflows, router_port, peer, > > - match, actions); > > + lr_sful_rec, match, actions); > > } > > } > > } > > @@ -14636,6 +14362,7 @@ static void > > build_lrouter_ipv4_ip_input(struct ovn_port *op, > > struct hmap *lflows, > > const struct lr_nat_record *lrnat_rec, > > + const struct lr_stateful_record *lr_sful_rec, > > struct ds *match, struct ds *actions, > > const struct shash *meter_groups) > > { > > @@ -14760,7 +14487,7 @@ build_lrouter_ipv4_ip_input(struct ovn_port *op, > > &op->nbrp->header_, lflows); > > } > > > > - if (sset_count(&op->od->lb_ips->ips_v4_reachable)) { > > + if (lr_sful_rec && sset_count(&lr_sful_rec->lb_ips->ips_v4_reachable)) { > > ds_clear(match); > > if (is_l3dgw_port(op)) { > > ds_put_format(match, "is_chassis_resident(%s)", > > @@ -14776,7 +14503,7 @@ build_lrouter_ipv4_ip_input(struct ovn_port *op, > > free(lb_ips_v4_as); > > } > > > > - if (sset_count(&op->od->lb_ips->ips_v6_reachable)) { > > + if (lr_sful_rec && sset_count(&lr_sful_rec->lb_ips->ips_v6_reachable)) { > > ds_clear(match); > > > > if (is_l3dgw_port(op)) { > > @@ -14878,8 +14605,8 @@ build_lrouter_ipv4_ip_input(struct ovn_port *op, > > * Priority 60. > > */ > > if (!lrnat_rec->lb_force_snat_router_ip) { > > - build_lrouter_drop_own_dest(op, lrnat_rec, S_ROUTER_IN_IP_INPUT, 60, > > - false, lflows); > > + build_lrouter_drop_own_dest(op, lrnat_rec, lr_sful_rec, > > + S_ROUTER_IN_IP_INPUT, 60, false, lflows); > > } > > /* ARP / ND handling for external IP addresses. > > * > > @@ -16018,6 +15745,7 @@ struct lswitch_flow_build_info { > > const struct hmap *lr_ports; > > const struct ls_port_group_table *ls_port_groups; > > const struct lr_nat_table *lr_nats; > > + const struct lr_stateful_table *lr_sful_table; > > struct hmap *lflows; > > struct hmap *igmp_groups; > > const struct shash *meter_groups; > > @@ -16101,14 +15829,15 @@ build_lswitch_and_lrouter_iterate_by_lr(struct ovn_datapath *od, > > * switch port. > > */ > > static void > > -build_lswitch_and_lrouter_iterate_by_lsp(struct ovn_port *op, > > - const struct hmap *ls_ports, > > - const struct hmap *lr_ports, > > - const struct lr_nat_table *lr_nats, > > - const struct shash *meter_groups, > > - struct ds *match, > > - struct ds *actions, > > - struct hmap *lflows) > > +build_lswitch_and_lrouter_iterate_by_lsp( > > + struct ovn_port *op, const struct hmap *ls_ports, > > + const struct hmap *lr_ports, > > + const struct lr_nat_table *lr_nats, > > + const struct lr_stateful_table *lr_sful_table, > > + const struct shash *meter_groups, > > + struct ds *match, > > + struct ds *actions, > > + struct hmap *lflows) > > { > > ovs_assert(op->nbsp); > > start_collecting_lflows(); > > @@ -16121,11 +15850,14 @@ build_lswitch_and_lrouter_iterate_by_lsp(struct ovn_port *op, > > meter_groups, actions, match); > > build_lswitch_dhcp_options_and_response(op, lflows, meter_groups); > > build_lswitch_external_port(op, lflows); > > - build_lswitch_ip_unicast_lookup(op, lr_nats, lflows, actions, match); > > + build_lswitch_ip_unicast_lookup(op, lr_nats, lr_sful_table, lflows, > > + actions, match); > > > > /* Build Logical Router Flows. */ > > - build_ip_routing_flows_for_router_type_lsp(op, lr_ports, lflows); > > - build_arp_resolve_flows_for_lsp(op, lflows, lr_ports, match, actions); > > + build_ip_routing_flows_for_router_type_lsp(op, lr_sful_table, lr_ports, > > + lflows); > > + build_arp_resolve_flows_for_lsp(op, lflows, lr_ports, lr_sful_table, > > + match, actions); > > > > link_ovn_port_to_lflows(op, &collected_lflows); > > end_collecting_lflows(); > > @@ -16144,6 +15876,8 @@ build_lswitch_and_lrouter_iterate_by_lrp(struct ovn_port *op, > > lsi->lr_nats, op->od->index); > > ovs_assert(lrnet_rec); > > > > + const struct lr_stateful_record *lr_sful_rec = > > + lr_stateful_table_find_by_index(lsi->lr_sful_table, op->od->index); > > build_adm_ctrl_flows_for_lrouter_port(op, lsi->lflows, &lsi->match, > > &lsi->actions); > > build_neigh_learning_flows_for_lrouter_port(op, lsi->lflows, &lsi->match, > > @@ -16151,15 +15885,15 @@ build_lswitch_and_lrouter_iterate_by_lrp(struct ovn_port *op, > > build_ip_routing_flows_for_lrp(op, lsi->lflows); > > build_ND_RA_flows_for_lrouter_port(op, lsi->lflows, &lsi->match, > > &lsi->actions, lsi->meter_groups); > > - build_arp_resolve_flows_for_lrp(op, lrnet_rec, lsi->lflows, &lsi->match, > > - &lsi->actions); > > + build_arp_resolve_flows_for_lrp(op, lrnet_rec, lr_sful_rec, lsi->lflows, > > + &lsi->match, &lsi->actions); > > build_egress_delivery_flows_for_lrouter_port(op, lsi->lflows, &lsi->match, > > &lsi->actions); > > build_dhcpv6_reply_flows_for_lrouter_port(op, lsi->lflows, &lsi->match); > > build_ipv6_input_flows_for_lrouter_port(op, lsi->lflows, > > &lsi->match, &lsi->actions, > > lsi->meter_groups); > > - build_lrouter_ipv4_ip_input(op, lsi->lflows, lrnet_rec, > > + build_lrouter_ipv4_ip_input(op, lsi->lflows, lrnet_rec, lr_sful_rec, > > &lsi->match, &lsi->actions, lsi->meter_groups); > > build_lrouter_force_snat_flows_op(op, lrnet_rec, lsi->lflows, &lsi->match, > > &lsi->actions); > > @@ -16220,12 +15954,13 @@ build_lflows_thread(void *arg) > > return NULL; > > } > > build_lswitch_and_lrouter_iterate_by_lsp(op, lsi->ls_ports, > > - lsi->lr_ports, > > - lsi->lr_nats, > > - lsi->meter_groups, > > - &lsi->match, > > - &lsi->actions, > > - lsi->lflows); > > + lsi->lr_ports, > > + lsi->lr_nats, > > + lsi->lr_sful_table, > > + lsi->meter_groups, > > + &lsi->match, > > + &lsi->actions, > > + lsi->lflows); > > } > > } > > for (bnum = control->id; > > @@ -16332,6 +16067,7 @@ build_lswitch_and_lrouter_flows(const struct ovn_datapaths *ls_datapaths, > > const struct hmap *lr_ports, > > const struct ls_port_group_table *ls_pgs, > > const struct lr_nat_table *lr_nats, > > + const struct lr_stateful_table *lr_sful_table, > > struct hmap *lflows, > > struct hmap *igmp_groups, > > const struct shash *meter_groups, > > @@ -16362,6 +16098,7 @@ build_lswitch_and_lrouter_flows(const struct ovn_datapaths *ls_datapaths, > > lsiv[index].lr_ports = lr_ports; > > lsiv[index].ls_port_groups = ls_pgs; > > lsiv[index].lr_nats = lr_nats; > > + lsiv[index].lr_sful_table = lr_sful_table; > > lsiv[index].igmp_groups = igmp_groups; > > lsiv[index].meter_groups = meter_groups; > > lsiv[index].lb_dps_map = lb_dps_map; > > @@ -16397,6 +16134,7 @@ build_lswitch_and_lrouter_flows(const struct ovn_datapaths *ls_datapaths, > > .lr_ports = lr_ports, > > .ls_port_groups = ls_pgs, > > .lr_nats = lr_nats, > > + .lr_sful_table = lr_sful_table, > > .lflows = lflows, > > .igmp_groups = igmp_groups, > > .meter_groups = meter_groups, > > @@ -16425,6 +16163,7 @@ build_lswitch_and_lrouter_flows(const struct ovn_datapaths *ls_datapaths, > > build_lswitch_and_lrouter_iterate_by_lsp(op, lsi.ls_ports, > > lsi.lr_ports, > > lsi.lr_nats, > > + lsi.lr_sful_table, > > lsi.meter_groups, > > &lsi.match, &lsi.actions, > > lsi.lflows); > > @@ -16546,6 +16285,7 @@ void build_lflows(struct ovsdb_idl_txn *ovnsb_txn, > > input_data->lr_ports, > > input_data->ls_port_groups, > > input_data->lr_nats, > > + input_data->lr_sful_table, > > lflows, > > &igmp_groups, > > input_data->meter_groups, > > @@ -17024,6 +16764,7 @@ lflow_handle_northd_port_changes(struct ovsdb_idl_txn *ovnsb_txn, > > build_lswitch_and_lrouter_iterate_by_lsp(op, lflow_input->ls_ports, > > lflow_input->lr_ports, > > lflow_input->lr_nats, > > + lflow_input->lr_sful_table, > > lflow_input->meter_groups, > > &match, &actions, > > lflows); > > @@ -17059,11 +16800,11 @@ lflow_handle_northd_port_changes(struct ovsdb_idl_txn *ovnsb_txn, > > struct ds match = DS_EMPTY_INITIALIZER; > > struct ds actions = DS_EMPTY_INITIALIZER; > > build_lswitch_and_lrouter_iterate_by_lsp(op, lflow_input->ls_ports, > > - lflow_input->lr_ports, > > - lflow_input->lr_nats, > > - lflow_input->meter_groups, > > - &match, &actions, > > - lflows); > > + lflow_input->lr_ports, > > + lflow_input->lr_nats, > > + lflow_input->lr_sful_table, > > + lflow_input->meter_groups, > > + &match, &actions, lflows); > > ds_destroy(&match); > > ds_destroy(&actions); > > > > diff --git a/northd/northd.h b/northd/northd.h > > index 4dd260761e..6d64fff626 100644 > > --- a/northd/northd.h > > +++ b/northd/northd.h > > @@ -180,6 +180,7 @@ struct lflow_input { > > const struct hmap *lr_ports; > > const struct ls_port_group_table *ls_port_groups; > > const struct lr_nat_table *lr_nats; > > + const struct lr_stateful_table *lr_sful_table; > > const struct shash *meter_groups; > > const struct hmap *lb_datapaths_map; > > const struct hmap *bfd_connections; > > @@ -319,9 +320,6 @@ struct ovn_datapath { > > /* router datapath has a logical port with redirect-type set to bridged. */ > > bool redirect_bridged; > > > > - /* Load Balancer vIPs relevant for this datapath. */ > > - struct ovn_lb_ip_set *lb_ips; > > - > > struct ovn_port **localnet_ports; > > size_t n_localnet_ports; > > > > @@ -338,6 +336,119 @@ struct ovn_datapath { > > const struct ovn_datapath *ovn_datapath_find(const struct hmap *datapaths, > > const struct uuid *uuid); > > > > +/* A logical switch port or logical router port. > > + * > > + * In steady state, an ovn_port points to a northbound Logical_Switch_Port > > + * record (via 'nbsp') *or* a Logical_Router_Port record (via 'nbrp'), and to a > > + * southbound Port_Binding record (via 'sb'). As the state of the system > > + * changes, join_logical_ports() may determine that there is a new LSP or LRP > > + * that has no corresponding Port_Binding record (in which case build_ports()) > > + * will create the missing Port_Binding) or that a Port_Binding record exists > > + * that has no coresponding LSP (in which case build_ports() will delete the > > + * spurious Port_Binding). Thus, after build_ports() runs, any given ovn_port > > + * will have 'sb' nonnull, and 'nbsp' xor 'nbrp' nonnull. > > + * > > + * Ordinarily there is only one ovn_port that points to a given LSP or LRP (but > > + * distributed gateway ports point a "derived" ovn_port to a duplicate LRP). > > + */ > > +struct ovn_port { > > + /* Port name aka key. > > + * > > + * This is ordinarily the same as nbsp->name or nbrp->name and > > + * sb->logical_port. (A distributed gateway port creates a "derived" > > + * ovn_port with key "cr-%s" % nbrp->name.) */ > > + struct hmap_node key_node; /* Index on 'key'. */ > > + char *key; /* nbsp->name, nbrp->name, sb->logical_port. */ > > + char *json_key; /* 'key', quoted for use in JSON. */ > > + > > + const struct sbrec_port_binding *sb; /* May be NULL. */ > > + > > + uint32_t tunnel_key; > > + > > + /* Logical switch port data. */ > > + const struct nbrec_logical_switch_port *nbsp; /* May be NULL. */ > > + > > + struct lport_addresses *lsp_addrs; /* Logical switch port addresses. */ > > + unsigned int n_lsp_addrs; /* Total length of lsp_addrs. */ > > + unsigned int n_lsp_non_router_addrs; /* Number of elements from the > > + * beginning of 'lsp_addrs' extracted > > + * directly from LSP 'addresses'. */ > > + > > + struct lport_addresses *ps_addrs; /* Port security addresses. */ > > + unsigned int n_ps_addrs; > > + > > + bool lsp_can_be_inc_processed; /* If it can be incrementally processed when > > + the port changes. */ > > + > > + /* Logical router port data. */ > > + const struct nbrec_logical_router_port *nbrp; /* May be NULL. */ > > + > > + struct lport_addresses lrp_networks; > > + > > + /* Logical port multicast data. */ > > + struct mcast_port_info mcast_info; > > + > > + /* At most one of l3dgw_port and cr_port can be not NULL. */ > > + > > + /* This is set to a distributed gateway port if and only if this ovn_port > > + * is "derived" from it. Otherwise this is set to NULL. The derived > > + * ovn_port represents the instance of distributed gateway port on the > > + * gateway chassis.*/ > > + struct ovn_port *l3dgw_port; > > + > > + /* This is set to the "derived" chassis-redirect port of this port if and > > + * only if this port is a distributed gateway port. Otherwise this is set > > + * to NULL. */ > > + struct ovn_port *cr_port; > > + > > + bool has_unknown; /* If the addresses have 'unknown' defined. */ > > + > > + bool has_bfd; > > + > > + /* The port's peer: > > + * > > + * - A switch port S of type "router" has a router port R as a peer, > > + * and R in turn has S has its peer. > > + * > > + * - Two connected logical router ports have each other as peer. > > + * > > + * - Other kinds of ports have no peer. */ > > + struct ovn_port *peer; > > + > > + struct ovn_datapath *od; > > + > > + struct ovs_list list; /* In list of similar records. */ > > + > > + struct hmap_node dp_node; /* Node in od->ports. */ > > + > > + struct lport_addresses proxy_arp_addrs; > > + > > + /* Temporarily used for traversing a list (or hmap) of ports. */ > > + bool visited; > > + > > + /* List of struct lflow_ref_node that points to the lflows generated by > > + * this ovn_port. > > + * > > + * This data is initialized and destroyed by the en_northd node, but > > + * populated and used only by the en_lflow node. Ideally this data should > > + * be maintained as part of en_lflow's data (struct lflow_data): a hash > > + * index from ovn_port key to lflows. However, it would be less efficient > > + * and more complex: > > + * > > + * 1. It would require an extra search (using the index) to find the > > + * lflows. > > + * > > + * 2. Building the index needs to be thread-safe, using either a global > > + * lock which is obviously less efficient, or hash-based lock array which > > + * is more complex. > > + * > > + * Adding the list here is more straightforward. The drawback is that we > > + * need to keep in mind that this data belongs to en_lflow node, so never > > + * access it from any other nodes. > > + */ > > + struct ovs_list lflows; > > +}; > > + > > void ovnnb_db_run(struct northd_input *input_data, > > struct northd_data *data, > > struct ovsdb_idl_txn *ovnnb_txn, > > @@ -397,9 +508,13 @@ void sync_lbs(struct ovsdb_idl_txn *, const struct sbrec_load_balancer_table *, > > struct chassis_features *chassis_features); > > bool check_sb_lb_duplicates(const struct sbrec_load_balancer_table *); > > > > +struct lr_stateful_table; > > void sync_pbs(struct ovsdb_idl_txn *, struct hmap *ls_ports, > > - struct hmap *lr_ports, const struct lr_nat_table *); > > -bool sync_pbs_for_northd_changed_ovn_ports(struct tracked_ovn_ports *); > > + struct hmap *lr_ports, > > + const struct lr_stateful_table *); > > +bool sync_pbs_for_northd_changed_ovn_ports( > > + struct tracked_ovn_ports *, > > + const struct lr_stateful_table *); > > > > static inline bool > > northd_has_tracked_data(struct northd_tracked_data *trk_nd_changes) { > > @@ -424,4 +539,15 @@ northd_has_lr_nats_in_tracked_data(struct northd_tracked_data *trk_nd_changes) > > return (trk_nd_changes->type & NORTHD_TRACKED_LR_NATS); > > } > > > > +/* Returns 'true' if the IPv4 'addr' is on the same subnet with one of the > > + * IPs configured on the router port. > > + */ > > +bool lrouter_port_ipv4_reachable(const struct ovn_port *, ovs_be32 addr); > > + > > +/* Returns 'true' if the IPv6 'addr' is on the same subnet with one of the > > + * IPs configured on the router port. > > + */ > > +bool lrouter_port_ipv6_reachable(const struct ovn_port *, > > + const struct in6_addr *); > > + > > #endif /* NORTHD_H */ > > diff --git a/northd/ovn-northd.c b/northd/ovn-northd.c > > index 40f9764b3a..e85e00cc9b 100644 > > --- a/northd/ovn-northd.c > > +++ b/northd/ovn-northd.c > > @@ -871,6 +871,7 @@ main(int argc, char *argv[]) > > stopwatch_create(PORT_GROUP_RUN_STOPWATCH_NAME, SW_MS); > > stopwatch_create(SYNC_METERS_RUN_STOPWATCH_NAME, SW_MS); > > stopwatch_create(LR_NAT_RUN_STOPWATCH_NAME, SW_MS); > > + stopwatch_create(LR_STATEFUL_RUN_STOPWATCH_NAME, SW_MS); > > > > /* Initialize incremental processing engine for ovn-northd */ > > inc_proc_northd_init(&ovnnb_idl_loop, &ovnsb_idl_loop); > > diff --git a/tests/ovn-northd.at b/tests/ovn-northd.at > > index dc01e43de0..03d62695db 100644 > > --- a/tests/ovn-northd.at > > +++ b/tests/ovn-northd.at > > @@ -10516,18 +10516,21 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb set load_balancer . ip_port_mappings:10.0.0.3=sw0-p1:10.0.0.2 > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute nocompute > > > > check ovn-nbctl --wait=sb set load_balancer . options:foo=bar > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute nocompute > > > > check ovn-nbctl --wait=sb -- lb-add lb2 20.0.0.10:80 20.0.0.20:80 -- lb-add lb3 30.0.0.10:80 30.0.0.20:80 > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute nocompute > > > > @@ -10537,6 +10540,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb -- lb-del lb2 -- lb-del lb3 > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute nocompute > > > > @@ -10550,6 +10554,7 @@ AT_CHECK([ovn-nbctl --wait=sb \ > > ]) > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd recompute nocompute > > +check_engine_stats lr_stateful recompute nocompute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute nocompute > > > > @@ -10567,6 +10572,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb clear Load_Balancer . health_check > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd recompute nocompute > > +check_engine_stats lr_stateful recompute nocompute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute nocompute > > > > @@ -10581,6 +10587,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > ovn-nbctl --wait=sb lsp-set-options sw0-lr0 router-port=lr0-sw0 > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd recompute compute > > +check_engine_stats lr_stateful recompute nocompute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute nocompute > > > > @@ -10589,6 +10596,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb ls-lb-add sw0 lb1 > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > # A LB applied to a switch/router triggers: > > # - a recompute in the first iteration (handling northd change) > > @@ -10601,6 +10609,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb set load_balancer lb1 vips:'"10.0.0.10:80"'='"10.0.0.100:80"' > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute compute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE(1) > > @@ -10610,6 +10619,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb clear load_Balancer lb1 vips > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute compute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE(1) > > @@ -10619,6 +10629,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb lb-add lb1 10.0.0.10:80 10.0.0.3:80 > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute compute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE(1) > > @@ -10628,6 +10639,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb lb-add lb1 10.0.0.20:80 10.0.0.30:8080 > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute compute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE(1) > > @@ -10637,6 +10649,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb ls-lb-del sw0 lb1 > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd recompute nocompute > > +check_engine_stats lr_stateful recompute nocompute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute compute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE(1) > > @@ -10647,6 +10660,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb ls-lb-add sw0 lb1 -- lsp-add sw0 sw0p1 > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute compute > > > > @@ -10667,6 +10681,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb lr-lb-add lr0 lb1 > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute compute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -10676,6 +10691,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb set load_balancer lb1 vips:'"10.0.0.10:80"'='"10.0.0.100:80"' > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute compute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -10685,6 +10701,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb clear load_Balancer lb1 vips > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute compute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -10694,6 +10711,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb lb-add lb1 10.0.0.10:80 10.0.0.3:80 > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute compute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -10703,6 +10721,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb lb-add lb1 10.0.0.20:80 10.0.0.30:8080 > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute compute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -10733,6 +10752,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb add load_balancer_group . load_Balancer $lb1_uuid > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute nocompute > > > > @@ -10740,6 +10760,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb clear load_balancer_group . load_Balancer > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd recompute nocompute > > +check_engine_stats lr_stateful recompute nocompute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute nocompute > > > > @@ -10756,6 +10777,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb add logical_switch sw0 load_balancer_group $lbg1_uuid > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute compute > > > > @@ -10772,6 +10794,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb set load_balancer lb1 vips:'"10.0.0.10:80"'='"10.0.0.100:80"' > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute compute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -10781,6 +10804,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb clear load_Balancer lb1 vips > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute compute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -10790,6 +10814,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb lb-add lb1 10.0.0.10:80 10.0.0.3:80 > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute compute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -10799,6 +10824,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb lb-add lb1 10.0.0.20:80 10.0.0.30:8080 > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute compute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -10814,6 +10840,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl add logical_router lr0 load_balancer_group $lbg1_uuid > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute compute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -10823,6 +10850,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb set load_balancer lb1 vips:'"10.0.0.10:80"'='"10.0.0.100:80"' > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute compute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -10832,6 +10860,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb clear load_Balancer lb1 vips > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute compute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -10841,6 +10870,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb lb-add lb1 10.0.0.10:80 10.0.0.3:80 > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute compute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -10850,6 +10880,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb lb-add lb1 10.0.0.20:80 10.0.0.30:8080 > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute compute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -10858,6 +10889,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb clear logical_router lr0 load_balancer_group > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd recompute nocompute > > +check_engine_stats lr_stateful recompute nocompute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute compute > > > > @@ -10866,6 +10898,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb add logical_switch sw0 load_balancer_group $lbg1_uuid > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute compute > > > > @@ -10874,6 +10907,7 @@ check ovn-nbctl --wait=sb clear logical_switch sw0 load_balancer_group -- \ > > destroy load_balancer_group $lbg1_uuid > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd recompute nocompute > > +check_engine_stats lr_stateful recompute nocompute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb compute compute > > > > @@ -10897,6 +10931,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > lbg1_uuid=$(ovn-nbctl --wait=sb create load_balancer_group name=lbg1) > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow norecompute nocompute > > check_engine_stats sync_to_sb_lb norecompute nocompute > > > > @@ -10904,6 +10939,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb set load_balancer_group . load_balancer="$lb2_uuid,$lb3_uuid,$lb4_uuid" > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute nocompute > > > > @@ -10911,6 +10947,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb set logical_switch sw0 load_balancer_group=$lbg1_uuid > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute compute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -10919,6 +10956,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb set logical_router lr1 load_balancer_group=$lbg1_uuid > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute compute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -10927,6 +10965,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb ls-lb-add sw0 lb2 > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute nocompute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -10935,6 +10974,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb ls-lb-add sw0 lb3 > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute nocompute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -10944,6 +10984,7 @@ check ovn-nbctl --wait=sb lr-lb-add lr1 lb1 > > check ovn-nbctl --wait=sb lr-lb-add lr1 lb2 > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute compute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -10952,6 +10993,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb ls-lb-del sw0 lb2 > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd recompute nocompute > > +check_engine_stats lr_stateful recompute nocompute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute nocompute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -10960,6 +11002,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb lr-lb-del lr1 lb2 > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd recompute nocompute > > +check_engine_stats lr_stateful recompute nocompute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute nocompute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -10970,6 +11013,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb lb-del lb4 > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute compute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -10980,6 +11024,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb lb-del lb2 > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute compute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -10988,6 +11033,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb remove load_balancer_group . load_balancer $lb3_uuid > > check_engine_stats lb_data norecompute compute > > check_engine_stats northd recompute nocompute > > +check_engine_stats lr_stateful recompute nocompute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_lb recompute compute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -11054,6 +11100,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb lr-add lr0 > > check_engine_stats northd recompute nocompute > > check_engine_stats lr_nat recompute nocompute > > +check_engine_stats lr_stateful recompute nocompute > > check_engine_stats sync_to_sb_pb recompute nocompute > > check_engine_stats lflow recompute nocompute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -11066,6 +11113,7 @@ check ovn-nbctl lrp-add lr0 lr0-sw0 00:00:00:00:ff:01 10.0.0.1/24 > > # for the SB port binding change. > > check_engine_stats northd recompute compute > > check_engine_stats lr_nat recompute nocompute > > +check_engine_stats lr_stateful recompute nocompute > > check_engine_stats sync_to_sb_pb recompute nocompute > > check_engine_stats lflow recompute nocompute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -11077,6 +11125,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb lsp-set-options sw0-lr0 router-port=lr0-sw0 > > check_engine_stats northd recompute compute > > check_engine_stats lr_nat recompute nocompute > > +check_engine_stats lr_stateful recompute nocompute > > check_engine_stats sync_to_sb_pb recompute nocompute > > check_engine_stats lflow recompute nocompute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -11102,6 +11151,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl set logical_router_port lr0-sw0 options:foo=bar > > check_engine_stats northd recompute nocompute > > check_engine_stats lr_nat recompute nocompute > > +check_engine_stats lr_stateful recompute nocompute > > check_engine_stats sync_to_sb_pb recompute nocompute > > check_engine_stats lflow recompute nocompute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -11131,6 +11181,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb set NAT . external_ip=172.168.0.120 > > check_engine_stats northd norecompute compute > > check_engine_stats lr_nat norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_pb recompute nocompute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -11140,6 +11191,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb set NAT . logical_ip=10.0.0.10 > > check_engine_stats northd norecompute compute > > check_engine_stats lr_nat norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_pb recompute nocompute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -11149,6 +11201,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb set NAT . type=snat > > check_engine_stats northd norecompute compute > > check_engine_stats lr_nat norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_pb recompute nocompute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -11158,6 +11211,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb lr-nat-add lr0 dnat_and_snat 172.168.0.110 10.0.0.4 sw0p1 30:54:00:00:00:03 > > check_engine_stats northd norecompute compute > > check_engine_stats lr_nat norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_pb recompute nocompute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -11168,6 +11222,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb set NAT $nat2_uuid external_mac='"30:54:00:00:00:04"' > > check_engine_stats northd norecompute compute > > check_engine_stats lr_nat norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_pb recompute nocompute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -11184,6 +11239,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb lr-nat-add lr0 dnat_and_snat 172.168.0.140 10.0.0.20 > > check_engine_stats northd norecompute compute > > check_engine_stats lr_nat norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats sync_to_sb_pb recompute nocompute > > check_engine_stats lflow recompute nocompute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -11192,6 +11248,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb lr-nat-add lr0 dnat_and_snat 172.168.0.150 10.0.0.41 > > check_engine_stats northd norecompute compute > > check_engine_stats lr_nat norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats sync_to_sb_pb recompute nocompute > > check_engine_stats lflow recompute nocompute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -11200,6 +11257,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb lr-nat-del lr0 dnat_and_snat 172.168.0.150 > > check_engine_stats northd norecompute compute > > check_engine_stats lr_nat norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats sync_to_sb_pb recompute nocompute > > check_engine_stats lflow recompute nocompute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -11208,6 +11266,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb lr-nat-del lr0 dnat_and_snat 172.168.0.140 > > check_engine_stats northd norecompute compute > > check_engine_stats lr_nat norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats sync_to_sb_pb recompute nocompute > > check_engine_stats lflow recompute nocompute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -11217,6 +11276,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb clear logical_router lr0 nat > > check_engine_stats northd norecompute compute > > check_engine_stats lr_nat norecompute compute > > +check_engine_stats lr_stateful norecompute compute > > check_engine_stats lflow recompute nocompute > > check_engine_stats sync_to_sb_pb recompute nocompute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -11226,6 +11286,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb lr-policy-add lr0 10 "ip4.src == 10.0.0.3" reroute 172.168.0.101,172.168.0.102 > > check_engine_stats northd recompute nocompute > > check_engine_stats lr_nat recompute nocompute > > +check_engine_stats lr_stateful recompute nocompute > > check_engine_stats sync_to_sb_pb recompute nocompute > > check_engine_stats lflow recompute nocompute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > @@ -11234,6 +11295,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats > > check ovn-nbctl --wait=sb lr-policy-del lr0 10 "ip4.src == 10.0.0.3" > > check_engine_stats northd recompute nocompute > > check_engine_stats lr_nat recompute nocompute > > +check_engine_stats lr_stateful recompute nocompute > > check_engine_stats sync_to_sb_pb recompute nocompute > > check_engine_stats lflow recompute nocompute > > CHECK_NO_CHANGE_AFTER_RECOMPUTE > > _______________________________________________ > dev mailing list > dev@openvswitch.org > https://mail.openvswitch.org/mailman/listinfo/ovs-dev >
diff --git a/lib/stopwatch-names.h b/lib/stopwatch-names.h index 0a16da211e..63a04db9ca 100644 --- a/lib/stopwatch-names.h +++ b/lib/stopwatch-names.h @@ -33,5 +33,6 @@ #define PORT_GROUP_RUN_STOPWATCH_NAME "port_group_run" #define SYNC_METERS_RUN_STOPWATCH_NAME "sync_meters_run" #define LR_NAT_RUN_STOPWATCH_NAME "lr_nat_run" +#define LR_STATEFUL_RUN_STOPWATCH_NAME "lr_stateful" #endif diff --git a/northd/automake.mk b/northd/automake.mk index ae367a2a8b..3666cd452c 100644 --- a/northd/automake.mk +++ b/northd/automake.mk @@ -26,6 +26,8 @@ northd_ovn_northd_SOURCES = \ northd/en-lb-data.h \ northd/en-lr-nat.c \ northd/en-lr-nat.h \ + northd/en-lr-stateful.c \ + northd/en-lr-stateful.h \ northd/inc-proc-northd.c \ northd/inc-proc-northd.h \ northd/ipam.c \ diff --git a/northd/en-lflow.c b/northd/en-lflow.c index e4f875ef7c..0aee073520 100644 --- a/northd/en-lflow.c +++ b/northd/en-lflow.c @@ -20,6 +20,7 @@ #include "en-lflow.h" #include "en-lr-nat.h" +#include "en-lr-stateful.h" #include "en-northd.h" #include "en-meters.h" @@ -43,6 +44,8 @@ lflow_get_input_data(struct engine_node *node, engine_get_input_data("sync_meters", node); struct ed_type_lr_nat_data *lr_nat_data = engine_get_input_data("lr_nat", node); + struct ed_type_lr_stateful *lr_sful_data = + engine_get_input_data("lr_stateful", node); lflow_input->nbrec_bfd_table = EN_OVSDB_GET(engine_get_input("NB_bfd", node)); @@ -66,6 +69,7 @@ lflow_get_input_data(struct engine_node *node, lflow_input->lr_ports = &northd_data->lr_ports; lflow_input->ls_port_groups = &pg_data->ls_port_groups; lflow_input->lr_nats = &lr_nat_data->lr_nats; + lflow_input->lr_sful_table = &lr_sful_data->lr_sful_table; lflow_input->meter_groups = &sync_meters_data->meter_groups; lflow_input->lb_datapaths_map = &northd_data->lb_datapaths_map; lflow_input->svc_monitor_map = &northd_data->svc_monitor_map; diff --git a/northd/en-lr-nat.h b/northd/en-lr-nat.h index 3ec4c7b506..4d8e142ae6 100644 --- a/northd/en-lr-nat.h +++ b/northd/en-lr-nat.h @@ -86,6 +86,9 @@ struct lr_nat_table { const struct lr_nat_record * lr_nat_table_find_by_index( const struct lr_nat_table *, size_t od_index); +#define LR_NAT_TABLE_FOR_EACH(LR_NAT_REC, TABLE) \ + HMAP_FOR_EACH (LR_NAT_REC, key_node, &(TABLE)->entries) + struct ed_type_lr_nat_data { struct lr_nat_table lr_nats; diff --git a/northd/en-lr-stateful.c b/northd/en-lr-stateful.c new file mode 100644 index 0000000000..bd03c586da --- /dev/null +++ b/northd/en-lr-stateful.c @@ -0,0 +1,647 @@ +/* + * Copyright (c) 2023, Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <config.h> + +#include <getopt.h> +#include <stdlib.h> +#include <stdio.h> + +/* OVS includes */ +#include "include/openvswitch/hmap.h" +#include "lib/bitmap.h" +#include "lib/socket-util.h" +#include "lib/uuidset.h" +#include "openvswitch/util.h" +#include "openvswitch/vlog.h" +#include "stopwatch.h" + +/* OVN includes */ +#include "en-lb-data.h" +#include "en-lr-nat.h" +#include "en-lr-stateful.h" +#include "lib/inc-proc-eng.h" +#include "lib/lb.h" +#include "lib/ovn-nb-idl.h" +#include "lib/ovn-sb-idl.h" +#include "lib/ovn-util.h" +#include "lib/stopwatch-names.h" +#include "northd.h" + +VLOG_DEFINE_THIS_MODULE(en_lr_stateful); + +/* Static function declarations. */ +static void lr_stateful_table_init(struct lr_stateful_table *); +static void lr_stateful_table_clear(struct lr_stateful_table *); +static void lr_stateful_table_destroy(struct lr_stateful_table *); +static struct lr_stateful_record *lr_stateful_table_find_( + const struct lr_stateful_table *, const struct nbrec_logical_router *); +static struct lr_stateful_record *lr_stateful_table_find_by_index_( + const struct lr_stateful_table *table, size_t od_index); + +static void lr_stateful_table_build(struct lr_stateful_table *, + const struct lr_nat_table *, + const struct ovn_datapaths *lr_datapaths, + const struct hmap *lb_datapaths_map, + const struct hmap *lbgrp_datapaths_map); + +static struct lr_stateful_input lr_stateful_get_input_data( + struct engine_node *); + +static struct lr_stateful_record *lr_stateful_record_create( + struct lr_stateful_table *, const struct lr_nat_record *, + const struct hmap *lb_datapaths_map, + const struct hmap *lbgrp_datapaths_map); +static void lr_stateful_record_destroy(struct lr_stateful_record *); +static void lr_stateful_record_init( + struct lr_stateful_record *, + const struct hmap *lb_datapaths_map, + const struct hmap *lbgrp_datapaths_map); + +static void build_lrouter_lb_reachable_ips(struct lr_stateful_record *, + const struct ovn_northd_lb *); +static void add_neigh_ips_to_lrouter(struct lr_stateful_record *, + enum lb_neighbor_responder_mode, + const struct sset *lb_ips_v4, + const struct sset *lb_ips_v6); +static void remove_lrouter_lb_reachable_ips(struct lr_stateful_record *, + enum lb_neighbor_responder_mode, + const struct sset *lb_ips_v4, + const struct sset *lb_ips_v6); +static void lr_stateful_build_vip_nats(struct lr_stateful_record *); + +/* 'lr_stateful' engine node manages the NB logical router LB data. + */ +void * +en_lr_stateful_init(struct engine_node *node OVS_UNUSED, + struct engine_arg *arg OVS_UNUSED) +{ + struct ed_type_lr_stateful *data = xzalloc(sizeof *data); + lr_stateful_table_init(&data->lr_sful_table); + hmapx_init(&data->trk_data.crupdated); + return data; +} + +void +en_lr_stateful_cleanup(void *data_) +{ + struct ed_type_lr_stateful *data = + (struct ed_type_lr_stateful *) data_; + lr_stateful_table_destroy(&data->lr_sful_table); + hmapx_destroy(&data->trk_data.crupdated); +} + +void +en_lr_stateful_clear_tracked_data(void *data_) +{ + struct ed_type_lr_stateful *data = (struct ed_type_lr_stateful *) data_; + + hmapx_clear(&data->trk_data.crupdated); +} + +void +en_lr_stateful_run(struct engine_node *node, void *data_) +{ + struct lr_stateful_input input_data = lr_stateful_get_input_data(node); + struct ed_type_lr_stateful *data = data_; + + stopwatch_start(LR_STATEFUL_RUN_STOPWATCH_NAME, time_msec()); + + lr_stateful_table_clear(&data->lr_sful_table); + lr_stateful_table_build(&data->lr_sful_table, input_data.lr_nats, + input_data.lr_datapaths, + input_data.lb_datapaths_map, + input_data.lbgrp_datapaths_map); + + stopwatch_stop(LR_STATEFUL_RUN_STOPWATCH_NAME, time_msec()); + engine_set_node_state(node, EN_UPDATED); +} + +bool +lr_stateful_northd_handler(struct engine_node *node, void *data OVS_UNUSED) +{ + struct northd_data *northd_data = engine_get_input_data("northd", node); + if (!northd_has_tracked_data(&northd_data->trk_data)) { + return false; + } + + /* lr_stateful node needs inputs for any changes to NAT and load balancers. + * Changes to NAT is provided by the lr_nat tracked data and changes + * to lbs and lb grps is provided by lb_data's tracked data. + * So we don't need to do anything here for northd changes. + * But we do need to access the datapaths and lb_datapaths from the + * northd engine node and hence its an input. + * */ + return true; +} + +bool +lr_stateful_lb_data_handler(struct engine_node *node, void *data_) +{ + struct ed_type_lb_data *lb_data = engine_get_input_data("lb_data", node); + if (!lb_data->tracked) { + return false; + } + + struct ed_type_lr_stateful *data = + (struct ed_type_lr_stateful *) data_; + struct lr_stateful_input input_data = + lr_stateful_get_input_data(node); + struct lr_stateful_record *lr_sful_rec; + size_t index; + + const struct tracked_lb_data *trk_lb_data = &lb_data->tracked_lb_data; + const struct ovn_lb_group_datapaths *lbgrp_dps; + const struct crupdated_lbgrp *crupdated_lbgrp; + const struct crupdated_od_lb_data *codlb; + const struct ovn_lb_datapaths *lb_dps; + const struct crupdated_lb *clb; + const struct ovn_northd_lb *lb; + const struct ovn_datapath *od; + + LIST_FOR_EACH (codlb, list_node, &trk_lb_data->crupdated_lr_lbs) { + od = ovn_datapath_find(&input_data.lr_datapaths->datapaths, + &codlb->od_uuid); + ovs_assert(od); + + lr_sful_rec = lr_stateful_table_find_(&data->lr_sful_table, od->nbr); + if (!lr_sful_rec) { + const struct lr_nat_record *lrnat_rec = lr_nat_table_find_by_index( + input_data.lr_nats, od->index); + ovs_assert(lrnat_rec); + + lr_sful_rec = lr_stateful_record_create(&data->lr_sful_table, + lrnat_rec, + input_data.lb_datapaths_map, + input_data.lbgrp_datapaths_map); + + /* Add the lr_sful_rec rec to the tracking data. */ + hmapx_add(&data->trk_data.crupdated, lr_sful_rec); + continue; + } + + struct uuidset_node *uuidnode; + UUIDSET_FOR_EACH (uuidnode, &codlb->assoc_lbs) { + lb_dps = ovn_lb_datapaths_find( + input_data.lb_datapaths_map, &uuidnode->uuid); + ovs_assert(lb_dps); + + /* Add the lb_ips of lb_dps to the od. */ + build_lrouter_lb_ips(lr_sful_rec->lb_ips, lb_dps->lb); + build_lrouter_lb_reachable_ips(lr_sful_rec, lb_dps->lb); + } + + UUIDSET_FOR_EACH (uuidnode, &codlb->assoc_lbgrps) { + lbgrp_dps = ovn_lb_group_datapaths_find( + input_data.lbgrp_datapaths_map, &uuidnode->uuid); + ovs_assert(lbgrp_dps); + + for (size_t j = 0; j < lbgrp_dps->lb_group->n_lbs; j++) { + const struct uuid *lb_uuid + = &lbgrp_dps->lb_group->lbs[j]->nlb->header_.uuid; + lb_dps = ovn_lb_datapaths_find(input_data.lb_datapaths_map, + lb_uuid); + ovs_assert(lb_dps); + + /* Add the lb_ips of lb_dps to the od. */ + build_lrouter_lb_ips(lr_sful_rec->lb_ips, lb_dps->lb); + build_lrouter_lb_reachable_ips(lr_sful_rec, lb_dps->lb); + } + } + + /* Add the lr_sful_rec rec to the tracking data. */ + hmapx_add(&data->trk_data.crupdated, lr_sful_rec); + } + + HMAP_FOR_EACH (clb, hmap_node, &trk_lb_data->crupdated_lbs) { + lb = clb->lb; + const struct uuid *lb_uuid = &lb->nlb->header_.uuid; + + lb_dps = ovn_lb_datapaths_find(input_data.lb_datapaths_map, lb_uuid); + ovs_assert(lb_dps); + + BITMAP_FOR_EACH_1 (index, ods_size(input_data.lr_datapaths), + lb_dps->nb_lr_map) { + od = input_data.lr_datapaths->array[index]; + + lr_sful_rec = lr_stateful_table_find_(&data->lr_sful_table, + od->nbr); + ovs_assert(lr_sful_rec); + + /* Update the od->lb_ips with the deleted and inserted + * vips (if any). */ + remove_ips_from_lb_ip_set(lr_sful_rec->lb_ips, lb->routable, + &clb->deleted_vips_v4, + &clb->deleted_vips_v6); + add_ips_to_lb_ip_set(lr_sful_rec->lb_ips, lb->routable, + &clb->inserted_vips_v4, + &clb->inserted_vips_v6); + + remove_lrouter_lb_reachable_ips(lr_sful_rec, lb->neigh_mode, + &clb->deleted_vips_v4, + &clb->deleted_vips_v6); + add_neigh_ips_to_lrouter(lr_sful_rec, lb->neigh_mode, + &clb->inserted_vips_v4, + &clb->inserted_vips_v6); + + /* Add the lr_sful_rec rec to the tracking data. */ + hmapx_add(&data->trk_data.crupdated, lr_sful_rec); + } + } + + HMAP_FOR_EACH (crupdated_lbgrp, hmap_node, + &trk_lb_data->crupdated_lbgrps) { + const struct uuid *lb_uuid = &crupdated_lbgrp->lbgrp->uuid; + + lbgrp_dps = ovn_lb_group_datapaths_find(input_data.lbgrp_datapaths_map, + lb_uuid); + ovs_assert(lbgrp_dps); + + struct hmapx_node *hnode; + HMAPX_FOR_EACH (hnode, &crupdated_lbgrp->assoc_lbs) { + lb = hnode->data; + lb_uuid = &lb->nlb->header_.uuid; + lb_dps = ovn_lb_datapaths_find(input_data.lb_datapaths_map, + lb_uuid); + ovs_assert(lb_dps); + for (size_t i = 0; i < lbgrp_dps->n_lr; i++) { + od = lbgrp_dps->lr[i]; + lr_sful_rec = lr_stateful_table_find_(&data->lr_sful_table, + od->nbr); + ovs_assert(lr_sful_rec); + /* Add the lb_ips of lb_dps to the lr lb data. */ + build_lrouter_lb_ips(lr_sful_rec->lb_ips, lb_dps->lb); + build_lrouter_lb_reachable_ips(lr_sful_rec, lb_dps->lb); + + /* Add the lr_sful_rec rec to the tracking data. */ + hmapx_add(&data->trk_data.crupdated, lr_sful_rec); + } + } + } + + if (lr_stateful_has_tracked_data(&data->trk_data)) { + struct hmapx_node *hmapx_node; + /* For all the modified lr_stateful records (re)build the + * vip nats. */ + HMAPX_FOR_EACH (hmapx_node, &data->trk_data.crupdated) { + lr_stateful_build_vip_nats(hmapx_node->data); + } + + engine_set_node_state(node, EN_UPDATED); + } + + return true; +} + +bool +lr_stateful_lr_nat_handler(struct engine_node *node, void *data_) +{ + struct ed_type_lr_nat_data *lr_nat_data = + engine_get_input_data("lr_nat", node); + + if (!lr_nat_has_tracked_data(&lr_nat_data->trk_data)) { + return false; + } + + struct ed_type_lr_stateful *data = + (struct ed_type_lr_stateful *) data_; + struct lr_stateful_input input_data = + lr_stateful_get_input_data(node); + const struct lr_nat_record *lrnat_rec; + struct lr_stateful_record *lr_sful_rec; + struct hmapx_node *hmapx_node; + + HMAPX_FOR_EACH (hmapx_node, &lr_nat_data->trk_data.crupdated) { + lrnat_rec = hmapx_node->data; + lr_sful_rec = lr_stateful_table_find_(&data->lr_sful_table, + lrnat_rec->od->nbr); + if (!lr_sful_rec) { + lr_sful_rec = lr_stateful_record_create(&data->lr_sful_table, + lrnat_rec, + input_data.lb_datapaths_map, + input_data.lbgrp_datapaths_map); + } else { + lr_stateful_build_vip_nats(lr_sful_rec); + } + + /* Add the lr_sful_rec rec to the tracking data. */ + hmapx_add(&data->trk_data.crupdated, lr_sful_rec); + } + + if (lr_stateful_has_tracked_data(&data->trk_data)) { + engine_set_node_state(node, EN_UPDATED); + } + + return true; +} + +const struct lr_stateful_record * +lr_stateful_table_find_by_index(const struct lr_stateful_table *table, + size_t od_index) +{ + return lr_stateful_table_find_by_index_(table, od_index); +} + +/* static functions. */ +static void +lr_stateful_table_init(struct lr_stateful_table *table) +{ + *table = (struct lr_stateful_table) { + .entries = HMAP_INITIALIZER(&table->entries), + }; +} + +static void +lr_stateful_table_destroy(struct lr_stateful_table *table) +{ + lr_stateful_table_clear(table); + hmap_destroy(&table->entries); +} + +static void +lr_stateful_table_clear(struct lr_stateful_table *table) +{ + struct lr_stateful_record *lr_sful_rec; + HMAP_FOR_EACH_POP (lr_sful_rec, key_node, &table->entries) { + lr_stateful_record_destroy(lr_sful_rec); + } + + free(table->array); + table->array = NULL; +} + +static void +lr_stateful_table_build(struct lr_stateful_table *table, + const struct lr_nat_table *lr_nats, + const struct ovn_datapaths *lr_datapaths, + const struct hmap *lb_datapaths_map, + const struct hmap *lbgrp_datapaths_map) +{ + table->array = xrealloc(table->array, + ods_size(lr_datapaths) * sizeof *table->array); + const struct lr_nat_record *lrnat_rec; + LR_NAT_TABLE_FOR_EACH (lrnat_rec, lr_nats) { + lr_stateful_record_create(table, lrnat_rec, lb_datapaths_map, + lbgrp_datapaths_map); + } +} + +static struct lr_stateful_record * +lr_stateful_table_find_(const struct lr_stateful_table *table, + const struct nbrec_logical_router *nbr) +{ + struct lr_stateful_record *lr_sful_rec; + + HMAP_FOR_EACH_WITH_HASH (lr_sful_rec, key_node, + uuid_hash(&nbr->header_.uuid), &table->entries) { + if (nbr == lr_sful_rec->od->nbr) { + return lr_sful_rec; + } + } + return NULL; +} + +static struct lr_stateful_record * +lr_stateful_table_find_by_index_(const struct lr_stateful_table *table, + size_t od_index) +{ + ovs_assert(od_index <= hmap_count(&table->entries)); + return table->array[od_index]; +} + +static struct lr_stateful_record * +lr_stateful_record_create(struct lr_stateful_table *table, + const struct lr_nat_record *lrnat_rec, + const struct hmap *lb_datapaths_map, + const struct hmap *lbgrp_datapaths_map) +{ + struct lr_stateful_record *lr_sful_rec = xzalloc(sizeof *lr_sful_rec); + lr_sful_rec->lrnat_rec = lrnat_rec; + lr_sful_rec->od = lrnat_rec->od; + lr_stateful_record_init(lr_sful_rec, lb_datapaths_map, + lbgrp_datapaths_map); + + hmap_insert(&table->entries, &lr_sful_rec->key_node, + uuid_hash(&lr_sful_rec->od->nbr->header_.uuid)); + + table->array[lr_sful_rec->od->index] = lr_sful_rec; + return lr_sful_rec; +} + +static void +lr_stateful_record_destroy(struct lr_stateful_record *lr_sful_rec) +{ + ovn_lb_ip_set_destroy(lr_sful_rec->lb_ips); + lr_sful_rec->lb_ips = NULL; + sset_destroy(&lr_sful_rec->vip_nats); + free(lr_sful_rec); +} + +static void +lr_stateful_record_init(struct lr_stateful_record *lr_sful_rec, + const struct hmap *lb_datapaths_map, + const struct hmap *lbgrp_datapaths_map) +{ + const struct nbrec_load_balancer_group *nbrec_lb_group; + const struct ovn_lb_group_datapaths *lb_group_dps; + const struct ovn_lb_datapaths *lb_dps; + + /* Checking load balancer groups first, starting from the largest one, + * to more efficiently copy IP sets. */ + size_t largest_group = 0; + + const struct nbrec_logical_router *nbr = lr_sful_rec->od->nbr; + for (size_t i = 1; i < nbr->n_load_balancer_group; i++) { + if (nbr->load_balancer_group[i]->n_load_balancer > + nbr->load_balancer_group[largest_group]->n_load_balancer) { + largest_group = i; + } + } + + for (size_t i = 0; i < nbr->n_load_balancer_group; i++) { + size_t idx = (i + largest_group) % nbr->n_load_balancer_group; + + nbrec_lb_group = nbr->load_balancer_group[idx]; + const struct uuid *lbgrp_uuid = &nbrec_lb_group->header_.uuid; + + lb_group_dps = + ovn_lb_group_datapaths_find(lbgrp_datapaths_map, + lbgrp_uuid); + ovs_assert(lb_group_dps); + + if (!lr_sful_rec->lb_ips) { + lr_sful_rec->lb_ips = + ovn_lb_ip_set_clone(lb_group_dps->lb_group->lb_ips); + } else { + for (size_t j = 0; j < lb_group_dps->lb_group->n_lbs; j++) { + build_lrouter_lb_ips(lr_sful_rec->lb_ips, + lb_group_dps->lb_group->lbs[j]); + } + } + + for (size_t j = 0; j < lb_group_dps->lb_group->n_lbs; j++) { + build_lrouter_lb_reachable_ips(lr_sful_rec, + lb_group_dps->lb_group->lbs[j]); + } + } + + if (!lr_sful_rec->lb_ips) { + lr_sful_rec->lb_ips = ovn_lb_ip_set_create(); + } + + for (size_t i = 0; i < nbr->n_load_balancer; i++) { + const struct uuid *lb_uuid = + &nbr->load_balancer[i]->header_.uuid; + lb_dps = ovn_lb_datapaths_find(lb_datapaths_map, lb_uuid); + ovs_assert(lb_dps); + build_lrouter_lb_ips(lr_sful_rec->lb_ips, lb_dps->lb); + build_lrouter_lb_reachable_ips(lr_sful_rec, lb_dps->lb); + } + + sset_init(&lr_sful_rec->vip_nats); + + if (!nbr->n_nat) { + lr_stateful_build_vip_nats(lr_sful_rec); + } +} + +static struct lr_stateful_input +lr_stateful_get_input_data(struct engine_node *node) +{ + struct northd_data *northd_data = engine_get_input_data("northd", node); + struct ed_type_lr_nat_data *lr_nat_data = + engine_get_input_data("lr_nat", node); + + return (struct lr_stateful_input) { + .lr_datapaths = &northd_data->lr_datapaths, + .lb_datapaths_map = &northd_data->lb_datapaths_map, + .lbgrp_datapaths_map = &northd_data->lb_group_datapaths_map, + .lr_nats = &lr_nat_data->lr_nats, + }; +} + +static void +build_lrouter_lb_reachable_ips(struct lr_stateful_record *lr_sful_rec, + const struct ovn_northd_lb *lb) +{ + add_neigh_ips_to_lrouter(lr_sful_rec, lb->neigh_mode, &lb->ips_v4, + &lb->ips_v6); +} + +static void +add_neigh_ips_to_lrouter(struct lr_stateful_record *lr_sful_rec, + enum lb_neighbor_responder_mode neigh_mode, + const struct sset *lb_ips_v4, + const struct sset *lb_ips_v6) +{ + /* If configured to not reply to any neighbor requests for all VIPs + * return early. + */ + if (neigh_mode == LB_NEIGH_RESPOND_NONE) { + return; + } + + const char *ip_address; + + /* If configured to reply to neighbor requests for all VIPs force them + * all to be considered "reachable". + */ + if (neigh_mode == LB_NEIGH_RESPOND_ALL) { + SSET_FOR_EACH (ip_address, lb_ips_v4) { + sset_add(&lr_sful_rec->lb_ips->ips_v4_reachable, ip_address); + } + SSET_FOR_EACH (ip_address, lb_ips_v6) { + sset_add(&lr_sful_rec->lb_ips->ips_v6_reachable, ip_address); + } + + return; + } + + /* Otherwise, a VIP is reachable if there's at least one router + * subnet that includes it. + */ + ovs_assert(neigh_mode == LB_NEIGH_RESPOND_REACHABLE); + + SSET_FOR_EACH (ip_address, lb_ips_v4) { + struct ovn_port *op; + ovs_be32 vip_ip4; + if (ip_parse(ip_address, &vip_ip4)) { + HMAP_FOR_EACH (op, dp_node, &lr_sful_rec->od->ports) { + if (lrouter_port_ipv4_reachable(op, vip_ip4)) { + sset_add(&lr_sful_rec->lb_ips->ips_v4_reachable, + ip_address); + break; + } + } + } + } + + SSET_FOR_EACH (ip_address, lb_ips_v6) { + struct ovn_port *op; + struct in6_addr vip; + if (ipv6_parse(ip_address, &vip)) { + HMAP_FOR_EACH (op, dp_node, &lr_sful_rec->od->ports) { + if (lrouter_port_ipv6_reachable(op, &vip)) { + sset_add(&lr_sful_rec->lb_ips->ips_v6_reachable, + ip_address); + break; + } + } + } + } +} + +static void +remove_lrouter_lb_reachable_ips(struct lr_stateful_record *lr_sful_rec, + enum lb_neighbor_responder_mode neigh_mode, + const struct sset *lb_ips_v4, + const struct sset *lb_ips_v6) +{ + if (neigh_mode == LB_NEIGH_RESPOND_NONE) { + return; + } + + const char *ip_address; + SSET_FOR_EACH (ip_address, lb_ips_v4) { + sset_find_and_delete(&lr_sful_rec->lb_ips->ips_v4_reachable, + ip_address); + } + SSET_FOR_EACH (ip_address, lb_ips_v6) { + sset_find_and_delete(&lr_sful_rec->lb_ips->ips_v6_reachable, + ip_address); + } +} + +static void +lr_stateful_build_vip_nats(struct lr_stateful_record *lr_sful_rec) +{ + sset_clear(&lr_sful_rec->vip_nats); + const char *external_ip; + SSET_FOR_EACH (external_ip, &lr_sful_rec->lrnat_rec->external_ips) { + bool is_vip_nat = false; + if (addr_is_ipv6(external_ip)) { + is_vip_nat = sset_contains(&lr_sful_rec->lb_ips->ips_v6, + external_ip); + } else { + is_vip_nat = sset_contains(&lr_sful_rec->lb_ips->ips_v4, + external_ip); + } + + if (is_vip_nat) { + sset_add(&lr_sful_rec->vip_nats, external_ip); + } + } +} diff --git a/northd/en-lr-stateful.h b/northd/en-lr-stateful.h new file mode 100644 index 0000000000..a72f000d36 --- /dev/null +++ b/northd/en-lr-stateful.h @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2023, Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef EN_LR_STATEFUL_H +#define EN_LR_STATEFUL_H 1 + +#include <stdint.h> + +/* OVS includes. */ +#include "lib/hmapx.h" +#include "openvswitch/hmap.h" +#include "sset.h" + +/* OVN includes. */ +#include "lib/inc-proc-eng.h" +#include "lib/lb.h" +#include "lib/ovn-nb-idl.h" +#include "lib/ovn-sb-idl.h" +#include "lib/ovn-util.h" + +struct ovn_datapath; +struct lr_nat_record; + +/* lr_stateful_table: This represents a table of logical routers with + * stateful related data. + * stateful related data has two main components + * - NAT and + * - Load balancers. + * + * lr_stateful_record: It is a record in the lr_stateful_table for each + * logical router. + */ + +struct lr_stateful_record { + struct hmap_node key_node; /* Index on 'nbr->header_.uuid'. */ + + const struct ovn_datapath *od; + const struct lr_nat_record *lrnat_rec; + + /* Load Balancer vIPs relevant for this datapath. */ + struct ovn_lb_ip_set *lb_ips; + + /* sset of vips which are also part of lr nats. */ + struct sset vip_nats; +}; + +struct lr_stateful_table { + struct hmap entries; + + /* The array index of each element in 'entries'. */ + struct lr_stateful_record **array; +}; + +#define LR_STATEFUL_TABLE_FOR_EACH(LR_LB_NAT_REC, TABLE) \ + HMAP_FOR_EACH (LR_LB_NAT_REC, key_node, &(TABLE)->entries) + +struct lr_stateful_tracked_data { + /* Created or updated logical router with LB and/or NAT data. */ + struct hmapx crupdated; /* Stores 'struct lr_stateful_record'. */ +}; + +struct ed_type_lr_stateful { + struct lr_stateful_table lr_sful_table; + + struct lr_stateful_tracked_data trk_data; +}; + +struct lr_stateful_input { + const struct ovn_datapaths *lr_datapaths; + const struct hmap *lb_datapaths_map; + const struct hmap *lbgrp_datapaths_map; + const struct lr_nat_table *lr_nats; +}; + +void *en_lr_stateful_init(struct engine_node *, struct engine_arg *); +void en_lr_stateful_cleanup(void *data); +void en_lr_stateful_clear_tracked_data(void *data); +void en_lr_stateful_run(struct engine_node *, void *data); + +bool lr_stateful_northd_handler(struct engine_node *, void *data); +bool lr_stateful_lr_nat_handler(struct engine_node *, void *data); +bool lr_stateful_lb_data_handler(struct engine_node *, void *data); + +const struct lr_stateful_record *lr_stateful_table_find_by_index( + const struct lr_stateful_table *, size_t od_index); + +static inline bool +lr_stateful_has_tracked_data(struct lr_stateful_tracked_data *trk_data) { + return !hmapx_is_empty(&trk_data->crupdated); +} + +#endif /* EN_lr_stateful_H */ diff --git a/northd/en-sync-sb.c b/northd/en-sync-sb.c index 11e12428f7..733fb9024e 100644 --- a/northd/en-sync-sb.c +++ b/northd/en-sync-sb.c @@ -22,6 +22,7 @@ #include "openvswitch/util.h" #include "en-lr-nat.h" +#include "en-lr-stateful.h" #include "en-sync-sb.h" #include "lib/inc-proc-eng.h" #include "lib/lb.h" @@ -41,7 +42,7 @@ static void sync_addr_sets(struct ovsdb_idl_txn *ovnsb_txn, const struct nbrec_address_set_table *, const struct nbrec_port_group_table *, const struct sbrec_address_set_table *, - const struct ovn_datapaths *lr_datapaths); + const struct lr_stateful_table *); static const struct sbrec_address_set *sb_address_set_lookup_by_name( struct ovsdb_idl_index *, const char *name); static void update_sb_addr_set(struct sorted_array *, @@ -87,11 +88,11 @@ en_sync_to_sb_addr_set_run(struct engine_node *node, void *data OVS_UNUSED) EN_OVSDB_GET(engine_get_input("SB_address_set", node)); const struct engine_context *eng_ctx = engine_get_context(); - struct northd_data *northd_data = engine_get_input_data("northd", node); - + const struct ed_type_lr_stateful *lr_stateful_data = + engine_get_input_data("lr_stateful", node); sync_addr_sets(eng_ctx->ovnsb_idl_txn, nb_address_set_table, nb_port_group_table, sb_address_set_table, - &northd_data->lr_datapaths); + &lr_stateful_data->lr_sful_table); engine_set_node_state(node, EN_UPDATED); } @@ -288,10 +289,12 @@ en_sync_to_sb_pb_run(struct engine_node *node, void *data OVS_UNUSED) { const struct engine_context *eng_ctx = engine_get_context(); struct northd_data *northd_data = engine_get_input_data("northd", node); - struct ed_type_lr_nat_data *lr_nat_data = - engine_get_input_data("lr_nat", node); + struct ed_type_lr_stateful *lr_stateful_data = + engine_get_input_data("lr_stateful", node); + sync_pbs(eng_ctx->ovnsb_idl_txn, &northd_data->ls_ports, - &northd_data->lr_ports, &lr_nat_data->lr_nats); + &northd_data->lr_ports, + &lr_stateful_data->lr_sful_table); engine_set_node_state(node, EN_UPDATED); } @@ -316,7 +319,11 @@ sync_to_sb_pb_northd_handler(struct engine_node *node, void *data OVS_UNUSED) return false; } - if (!sync_pbs_for_northd_changed_ovn_ports(&nd->trk_data.trk_lsps)) { + struct ed_type_lr_stateful *lr_sful_data = + engine_get_input_data("lr_stateful", node); + + if (!sync_pbs_for_northd_changed_ovn_ports(&nd->trk_data.trk_lsps, + &lr_sful_data->lr_sful_table)) { return false; } @@ -362,7 +369,7 @@ sync_addr_sets(struct ovsdb_idl_txn *ovnsb_txn, const struct nbrec_address_set_table *nb_address_set_table, const struct nbrec_port_group_table *nb_port_group_table, const struct sbrec_address_set_table *sb_address_set_table, - const struct ovn_datapaths *lr_datapaths) + const struct lr_stateful_table *lr_statefuls) { struct shash sb_address_sets = SHASH_INITIALIZER(&sb_address_sets); @@ -406,16 +413,14 @@ sync_addr_sets(struct ovsdb_idl_txn *ovnsb_txn, } /* Sync router load balancer VIP generated address sets. */ - struct ovn_datapath *od; - HMAP_FOR_EACH (od, key_node, &lr_datapaths->datapaths) { - ovs_assert(od->nbr); - - if (sset_count(&od->lb_ips->ips_v4_reachable)) { - char *ipv4_addrs_name = lr_lb_address_set_name(od->tunnel_key, - AF_INET); + const struct lr_stateful_record *lr_sful_rec; + LR_STATEFUL_TABLE_FOR_EACH (lr_sful_rec, lr_statefuls) { + if (sset_count(&lr_sful_rec->lb_ips->ips_v4_reachable)) { + char *ipv4_addrs_name = + lr_lb_address_set_name(lr_sful_rec->od->tunnel_key, AF_INET); struct sorted_array ipv4_addrs_sorted = - sorted_array_from_sset(&od->lb_ips->ips_v4_reachable); + sorted_array_from_sset(&lr_sful_rec->lb_ips->ips_v4_reachable); sync_addr_set(ovnsb_txn, ipv4_addrs_name, &ipv4_addrs_sorted, &sb_address_sets); @@ -423,11 +428,11 @@ sync_addr_sets(struct ovsdb_idl_txn *ovnsb_txn, free(ipv4_addrs_name); } - if (sset_count(&od->lb_ips->ips_v6_reachable)) { - char *ipv6_addrs_name = lr_lb_address_set_name(od->tunnel_key, - AF_INET6); - struct sorted_array ipv6_addrs_sorted = - sorted_array_from_sset(&od->lb_ips->ips_v6_reachable); + if (sset_count(&lr_sful_rec->lb_ips->ips_v6_reachable)) { + char *ipv6_addrs_name = + lr_lb_address_set_name(lr_sful_rec->od->tunnel_key, AF_INET6); + struct sorted_array ipv6_addrs_sorted = sorted_array_from_sset( + &lr_sful_rec->lb_ips->ips_v6_reachable); sync_addr_set(ovnsb_txn, ipv6_addrs_name, &ipv6_addrs_sorted, &sb_address_sets); diff --git a/northd/inc-proc-northd.c b/northd/inc-proc-northd.c index 1f211b278e..97bcce9655 100644 --- a/northd/inc-proc-northd.c +++ b/northd/inc-proc-northd.c @@ -31,6 +31,7 @@ #include "openvswitch/vlog.h" #include "inc-proc-northd.h" #include "en-lb-data.h" +#include "en-lr-stateful.h" #include "en-lr-nat.h" #include "en-northd.h" #include "en-lflow.h" @@ -148,6 +149,7 @@ static ENGINE_NODE(sync_to_sb_lb, "sync_to_sb_lb"); static ENGINE_NODE(sync_to_sb_pb, "sync_to_sb_pb"); static ENGINE_NODE_WITH_CLEAR_TRACK_DATA(lb_data, "lb_data"); static ENGINE_NODE_WITH_CLEAR_TRACK_DATA(lr_nat, "lr_nat"); +static ENGINE_NODE_WITH_CLEAR_TRACK_DATA(lr_stateful, "lr_stateful"); void inc_proc_northd_init(struct ovsdb_idl_loop *nb, struct ovsdb_idl_loop *sb) @@ -193,6 +195,11 @@ void inc_proc_northd_init(struct ovsdb_idl_loop *nb, engine_add_input(&en_lr_nat, &en_northd, lr_nat_northd_handler); + engine_add_input(&en_lr_stateful, &en_northd, lr_stateful_northd_handler); + engine_add_input(&en_lr_stateful, &en_lr_nat, lr_stateful_lr_nat_handler); + engine_add_input(&en_lr_stateful, &en_lb_data, + lr_stateful_lb_data_handler); + engine_add_input(&en_mac_binding_aging, &en_nb_nb_global, NULL); engine_add_input(&en_mac_binding_aging, &en_sb_mac_binding, NULL); engine_add_input(&en_mac_binding_aging, &en_northd, NULL); @@ -214,15 +221,17 @@ void inc_proc_northd_init(struct ovsdb_idl_loop *nb, engine_add_input(&en_lflow, &en_sb_logical_flow, NULL); engine_add_input(&en_lflow, &en_sb_multicast_group, NULL); engine_add_input(&en_lflow, &en_sb_igmp_group, NULL); + engine_add_input(&en_lflow, &en_lr_nat, NULL); + engine_add_input(&en_lflow, &en_lr_stateful, NULL); engine_add_input(&en_lflow, &en_northd, lflow_northd_handler); engine_add_input(&en_lflow, &en_port_group, lflow_port_group_handler); - engine_add_input(&en_lflow, &en_lr_nat, NULL); engine_add_input(&en_sync_to_sb_addr_set, &en_nb_address_set, sync_to_sb_addr_set_nb_address_set_handler); engine_add_input(&en_sync_to_sb_addr_set, &en_nb_port_group, sync_to_sb_addr_set_nb_port_group_handler); engine_add_input(&en_sync_to_sb_addr_set, &en_northd, NULL); + engine_add_input(&en_sync_to_sb_addr_set, &en_lr_stateful, NULL); engine_add_input(&en_sync_to_sb_addr_set, &en_sb_address_set, NULL); engine_add_input(&en_port_group, &en_nb_port_group, @@ -240,7 +249,7 @@ void inc_proc_northd_init(struct ovsdb_idl_loop *nb, engine_add_input(&en_sync_to_sb_pb, &en_northd, sync_to_sb_pb_northd_handler); - engine_add_input(&en_sync_to_sb_pb, &en_lr_nat, NULL); + engine_add_input(&en_sync_to_sb_pb, &en_lr_stateful, NULL); /* en_sync_to_sb engine node syncs the SB database tables from * the NB database tables. diff --git a/northd/northd.c b/northd/northd.c index 7f59415bbd..e9b8332e21 100644 --- a/northd/northd.c +++ b/northd/northd.c @@ -44,6 +44,7 @@ #include "northd.h" #include "en-lb-data.h" #include "en-lr-nat.h" +#include "en-lr-stateful.h" #include "lib/ovn-parallel-hmap.h" #include "ovn/actions.h" #include "ovn/features.h" @@ -617,13 +618,6 @@ init_lb_for_datapath(struct ovn_datapath *od) } } -static void -destroy_lb_for_datapath(struct ovn_datapath *od) -{ - ovn_lb_ip_set_destroy(od->lb_ips); - od->lb_ips = NULL; -} - /* A group of logical router datapaths which are connected - either * directly or indirectly. * Each logical router can belong to only one group. */ @@ -676,7 +670,6 @@ ovn_datapath_destroy(struct hmap *datapaths, struct ovn_datapath *od) destroy_ipam_info(&od->ipam_info); free(od->router_ports); free(od->ls_peers); - destroy_lb_for_datapath(od); free(od->localnet_ports); free(od->l3dgw_ports); destroy_mcast_info_for_datapath(od); @@ -1311,121 +1304,6 @@ struct lflow_ref_node { struct ovn_lflow *lflow; }; -/* A logical switch port or logical router port. - * - * In steady state, an ovn_port points to a northbound Logical_Switch_Port - * record (via 'nbsp') *or* a Logical_Router_Port record (via 'nbrp'), and to a - * southbound Port_Binding record (via 'sb'). As the state of the system - * changes, join_logical_ports() may determine that there is a new LSP or LRP - * that has no corresponding Port_Binding record (in which case build_ports()) - * will create the missing Port_Binding) or that a Port_Binding record exists - * that has no coresponding LSP (in which case build_ports() will delete the - * spurious Port_Binding). Thus, after build_ports() runs, any given ovn_port - * will have 'sb' nonnull, and 'nbsp' xor 'nbrp' nonnull. - * - * Ordinarily there is only one ovn_port that points to a given LSP or LRP (but - * distributed gateway ports point a "derived" ovn_port to a duplicate LRP). - */ -struct ovn_port { - /* Port name aka key. - * - * This is ordinarily the same as nbsp->name or nbrp->name and - * sb->logical_port. (A distributed gateway port creates a "derived" - * ovn_port with key "cr-%s" % nbrp->name.) */ - struct hmap_node key_node; /* Index on 'key'. */ - char *key; /* nbsp->name, nbrp->name, sb->logical_port. */ - char *json_key; /* 'key', quoted for use in JSON. */ - - const struct sbrec_port_binding *sb; /* May be NULL. */ - - uint32_t tunnel_key; - - /* Logical switch port data. */ - const struct nbrec_logical_switch_port *nbsp; /* May be NULL. */ - - struct lport_addresses *lsp_addrs; /* Logical switch port addresses. */ - unsigned int n_lsp_addrs; /* Total length of lsp_addrs. */ - unsigned int n_lsp_non_router_addrs; /* Number of elements from the - * beginning of 'lsp_addrs' extracted - * directly from LSP 'addresses'. */ - - struct lport_addresses *ps_addrs; /* Port security addresses. */ - unsigned int n_ps_addrs; - - bool lsp_can_be_inc_processed; /* If it can be incrementally processed when - the port changes. */ - - /* Logical router port data. */ - const struct nbrec_logical_router_port *nbrp; /* May be NULL. */ - - struct lport_addresses lrp_networks; - - struct ovn_port_routable_addresses routables; - - /* Logical port multicast data. */ - struct mcast_port_info mcast_info; - - /* At most one of l3dgw_port and cr_port can be not NULL. */ - - /* This is set to a distributed gateway port if and only if this ovn_port - * is "derived" from it. Otherwise this is set to NULL. The derived - * ovn_port represents the instance of distributed gateway port on the - * gateway chassis.*/ - struct ovn_port *l3dgw_port; - - /* This is set to the "derived" chassis-redirect port of this port if and - * only if this port is a distributed gateway port. Otherwise this is set - * to NULL. */ - struct ovn_port *cr_port; - - bool has_unknown; /* If the addresses have 'unknown' defined. */ - - bool has_bfd; - - /* The port's peer: - * - * - A switch port S of type "router" has a router port R as a peer, - * and R in turn has S has its peer. - * - * - Two connected logical router ports have each other as peer. - * - * - Other kinds of ports have no peer. */ - struct ovn_port *peer; - - struct ovn_datapath *od; - - struct ovs_list list; /* In list of similar records. */ - - struct hmap_node dp_node; /* Node in od->ports. */ - - struct lport_addresses proxy_arp_addrs; - - /* Temporarily used for traversing a list (or hmap) of ports. */ - bool visited; - - /* List of struct lflow_ref_node that points to the lflows generated by - * this ovn_port. - * - * This data is initialized and destroyed by the en_northd node, but - * populated and used only by the en_lflow node. Ideally this data should - * be maintained as part of en_lflow's data (struct lflow_data): a hash - * index from ovn_port key to lflows. However, it would be less efficient - * and more complex: - * - * 1. It would require an extra search (using the index) to find the - * lflows. - * - * 2. Building the index needs to be thread-safe, using either a global - * lock which is obviously less efficient, or hash-based lock array which - * is more complex. - * - * Adding the list here is more straightforward. The drawback is that we - * need to keep in mind that this data belongs to en_lflow node, so never - * access it from any other nodes. - */ - struct ovs_list lflows; -}; - static bool lsp_can_be_inc_processed(const struct nbrec_logical_switch_port *); static bool @@ -1450,16 +1328,21 @@ destroy_routable_addresses(struct ovn_port_routable_addresses *ra) } static char **get_nat_addresses(const struct ovn_port *op, size_t *n, - bool routable_only, bool include_lb_ips); + bool routable_only, bool include_lb_ips, + const struct lr_stateful_record *); -static void -assign_routable_addresses(struct ovn_port *op) +static struct ovn_port_routable_addresses +get_op_routable_addresses(struct ovn_port *op, + const struct lr_stateful_record *lr_sful_rec) { size_t n; - char **nats = get_nat_addresses(op, &n, true, true); + char **nats = get_nat_addresses(op, &n, true, true, lr_sful_rec); if (!nats) { - return; + return (struct ovn_port_routable_addresses) { + .laddrs = NULL, + .n_addrs = 0, + }; } struct lport_addresses *laddrs = xcalloc(n, sizeof(*laddrs)); @@ -1475,9 +1358,15 @@ assign_routable_addresses(struct ovn_port *op) } free(nats); - /* Everything seems to have worked out */ - op->routables.laddrs = laddrs; - op->routables.n_addrs = n_addrs; + if (!n_addrs) { + free(laddrs); + laddrs = NULL; + } + + return (struct ovn_port_routable_addresses) { + .laddrs = laddrs, + .n_addrs = n_addrs, + }; } @@ -1537,8 +1426,6 @@ ovn_port_destroy_orphan(struct ovn_port *port) } free(port->ps_addrs); - destroy_routable_addresses(&port->routables); - destroy_lport_addresses(&port->lrp_networks); destroy_lport_addresses(&port->proxy_arp_addrs); free(port->json_key); @@ -2580,9 +2467,7 @@ join_logical_ports(const struct sbrec_port_binding_table *sbrec_pb_table, sizeof *od->l3dgw_ports); } od->l3dgw_ports[od->n_l3dgw_ports++] = op; - - assign_routable_addresses(op); - } + } } } @@ -2679,7 +2564,8 @@ join_logical_ports(const struct sbrec_port_binding_table *sbrec_pb_table, * and must free the returned array when it is no longer needed. */ static char ** get_nat_addresses(const struct ovn_port *op, size_t *n, bool routable_only, - bool include_lb_ips) + bool include_lb_ips, + const struct lr_stateful_record *lr_sful_rec) { size_t n_nats = 0; struct eth_addr mac; @@ -2764,23 +2650,25 @@ get_nat_addresses(const struct ovn_port *op, size_t *n, bool routable_only, } } - if (include_lb_ips) { + if (include_lb_ips && lr_sful_rec) { const char *ip_address; if (routable_only) { - SSET_FOR_EACH (ip_address, &op->od->lb_ips->ips_v4_routable) { + SSET_FOR_EACH (ip_address, + &lr_sful_rec->lb_ips->ips_v4_routable) { ds_put_format(&c_addresses, " %s", ip_address); central_ip_address = true; } - SSET_FOR_EACH (ip_address, &op->od->lb_ips->ips_v6_routable) { + SSET_FOR_EACH (ip_address, + &lr_sful_rec->lb_ips->ips_v6_routable) { ds_put_format(&c_addresses, " %s", ip_address); central_ip_address = true; } } else { - SSET_FOR_EACH (ip_address, &op->od->lb_ips->ips_v4) { + SSET_FOR_EACH (ip_address, &lr_sful_rec->lb_ips->ips_v4) { ds_put_format(&c_addresses, " %s", ip_address); central_ip_address = true; } - SSET_FOR_EACH (ip_address, &op->od->lb_ips->ips_v6) { + SSET_FOR_EACH (ip_address, &lr_sful_rec->lb_ips->ips_v6) { ds_put_format(&c_addresses, " %s", ip_address); central_ip_address = true; } @@ -3851,21 +3739,8 @@ build_lb_datapaths(const struct hmap *lbs, const struct hmap *lb_groups, HMAP_FOR_EACH (od, key_node, &lr_datapaths->datapaths) { ovs_assert(od->nbr); - /* Checking load balancer groups first, starting from the largest one, - * to more efficiently copy IP sets. */ - size_t largest_group = 0; - - for (size_t i = 1; i < od->nbr->n_load_balancer_group; i++) { - if (od->nbr->load_balancer_group[i]->n_load_balancer > - od->nbr->load_balancer_group[largest_group]->n_load_balancer) { - largest_group = i; - } - } - for (size_t i = 0; i < od->nbr->n_load_balancer_group; i++) { - size_t idx = (i + largest_group) % od->nbr->n_load_balancer_group; - - nbrec_lb_group = od->nbr->load_balancer_group[idx]; + nbrec_lb_group = od->nbr->load_balancer_group[i]; const struct uuid *lb_group_uuid = &nbrec_lb_group->header_.uuid; lb_group_dps = @@ -3873,20 +3748,6 @@ build_lb_datapaths(const struct hmap *lbs, const struct hmap *lb_groups, lb_group_uuid); ovs_assert(lb_group_dps); ovn_lb_group_datapaths_add_lr(lb_group_dps, od); - - if (!od->lb_ips) { - od->lb_ips = - ovn_lb_ip_set_clone(lb_group_dps->lb_group->lb_ips); - } else { - for (size_t j = 0; j < lb_group_dps->lb_group->n_lbs; j++) { - build_lrouter_lb_ips(od->lb_ips, - lb_group_dps->lb_group->lbs[j]); - } - } - } - - if (!od->lb_ips) { - od->lb_ips = ovn_lb_ip_set_create(); } for (size_t i = 0; i < od->nbr->n_load_balancer; i++) { @@ -3895,7 +3756,6 @@ build_lb_datapaths(const struct hmap *lbs, const struct hmap *lb_groups, lb_dps = ovn_lb_datapaths_find(lb_datapaths_map, lb_uuid); ovs_assert(lb_dps); ovn_lb_datapaths_add_lr(lb_dps, 1, &od); - build_lrouter_lb_ips(od->lb_ips, lb_dps->lb); } } @@ -3949,102 +3809,6 @@ build_lb_svcs( } } -static bool lrouter_port_ipv4_reachable(const struct ovn_port *op, - ovs_be32 addr); -static bool lrouter_port_ipv6_reachable(const struct ovn_port *op, - const struct in6_addr *addr); - -static void -add_neigh_ips_to_lrouter(struct ovn_datapath *od, - enum lb_neighbor_responder_mode neigh_mode, - const struct sset *lb_ips_v4, - const struct sset *lb_ips_v6) -{ - /* If configured to not reply to any neighbor requests for all VIPs - * return early. - */ - if (neigh_mode == LB_NEIGH_RESPOND_NONE) { - return; - } - - const char *ip_address; - - /* If configured to reply to neighbor requests for all VIPs force them - * all to be considered "reachable". - */ - if (neigh_mode == LB_NEIGH_RESPOND_ALL) { - SSET_FOR_EACH (ip_address, lb_ips_v4) { - sset_add(&od->lb_ips->ips_v4_reachable, ip_address); - } - SSET_FOR_EACH (ip_address, lb_ips_v6) { - sset_add(&od->lb_ips->ips_v6_reachable, ip_address); - } - - return; - } - - /* Otherwise, a VIP is reachable if there's at least one router - * subnet that includes it. - */ - ovs_assert(neigh_mode == LB_NEIGH_RESPOND_REACHABLE); - - SSET_FOR_EACH (ip_address, lb_ips_v4) { - struct ovn_port *op; - ovs_be32 vip_ip4; - if (ip_parse(ip_address, &vip_ip4)) { - HMAP_FOR_EACH (op, dp_node, &od->ports) { - if (lrouter_port_ipv4_reachable(op, vip_ip4)) { - sset_add(&od->lb_ips->ips_v4_reachable, - ip_address); - break; - } - } - } - } - - SSET_FOR_EACH (ip_address, lb_ips_v6) { - struct ovn_port *op; - struct in6_addr vip; - if (ipv6_parse(ip_address, &vip)) { - HMAP_FOR_EACH (op, dp_node, &od->ports) { - if (lrouter_port_ipv6_reachable(op, &vip)) { - sset_add(&od->lb_ips->ips_v6_reachable, - ip_address); - break; - } - } - } - } -} - -static void -remove_lrouter_lb_reachable_ips(struct ovn_datapath *od, - enum lb_neighbor_responder_mode neigh_mode, - const struct sset *lb_ips_v4, - const struct sset *lb_ips_v6) -{ - if (neigh_mode == LB_NEIGH_RESPOND_NONE) { - return; - } - - const char *ip_address; - SSET_FOR_EACH (ip_address, lb_ips_v4) { - sset_find_and_delete(&od->lb_ips->ips_v4_reachable, ip_address); - } - SSET_FOR_EACH (ip_address, lb_ips_v6) { - sset_find_and_delete(&od->lb_ips->ips_v6_reachable, ip_address); - } -} - -static void -build_lrouter_lb_reachable_ips(struct ovn_datapath *od, - const struct ovn_northd_lb *lb) -{ - add_neigh_ips_to_lrouter(od, lb->neigh_mode, &lb->ips_v4, - &lb->ips_v6); -} - - static void build_lrouter_lbs_check(const struct ovn_datapaths *lr_datapaths) { @@ -4066,43 +3830,6 @@ build_lrouter_lbs_check(const struct ovn_datapaths *lr_datapaths) } } -static void -build_lrouter_lbs_reachable_ips(struct ovn_datapaths *lr_datapaths, - struct hmap *lb_dps_map, - struct hmap *lb_group_dps_map) -{ - struct ovn_datapath *od; - - HMAP_FOR_EACH (od, key_node, &lr_datapaths->datapaths) { - if (!od->nbr) { - continue; - } - - for (size_t i = 0; i < od->nbr->n_load_balancer; i++) { - struct ovn_lb_datapaths *lb_dps = - ovn_lb_datapaths_find(lb_dps_map, - &od->nbr->load_balancer[i]->header_.uuid); - ovs_assert(lb_dps); - build_lrouter_lb_reachable_ips(od, lb_dps->lb); - } - - for (size_t i = 0; i < od->nbr->n_load_balancer_group; i++) { - const struct nbrec_load_balancer_group *nbrec_lb_group = - od->nbr->load_balancer_group[i]; - struct ovn_lb_group_datapaths *lb_group_dps; - - lb_group_dps = - ovn_lb_group_datapaths_find(lb_group_dps_map, - &nbrec_lb_group->header_.uuid); - ovs_assert(lb_group_dps); - for (size_t j = 0; j < lb_group_dps->lb_group->n_lbs; j++) { - build_lrouter_lb_reachable_ips(od, - lb_group_dps->lb_group->lbs[j]); - } - } - } -} - static void build_lswitch_lbs_from_lrouter(struct ovn_datapaths *lr_datapaths, struct hmap *lb_dps_map, @@ -4166,8 +3893,6 @@ build_lb_port_related_data( struct hmap *svc_monitor_map) { build_lrouter_lbs_check(lr_datapaths); - build_lrouter_lbs_reachable_ips(lr_datapaths, lb_dps_map, - lb_group_dps_map); build_lb_svcs(ovnsb_txn, sbrec_service_monitor_table, ls_ports, lb_dps_map, svc_monitor_lsps, svc_monitor_map); build_lswitch_lbs_from_lrouter(lr_datapaths, lb_dps_map, lb_group_dps_map); @@ -4533,7 +4258,8 @@ check_sb_lb_duplicates(const struct sbrec_load_balancer_table *table) * Caller should make sure that the OVN SB IDL txn is not NULL. Presently it * only syncs the nat column of port binding corresponding to the 'op->nbsp' */ static void -sync_pb_for_lsp(struct ovn_port *op) +sync_pb_for_lsp(struct ovn_port *op, + const struct lr_stateful_table *lr_sful_table) { ovs_assert(op->nbsp); @@ -4552,10 +4278,17 @@ sync_pb_for_lsp(struct ovn_port *op) if (nat_addresses && !strcmp(nat_addresses, "router")) { if (op->peer && op->peer->od && (chassis || op->peer->od->n_l3dgw_ports)) { - bool exclude_lb_vips = smap_get_bool(&op->nbsp->options, + bool include_lb_vips = !smap_get_bool(&op->nbsp->options, "exclude-lb-vips-from-garp", false); + + const struct lr_stateful_record *lr_sful_rec = NULL; + + if (include_lb_vips) { + lr_sful_rec = lr_stateful_table_find_by_index( + lr_sful_table, op->peer->od->index); + } nats = get_nat_addresses(op->peer, &n_nats, false, - !exclude_lb_vips); + include_lb_vips, lr_sful_rec); } } else if (nat_addresses && (chassis || l3dgw_ports)) { struct lport_addresses laddrs; @@ -4662,7 +4395,8 @@ sync_pb_for_lsp(struct ovn_port *op) * Caller should make sure that the OVN SB IDL txn is not NULL. Presently it * only sets the port binding options column for the router ports */ static void -sync_pb_for_lrp(struct ovn_port *op, const struct lr_nat_table *lr_nats) +sync_pb_for_lrp(struct ovn_port *op, + const struct lr_stateful_table *lr_sful_table) { ovs_assert(op->nbrp); @@ -4671,14 +4405,14 @@ sync_pb_for_lrp(struct ovn_port *op, const struct lr_nat_table *lr_nats) const char *chassis_name = smap_get(&op->od->nbr->options, "chassis"); if (is_cr_port(op)) { - const struct lr_nat_record *lrnat_rec = - lr_nat_table_find_by_index(lr_nats, op->od->index); - ovs_assert(lrnat_rec); + const struct lr_stateful_record *lr_sful_rec = + lr_stateful_table_find_by_index(lr_sful_table, op->od->index); + ovs_assert(lr_sful_rec); smap_add(&new, "distributed-port", op->nbrp->name); bool always_redirect = - !lrnat_rec->has_distributed_nat && + !lr_sful_rec->lrnat_rec->has_distributed_nat && !l3dgw_port_has_associated_vtep_lports(op->l3dgw_port); const char *redirect_type = smap_get(&op->nbrp->options, @@ -4728,17 +4462,18 @@ static void ovn_update_ipv6_opt_for_op(struct ovn_port *op); * the logical switch ports. */ void sync_pbs(struct ovsdb_idl_txn *ovnsb_idl_txn, struct hmap *ls_ports, - struct hmap *lr_ports, const struct lr_nat_table *lr_nats) + struct hmap *lr_ports, + const struct lr_stateful_table *lr_sful_table) { ovs_assert(ovnsb_idl_txn); struct ovn_port *op; HMAP_FOR_EACH (op, key_node, ls_ports) { - sync_pb_for_lsp(op); + sync_pb_for_lsp(op, lr_sful_table); } HMAP_FOR_EACH (op, key_node, lr_ports) { - sync_pb_for_lrp(op, lr_nats); + sync_pb_for_lrp(op, lr_sful_table); } ovn_update_ipv6_options(lr_ports); @@ -4747,20 +4482,22 @@ sync_pbs(struct ovsdb_idl_txn *ovnsb_idl_txn, struct hmap *ls_ports, /* Sync the SB Port bindings for the added and updated logical switch ports * of the tracked northd engine data. */ bool -sync_pbs_for_northd_changed_ovn_ports(struct tracked_ovn_ports *trk_ovn_ports) +sync_pbs_for_northd_changed_ovn_ports( + struct tracked_ovn_ports *trk_ovn_ports, + const struct lr_stateful_table *lr_sful_table) { struct hmapx_node *hmapx_node; struct ovn_port *op; HMAPX_FOR_EACH (hmapx_node, &trk_ovn_ports->created) { op = hmapx_node->data; ovs_assert(op->nbsp); - sync_pb_for_lsp(op); + sync_pb_for_lsp(op, lr_sful_table); } HMAPX_FOR_EACH (hmapx_node, &trk_ovn_ports->updated) { op = hmapx_node->data; ovs_assert(op->nbsp); - sync_pb_for_lsp(op); + sync_pb_for_lsp(op, lr_sful_table); } return true; @@ -5425,14 +5162,13 @@ fail: } /* Returns true if the logical router has changes which can be - * incrementally handled. + * incrementally handled or the changes can be ignored. * Presently supports i-p for the below changes: * - load balancers and load balancer groups. * - NAT changes */ static bool -lr_changes_can_be_handled( - const struct nbrec_logical_router *lr) +lr_changes_can_be_handled(const struct nbrec_logical_router *lr) { /* Check if the columns are changed in this row. */ enum nbrec_logical_router_column_id col; @@ -5496,17 +5232,6 @@ is_lr_nats_changed(const struct nbrec_logical_router *nbr) { || is_lr_nats_seqno_changed(nbr)); } -static bool -lr_has_routable_nats(const struct nbrec_logical_router *nbr) { - for (size_t i = 0; i < nbr->n_nat; i++) { - if (smap_get_bool(&nbr->nat[i]->options, "add_route", false)) { - return true; - } - } - - return false; -} - /* Return true if changes are handled incrementally, false otherwise. * * Note: Changes to load balancer and load balancer groups associated with @@ -5533,12 +5258,6 @@ northd_handle_lr_changes(const struct northd_input *ni, } if (is_lr_nats_changed(changed_lr)) { - if (lr_has_routable_nats(changed_lr)) { - /* router has routable NATs. We can't handle these changes - * incrementally yet. Fall back to recompute. */ - goto fail; - } - struct ovn_datapath *od = ovn_datapath_find_( &nd->lr_datapaths.datapaths, &changed_lr->header_.uuid); @@ -5820,10 +5539,6 @@ northd_handle_lb_data_changes(struct tracked_lb_data *trk_lb_data, ovs_assert(lb_dps); ovn_lb_datapaths_add_lr(lb_dps, 1, &od); - /* Add the lb_ips of lb_dps to the od. */ - build_lrouter_lb_ips(od->lb_ips, lb_dps->lb); - build_lrouter_lb_reachable_ips(od, lb_dps->lb); - /* Add the lb to the northd tracked data. */ hmapx_add(&nd_changes->trk_lbs.crupdated, lb_dps); } @@ -5842,10 +5557,6 @@ northd_handle_lb_data_changes(struct tracked_lb_data *trk_lb_data, ovs_assert(lb_dps); ovn_lb_datapaths_add_lr(lb_dps, 1, &od); - /* Add the lb_ips of lb_dps to the od. */ - build_lrouter_lb_ips(od->lb_ips, lb_dps->lb); - build_lrouter_lb_reachable_ips(od, lb_dps->lb); - /* Add the lb to the northd tracked data. */ hmapx_add(&nd_changes->trk_lbs.crupdated, lb_dps); } @@ -5874,22 +5585,6 @@ northd_handle_lb_data_changes(struct tracked_lb_data *trk_lb_data, od = lr_datapaths->array[index]; /* Re-evaluate 'od->has_lb_vip' */ init_lb_for_datapath(od); - - /* Update the od->lb_ips with the deleted and inserted - * vips (if any). */ - remove_ips_from_lb_ip_set(od->lb_ips, lb->routable, - &clb->deleted_vips_v4, - &clb->deleted_vips_v6); - add_ips_to_lb_ip_set(od->lb_ips, lb->routable, - &clb->inserted_vips_v4, - &clb->inserted_vips_v6); - - remove_lrouter_lb_reachable_ips(od, lb->neigh_mode, - &clb->deleted_vips_v4, - &clb->deleted_vips_v6); - add_neigh_ips_to_lrouter(od, lb->neigh_mode, - &clb->inserted_vips_v4, - &clb->inserted_vips_v6); } } @@ -5914,9 +5609,6 @@ northd_handle_lb_data_changes(struct tracked_lb_data *trk_lb_data, /* Re-evaluate 'od->has_lb_vip' */ init_lb_for_datapath(od); - - /* Add the lb_ips of lb_dps to the od. */ - build_lrouter_lb_ips(od->lb_ips, lb_dps->lb); } for (size_t i = 0; i < lbgrp_dps->n_ls; i++) { @@ -9210,7 +8902,7 @@ arp_nd_ns_match(const char *ips, int addr_family, struct ds *match) /* Returns 'true' if the IPv4 'addr' is on the same subnet with one of the * IPs configured on the router port. */ -static bool +bool lrouter_port_ipv4_reachable(const struct ovn_port *op, ovs_be32 addr) { for (size_t i = 0; i < op->lrp_networks.n_ipv4_addrs; i++) { @@ -9226,7 +8918,7 @@ lrouter_port_ipv4_reachable(const struct ovn_port *op, ovs_be32 addr) /* Returns 'true' if the IPv6 'addr' is on the same subnet with one of the * IPs configured on the router port. */ -static bool +bool lrouter_port_ipv6_reachable(const struct ovn_port *op, const struct in6_addr *addr) { @@ -9289,11 +8981,12 @@ build_lswitch_rport_arp_req_flow(const char *ips, */ static void build_lswitch_rport_arp_req_flows(struct ovn_port *op, - struct ovn_datapath *sw_od, - struct ovn_port *sw_op, - const struct lr_nat_table *lr_nats, - struct hmap *lflows, - const struct ovsdb_idl_row *stage_hint) + struct ovn_datapath *sw_od, + struct ovn_port *sw_op, + const struct lr_nat_table *lr_nats, + const struct lr_stateful_table *lr_sful_table, + struct hmap *lflows, + const struct ovsdb_idl_row *stage_hint) { if (!op || !op->nbrp) { return; @@ -9307,32 +9000,38 @@ build_lswitch_rport_arp_req_flows(struct ovn_port *op, * router port. * Priority: 80. */ - - const char *ip_addr; - SSET_FOR_EACH (ip_addr, &op->od->lb_ips->ips_v4_reachable) { - ovs_be32 ipv4_addr; - - /* Check if the ovn port has a network configured on which we could - * expect ARP requests for the LB VIP. - */ - if (ip_parse(ip_addr, &ipv4_addr) && - lrouter_port_ipv4_reachable(op, ipv4_addr)) { - build_lswitch_rport_arp_req_flow( - ip_addr, AF_INET, sw_op, sw_od, 80, lflows, - stage_hint); + const struct lr_stateful_record *lr_sful_rec = NULL; + if (op->od->nbr->n_load_balancer || op->od->nbr->n_load_balancer_group) { + lr_sful_rec = lr_stateful_table_find_by_index(lr_sful_table, + op->od->index); + ovs_assert(lr_sful_rec); + + const char *ip_addr; + SSET_FOR_EACH (ip_addr, &lr_sful_rec->lb_ips->ips_v4_reachable) { + ovs_be32 ipv4_addr; + + /* Check if the ovn port has a network configured on which we could + * expect ARP requests for the LB VIP. + */ + if (ip_parse(ip_addr, &ipv4_addr) && + lrouter_port_ipv4_reachable(op, ipv4_addr)) { + build_lswitch_rport_arp_req_flow( + ip_addr, AF_INET, sw_op, sw_od, 80, lflows, + stage_hint); + } } - } - SSET_FOR_EACH (ip_addr, &op->od->lb_ips->ips_v6_reachable) { - struct in6_addr ipv6_addr; + SSET_FOR_EACH (ip_addr, &lr_sful_rec->lb_ips->ips_v6_reachable) { + struct in6_addr ipv6_addr; - /* Check if the ovn port has a network configured on which we could - * expect NS requests for the LB VIP. - */ - if (ipv6_parse(ip_addr, &ipv6_addr) && - lrouter_port_ipv6_reachable(op, &ipv6_addr)) { - build_lswitch_rport_arp_req_flow( - ip_addr, AF_INET6, sw_op, sw_od, 80, lflows, - stage_hint); + /* Check if the ovn port has a network configured on which we could + * expect NS requests for the LB VIP. + */ + if (ipv6_parse(ip_addr, &ipv6_addr) && + lrouter_port_ipv6_reachable(op, &ipv6_addr)) { + build_lswitch_rport_arp_req_flow( + ip_addr, AF_INET6, sw_op, sw_od, 80, lflows, + stage_hint); + } } } @@ -9382,13 +9081,15 @@ build_lswitch_rport_arp_req_flows(struct ovn_port *op, * expect ARP requests/NS for the DNAT external_ip. */ if (nat_entry_is_v6(nat_entry)) { - if (!sset_contains(&op->od->lb_ips->ips_v6, nat->external_ip)) { + if (!lr_sful_rec || !sset_contains(&lr_sful_rec->lb_ips->ips_v6, + nat->external_ip)) { build_lswitch_rport_arp_req_flow( nat->external_ip, AF_INET6, sw_op, sw_od, 80, lflows, stage_hint); } } else { - if (!sset_contains(&op->od->lb_ips->ips_v4, nat->external_ip)) { + if (!lr_sful_rec || !sset_contains(&lr_sful_rec->lb_ips->ips_v4, + nat->external_ip)) { build_lswitch_rport_arp_req_flow( nat->external_ip, AF_INET, sw_op, sw_od, 80, lflows, stage_hint); @@ -10429,6 +10130,7 @@ build_lswitch_ip_mcast_igmp_mld(struct ovn_igmp_group *igmp_group, static void build_lswitch_ip_unicast_lookup(struct ovn_port *op, const struct lr_nat_table *lr_nats, + const struct lr_stateful_table *lr_sful_table, struct hmap *lflows, struct ds *actions, struct ds *match) @@ -10444,7 +10146,8 @@ build_lswitch_ip_unicast_lookup(struct ovn_port *op, */ if (lsp_is_router(op->nbsp)) { build_lswitch_rport_arp_req_flows(op->peer, op->od, op, lr_nats, - lflows, &op->nbsp->header_); + lr_sful_table, lflows, + &op->nbsp->header_); } for (size_t i = 0; i < op->nbsp->n_addresses; i++) { @@ -12634,6 +12337,7 @@ build_lrouter_port_nat_arp_nd_flow(struct ovn_port *op, static void build_lrouter_drop_own_dest(struct ovn_port *op, const struct lr_nat_record *lrnat_rec, + const struct lr_stateful_record *lr_sful_rec, enum ovn_stage stage, uint16_t priority, bool drop_snat_ip, struct hmap *lflows) @@ -12646,8 +12350,9 @@ build_lrouter_drop_own_dest(struct ovn_port *op, bool router_ip_in_snat_ips = !!shash_find(&lrnat_rec->snat_ips, ip); - bool router_ip_in_lb_ips = - !!sset_find(&op->od->lb_ips->ips_v4, ip); + bool router_ip_in_lb_ips = (lr_sful_rec && + !!sset_find(&lr_sful_rec->lb_ips->ips_v4, + ip)); bool drop_router_ip = (drop_snat_ip == (router_ip_in_snat_ips || router_ip_in_lb_ips)); @@ -12676,8 +12381,9 @@ build_lrouter_drop_own_dest(struct ovn_port *op, bool router_ip_in_snat_ips = !!shash_find(&lrnat_rec->snat_ips, ip); - bool router_ip_in_lb_ips = - !!sset_find(&op->od->lb_ips->ips_v6, ip); + bool router_ip_in_lb_ips = (lr_sful_rec && + !!sset_find(&lr_sful_rec->lb_ips->ips_v6, + ip)); bool drop_router_ip = (drop_snat_ip == (router_ip_in_snat_ips || router_ip_in_lb_ips)); @@ -13389,7 +13095,8 @@ build_ip_routing_flows_for_lrp( */ static void build_ip_routing_flows_for_router_type_lsp( - struct ovn_port *op, const struct hmap *lr_ports, struct hmap *lflows) + struct ovn_port *op, const struct lr_stateful_table *lr_sful_table, + const struct hmap *lr_ports, struct hmap *lflows) { ovs_assert(op->nbsp); if (!lsp_is_router(op->nbsp)) { @@ -13397,7 +13104,8 @@ build_ip_routing_flows_for_router_type_lsp( } struct ovn_port *peer = ovn_port_get_peer(lr_ports, op); - if (!peer || !peer->nbrp || !peer->lrp_networks.n_ipv4_addrs) { + if (!peer || !peer->nbrp || !peer->lrp_networks.n_ipv4_addrs + || !op->od->n_router_ports) { return; } @@ -13408,19 +13116,29 @@ build_ip_routing_flows_for_router_type_lsp( continue; } - struct ovn_port_routable_addresses *ra = &router_port->routables; - for (size_t j = 0; j < ra->n_addrs; j++) { - struct lport_addresses *laddrs = &ra->laddrs[j]; - for (size_t k = 0; k < laddrs->n_ipv4_addrs; k++) { - add_route(lflows, peer->od, peer, - peer->lrp_networks.ipv4_addrs[0].addr_s, - laddrs->ipv4_addrs[k].network_s, - laddrs->ipv4_addrs[k].plen, NULL, false, 0, - &peer->nbrp->header_, false, - ROUTE_PRIO_OFFSET_CONNECTED); + const struct lr_stateful_record *lr_sful_rec = + lr_stateful_table_find_by_index(lr_sful_table, + router_port->od->index); + + if (router_port->nbrp->ha_chassis_group || + router_port->nbrp->n_gateway_chassis) { + struct ovn_port_routable_addresses ra = + get_op_routable_addresses(router_port, lr_sful_rec); + for (size_t j = 0; j < ra.n_addrs; j++) { + struct lport_addresses *laddrs = &ra.laddrs[j]; + for (size_t k = 0; k < laddrs->n_ipv4_addrs; k++) { + add_route(lflows, peer->od, peer, + peer->lrp_networks.ipv4_addrs[0].addr_s, + laddrs->ipv4_addrs[k].network_s, + laddrs->ipv4_addrs[k].plen, NULL, false, 0, + &peer->nbrp->header_, false, + ROUTE_PRIO_OFFSET_CONNECTED); + } } + destroy_routable_addresses(&ra); } } + } static void @@ -13644,33 +13362,36 @@ build_arp_resolve_flows_for_lrouter( static void routable_addresses_to_lflows(struct hmap *lflows, struct ovn_port *router_port, - struct ovn_port *peer, struct ds *match, - struct ds *actions) + struct ovn_port *peer, + const struct lr_stateful_record *lr_sful_rec, + struct ds *match, struct ds *actions) { - struct ovn_port_routable_addresses *ra = &router_port->routables; - if (!ra->n_addrs) { + struct ovn_port_routable_addresses ra = + get_op_routable_addresses(router_port, lr_sful_rec); + if (!ra.n_addrs) { return; } - for (size_t i = 0; i < ra->n_addrs; i++) { + for (size_t i = 0; i < ra.n_addrs; i++) { ds_clear(match); ds_put_format(match, "outport == %s && "REG_NEXT_HOP_IPV4" == {", peer->json_key); bool first = true; - for (size_t j = 0; j < ra->laddrs[i].n_ipv4_addrs; j++) { + for (size_t j = 0; j < ra.laddrs[i].n_ipv4_addrs; j++) { if (!first) { ds_put_cstr(match, ", "); } - ds_put_cstr(match, ra->laddrs[i].ipv4_addrs[j].addr_s); + ds_put_cstr(match, ra.laddrs[i].ipv4_addrs[j].addr_s); first = false; } ds_put_cstr(match, "}"); ds_clear(actions); - ds_put_format(actions, "eth.dst = %s; next;", ra->laddrs[i].ea_s); + ds_put_format(actions, "eth.dst = %s; next;", ra.laddrs[i].ea_s); ovn_lflow_add(lflows, peer->od, S_ROUTER_IN_ARP_RESOLVE, 100, ds_cstr(match), ds_cstr(actions)); } + destroy_routable_addresses(&ra); } /* Local router ingress table ARP_RESOLVE: ARP Resolution. @@ -13687,6 +13408,7 @@ routable_addresses_to_lflows(struct hmap *lflows, struct ovn_port *router_port, static void build_arp_resolve_flows_for_lrp( struct ovn_port *op, const struct lr_nat_record *lrnat_rec, + const struct lr_stateful_record *lr_sful_rec, struct hmap *lflows, struct ds *match, struct ds *actions) { ovs_assert(op->nbrp); @@ -13763,8 +13485,8 @@ build_arp_resolve_flows_for_lrp( * * Priority 2. */ - build_lrouter_drop_own_dest(op, lrnat_rec, S_ROUTER_IN_ARP_RESOLVE, 2, - true, lflows); + build_lrouter_drop_own_dest(op, lrnat_rec, lr_sful_rec, + S_ROUTER_IN_ARP_RESOLVE, 2, true, lflows); } /* This function adds ARP resolve flows related to a LSP. */ @@ -13772,6 +13494,7 @@ static void build_arp_resolve_flows_for_lsp( struct ovn_port *op, struct hmap *lflows, const struct hmap *lr_ports, + const struct lr_stateful_table *lr_sful_table, struct ds *match, struct ds *actions) { ovs_assert(op->nbsp); @@ -13915,8 +13638,11 @@ build_arp_resolve_flows_for_lsp( if (smap_get(&peer->od->nbr->options, "chassis") || peer->cr_port) { + const struct lr_stateful_record *lr_sful_rec; + lr_sful_rec = lr_stateful_table_find_by_index(lr_sful_table, + router_port->od->index); routable_addresses_to_lflows(lflows, router_port, peer, - match, actions); + lr_sful_rec, match, actions); } } } @@ -14636,6 +14362,7 @@ static void build_lrouter_ipv4_ip_input(struct ovn_port *op, struct hmap *lflows, const struct lr_nat_record *lrnat_rec, + const struct lr_stateful_record *lr_sful_rec, struct ds *match, struct ds *actions, const struct shash *meter_groups) { @@ -14760,7 +14487,7 @@ build_lrouter_ipv4_ip_input(struct ovn_port *op, &op->nbrp->header_, lflows); } - if (sset_count(&op->od->lb_ips->ips_v4_reachable)) { + if (lr_sful_rec && sset_count(&lr_sful_rec->lb_ips->ips_v4_reachable)) { ds_clear(match); if (is_l3dgw_port(op)) { ds_put_format(match, "is_chassis_resident(%s)", @@ -14776,7 +14503,7 @@ build_lrouter_ipv4_ip_input(struct ovn_port *op, free(lb_ips_v4_as); } - if (sset_count(&op->od->lb_ips->ips_v6_reachable)) { + if (lr_sful_rec && sset_count(&lr_sful_rec->lb_ips->ips_v6_reachable)) { ds_clear(match); if (is_l3dgw_port(op)) { @@ -14878,8 +14605,8 @@ build_lrouter_ipv4_ip_input(struct ovn_port *op, * Priority 60. */ if (!lrnat_rec->lb_force_snat_router_ip) { - build_lrouter_drop_own_dest(op, lrnat_rec, S_ROUTER_IN_IP_INPUT, 60, - false, lflows); + build_lrouter_drop_own_dest(op, lrnat_rec, lr_sful_rec, + S_ROUTER_IN_IP_INPUT, 60, false, lflows); } /* ARP / ND handling for external IP addresses. * @@ -16018,6 +15745,7 @@ struct lswitch_flow_build_info { const struct hmap *lr_ports; const struct ls_port_group_table *ls_port_groups; const struct lr_nat_table *lr_nats; + const struct lr_stateful_table *lr_sful_table; struct hmap *lflows; struct hmap *igmp_groups; const struct shash *meter_groups; @@ -16101,14 +15829,15 @@ build_lswitch_and_lrouter_iterate_by_lr(struct ovn_datapath *od, * switch port. */ static void -build_lswitch_and_lrouter_iterate_by_lsp(struct ovn_port *op, - const struct hmap *ls_ports, - const struct hmap *lr_ports, - const struct lr_nat_table *lr_nats, - const struct shash *meter_groups, - struct ds *match, - struct ds *actions, - struct hmap *lflows) +build_lswitch_and_lrouter_iterate_by_lsp( + struct ovn_port *op, const struct hmap *ls_ports, + const struct hmap *lr_ports, + const struct lr_nat_table *lr_nats, + const struct lr_stateful_table *lr_sful_table, + const struct shash *meter_groups, + struct ds *match, + struct ds *actions, + struct hmap *lflows) { ovs_assert(op->nbsp); start_collecting_lflows(); @@ -16121,11 +15850,14 @@ build_lswitch_and_lrouter_iterate_by_lsp(struct ovn_port *op, meter_groups, actions, match); build_lswitch_dhcp_options_and_response(op, lflows, meter_groups); build_lswitch_external_port(op, lflows); - build_lswitch_ip_unicast_lookup(op, lr_nats, lflows, actions, match); + build_lswitch_ip_unicast_lookup(op, lr_nats, lr_sful_table, lflows, + actions, match); /* Build Logical Router Flows. */ - build_ip_routing_flows_for_router_type_lsp(op, lr_ports, lflows); - build_arp_resolve_flows_for_lsp(op, lflows, lr_ports, match, actions); + build_ip_routing_flows_for_router_type_lsp(op, lr_sful_table, lr_ports, + lflows); + build_arp_resolve_flows_for_lsp(op, lflows, lr_ports, lr_sful_table, + match, actions); link_ovn_port_to_lflows(op, &collected_lflows); end_collecting_lflows(); @@ -16144,6 +15876,8 @@ build_lswitch_and_lrouter_iterate_by_lrp(struct ovn_port *op, lsi->lr_nats, op->od->index); ovs_assert(lrnet_rec); + const struct lr_stateful_record *lr_sful_rec = + lr_stateful_table_find_by_index(lsi->lr_sful_table, op->od->index); build_adm_ctrl_flows_for_lrouter_port(op, lsi->lflows, &lsi->match, &lsi->actions); build_neigh_learning_flows_for_lrouter_port(op, lsi->lflows, &lsi->match, @@ -16151,15 +15885,15 @@ build_lswitch_and_lrouter_iterate_by_lrp(struct ovn_port *op, build_ip_routing_flows_for_lrp(op, lsi->lflows); build_ND_RA_flows_for_lrouter_port(op, lsi->lflows, &lsi->match, &lsi->actions, lsi->meter_groups); - build_arp_resolve_flows_for_lrp(op, lrnet_rec, lsi->lflows, &lsi->match, - &lsi->actions); + build_arp_resolve_flows_for_lrp(op, lrnet_rec, lr_sful_rec, lsi->lflows, + &lsi->match, &lsi->actions); build_egress_delivery_flows_for_lrouter_port(op, lsi->lflows, &lsi->match, &lsi->actions); build_dhcpv6_reply_flows_for_lrouter_port(op, lsi->lflows, &lsi->match); build_ipv6_input_flows_for_lrouter_port(op, lsi->lflows, &lsi->match, &lsi->actions, lsi->meter_groups); - build_lrouter_ipv4_ip_input(op, lsi->lflows, lrnet_rec, + build_lrouter_ipv4_ip_input(op, lsi->lflows, lrnet_rec, lr_sful_rec, &lsi->match, &lsi->actions, lsi->meter_groups); build_lrouter_force_snat_flows_op(op, lrnet_rec, lsi->lflows, &lsi->match, &lsi->actions); @@ -16220,12 +15954,13 @@ build_lflows_thread(void *arg) return NULL; } build_lswitch_and_lrouter_iterate_by_lsp(op, lsi->ls_ports, - lsi->lr_ports, - lsi->lr_nats, - lsi->meter_groups, - &lsi->match, - &lsi->actions, - lsi->lflows); + lsi->lr_ports, + lsi->lr_nats, + lsi->lr_sful_table, + lsi->meter_groups, + &lsi->match, + &lsi->actions, + lsi->lflows); } } for (bnum = control->id; @@ -16332,6 +16067,7 @@ build_lswitch_and_lrouter_flows(const struct ovn_datapaths *ls_datapaths, const struct hmap *lr_ports, const struct ls_port_group_table *ls_pgs, const struct lr_nat_table *lr_nats, + const struct lr_stateful_table *lr_sful_table, struct hmap *lflows, struct hmap *igmp_groups, const struct shash *meter_groups, @@ -16362,6 +16098,7 @@ build_lswitch_and_lrouter_flows(const struct ovn_datapaths *ls_datapaths, lsiv[index].lr_ports = lr_ports; lsiv[index].ls_port_groups = ls_pgs; lsiv[index].lr_nats = lr_nats; + lsiv[index].lr_sful_table = lr_sful_table; lsiv[index].igmp_groups = igmp_groups; lsiv[index].meter_groups = meter_groups; lsiv[index].lb_dps_map = lb_dps_map; @@ -16397,6 +16134,7 @@ build_lswitch_and_lrouter_flows(const struct ovn_datapaths *ls_datapaths, .lr_ports = lr_ports, .ls_port_groups = ls_pgs, .lr_nats = lr_nats, + .lr_sful_table = lr_sful_table, .lflows = lflows, .igmp_groups = igmp_groups, .meter_groups = meter_groups, @@ -16425,6 +16163,7 @@ build_lswitch_and_lrouter_flows(const struct ovn_datapaths *ls_datapaths, build_lswitch_and_lrouter_iterate_by_lsp(op, lsi.ls_ports, lsi.lr_ports, lsi.lr_nats, + lsi.lr_sful_table, lsi.meter_groups, &lsi.match, &lsi.actions, lsi.lflows); @@ -16546,6 +16285,7 @@ void build_lflows(struct ovsdb_idl_txn *ovnsb_txn, input_data->lr_ports, input_data->ls_port_groups, input_data->lr_nats, + input_data->lr_sful_table, lflows, &igmp_groups, input_data->meter_groups, @@ -17024,6 +16764,7 @@ lflow_handle_northd_port_changes(struct ovsdb_idl_txn *ovnsb_txn, build_lswitch_and_lrouter_iterate_by_lsp(op, lflow_input->ls_ports, lflow_input->lr_ports, lflow_input->lr_nats, + lflow_input->lr_sful_table, lflow_input->meter_groups, &match, &actions, lflows); @@ -17059,11 +16800,11 @@ lflow_handle_northd_port_changes(struct ovsdb_idl_txn *ovnsb_txn, struct ds match = DS_EMPTY_INITIALIZER; struct ds actions = DS_EMPTY_INITIALIZER; build_lswitch_and_lrouter_iterate_by_lsp(op, lflow_input->ls_ports, - lflow_input->lr_ports, - lflow_input->lr_nats, - lflow_input->meter_groups, - &match, &actions, - lflows); + lflow_input->lr_ports, + lflow_input->lr_nats, + lflow_input->lr_sful_table, + lflow_input->meter_groups, + &match, &actions, lflows); ds_destroy(&match); ds_destroy(&actions); diff --git a/northd/northd.h b/northd/northd.h index 4dd260761e..6d64fff626 100644 --- a/northd/northd.h +++ b/northd/northd.h @@ -180,6 +180,7 @@ struct lflow_input { const struct hmap *lr_ports; const struct ls_port_group_table *ls_port_groups; const struct lr_nat_table *lr_nats; + const struct lr_stateful_table *lr_sful_table; const struct shash *meter_groups; const struct hmap *lb_datapaths_map; const struct hmap *bfd_connections; @@ -319,9 +320,6 @@ struct ovn_datapath { /* router datapath has a logical port with redirect-type set to bridged. */ bool redirect_bridged; - /* Load Balancer vIPs relevant for this datapath. */ - struct ovn_lb_ip_set *lb_ips; - struct ovn_port **localnet_ports; size_t n_localnet_ports; @@ -338,6 +336,119 @@ struct ovn_datapath { const struct ovn_datapath *ovn_datapath_find(const struct hmap *datapaths, const struct uuid *uuid); +/* A logical switch port or logical router port. + * + * In steady state, an ovn_port points to a northbound Logical_Switch_Port + * record (via 'nbsp') *or* a Logical_Router_Port record (via 'nbrp'), and to a + * southbound Port_Binding record (via 'sb'). As the state of the system + * changes, join_logical_ports() may determine that there is a new LSP or LRP + * that has no corresponding Port_Binding record (in which case build_ports()) + * will create the missing Port_Binding) or that a Port_Binding record exists + * that has no coresponding LSP (in which case build_ports() will delete the + * spurious Port_Binding). Thus, after build_ports() runs, any given ovn_port + * will have 'sb' nonnull, and 'nbsp' xor 'nbrp' nonnull. + * + * Ordinarily there is only one ovn_port that points to a given LSP or LRP (but + * distributed gateway ports point a "derived" ovn_port to a duplicate LRP). + */ +struct ovn_port { + /* Port name aka key. + * + * This is ordinarily the same as nbsp->name or nbrp->name and + * sb->logical_port. (A distributed gateway port creates a "derived" + * ovn_port with key "cr-%s" % nbrp->name.) */ + struct hmap_node key_node; /* Index on 'key'. */ + char *key; /* nbsp->name, nbrp->name, sb->logical_port. */ + char *json_key; /* 'key', quoted for use in JSON. */ + + const struct sbrec_port_binding *sb; /* May be NULL. */ + + uint32_t tunnel_key; + + /* Logical switch port data. */ + const struct nbrec_logical_switch_port *nbsp; /* May be NULL. */ + + struct lport_addresses *lsp_addrs; /* Logical switch port addresses. */ + unsigned int n_lsp_addrs; /* Total length of lsp_addrs. */ + unsigned int n_lsp_non_router_addrs; /* Number of elements from the + * beginning of 'lsp_addrs' extracted + * directly from LSP 'addresses'. */ + + struct lport_addresses *ps_addrs; /* Port security addresses. */ + unsigned int n_ps_addrs; + + bool lsp_can_be_inc_processed; /* If it can be incrementally processed when + the port changes. */ + + /* Logical router port data. */ + const struct nbrec_logical_router_port *nbrp; /* May be NULL. */ + + struct lport_addresses lrp_networks; + + /* Logical port multicast data. */ + struct mcast_port_info mcast_info; + + /* At most one of l3dgw_port and cr_port can be not NULL. */ + + /* This is set to a distributed gateway port if and only if this ovn_port + * is "derived" from it. Otherwise this is set to NULL. The derived + * ovn_port represents the instance of distributed gateway port on the + * gateway chassis.*/ + struct ovn_port *l3dgw_port; + + /* This is set to the "derived" chassis-redirect port of this port if and + * only if this port is a distributed gateway port. Otherwise this is set + * to NULL. */ + struct ovn_port *cr_port; + + bool has_unknown; /* If the addresses have 'unknown' defined. */ + + bool has_bfd; + + /* The port's peer: + * + * - A switch port S of type "router" has a router port R as a peer, + * and R in turn has S has its peer. + * + * - Two connected logical router ports have each other as peer. + * + * - Other kinds of ports have no peer. */ + struct ovn_port *peer; + + struct ovn_datapath *od; + + struct ovs_list list; /* In list of similar records. */ + + struct hmap_node dp_node; /* Node in od->ports. */ + + struct lport_addresses proxy_arp_addrs; + + /* Temporarily used for traversing a list (or hmap) of ports. */ + bool visited; + + /* List of struct lflow_ref_node that points to the lflows generated by + * this ovn_port. + * + * This data is initialized and destroyed by the en_northd node, but + * populated and used only by the en_lflow node. Ideally this data should + * be maintained as part of en_lflow's data (struct lflow_data): a hash + * index from ovn_port key to lflows. However, it would be less efficient + * and more complex: + * + * 1. It would require an extra search (using the index) to find the + * lflows. + * + * 2. Building the index needs to be thread-safe, using either a global + * lock which is obviously less efficient, or hash-based lock array which + * is more complex. + * + * Adding the list here is more straightforward. The drawback is that we + * need to keep in mind that this data belongs to en_lflow node, so never + * access it from any other nodes. + */ + struct ovs_list lflows; +}; + void ovnnb_db_run(struct northd_input *input_data, struct northd_data *data, struct ovsdb_idl_txn *ovnnb_txn, @@ -397,9 +508,13 @@ void sync_lbs(struct ovsdb_idl_txn *, const struct sbrec_load_balancer_table *, struct chassis_features *chassis_features); bool check_sb_lb_duplicates(const struct sbrec_load_balancer_table *); +struct lr_stateful_table; void sync_pbs(struct ovsdb_idl_txn *, struct hmap *ls_ports, - struct hmap *lr_ports, const struct lr_nat_table *); -bool sync_pbs_for_northd_changed_ovn_ports(struct tracked_ovn_ports *); + struct hmap *lr_ports, + const struct lr_stateful_table *); +bool sync_pbs_for_northd_changed_ovn_ports( + struct tracked_ovn_ports *, + const struct lr_stateful_table *); static inline bool northd_has_tracked_data(struct northd_tracked_data *trk_nd_changes) { @@ -424,4 +539,15 @@ northd_has_lr_nats_in_tracked_data(struct northd_tracked_data *trk_nd_changes) return (trk_nd_changes->type & NORTHD_TRACKED_LR_NATS); } +/* Returns 'true' if the IPv4 'addr' is on the same subnet with one of the + * IPs configured on the router port. + */ +bool lrouter_port_ipv4_reachable(const struct ovn_port *, ovs_be32 addr); + +/* Returns 'true' if the IPv6 'addr' is on the same subnet with one of the + * IPs configured on the router port. + */ +bool lrouter_port_ipv6_reachable(const struct ovn_port *, + const struct in6_addr *); + #endif /* NORTHD_H */ diff --git a/northd/ovn-northd.c b/northd/ovn-northd.c index 40f9764b3a..e85e00cc9b 100644 --- a/northd/ovn-northd.c +++ b/northd/ovn-northd.c @@ -871,6 +871,7 @@ main(int argc, char *argv[]) stopwatch_create(PORT_GROUP_RUN_STOPWATCH_NAME, SW_MS); stopwatch_create(SYNC_METERS_RUN_STOPWATCH_NAME, SW_MS); stopwatch_create(LR_NAT_RUN_STOPWATCH_NAME, SW_MS); + stopwatch_create(LR_STATEFUL_RUN_STOPWATCH_NAME, SW_MS); /* Initialize incremental processing engine for ovn-northd */ inc_proc_northd_init(&ovnnb_idl_loop, &ovnsb_idl_loop); diff --git a/tests/ovn-northd.at b/tests/ovn-northd.at index dc01e43de0..03d62695db 100644 --- a/tests/ovn-northd.at +++ b/tests/ovn-northd.at @@ -10516,18 +10516,21 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb set load_balancer . ip_port_mappings:10.0.0.3=sw0-p1:10.0.0.2 check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute nocompute check ovn-nbctl --wait=sb set load_balancer . options:foo=bar check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute nocompute check ovn-nbctl --wait=sb -- lb-add lb2 20.0.0.10:80 20.0.0.20:80 -- lb-add lb3 30.0.0.10:80 30.0.0.20:80 check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute nocompute @@ -10537,6 +10540,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb -- lb-del lb2 -- lb-del lb3 check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute nocompute @@ -10550,6 +10554,7 @@ AT_CHECK([ovn-nbctl --wait=sb \ ]) check_engine_stats lb_data norecompute compute check_engine_stats northd recompute nocompute +check_engine_stats lr_stateful recompute nocompute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute nocompute @@ -10567,6 +10572,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb clear Load_Balancer . health_check check_engine_stats lb_data norecompute compute check_engine_stats northd recompute nocompute +check_engine_stats lr_stateful recompute nocompute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute nocompute @@ -10581,6 +10587,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats ovn-nbctl --wait=sb lsp-set-options sw0-lr0 router-port=lr0-sw0 check_engine_stats lb_data norecompute compute check_engine_stats northd recompute compute +check_engine_stats lr_stateful recompute nocompute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute nocompute @@ -10589,6 +10596,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb ls-lb-add sw0 lb1 check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute # A LB applied to a switch/router triggers: # - a recompute in the first iteration (handling northd change) @@ -10601,6 +10609,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb set load_balancer lb1 vips:'"10.0.0.10:80"'='"10.0.0.100:80"' check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute compute CHECK_NO_CHANGE_AFTER_RECOMPUTE(1) @@ -10610,6 +10619,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb clear load_Balancer lb1 vips check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute compute CHECK_NO_CHANGE_AFTER_RECOMPUTE(1) @@ -10619,6 +10629,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb lb-add lb1 10.0.0.10:80 10.0.0.3:80 check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute compute CHECK_NO_CHANGE_AFTER_RECOMPUTE(1) @@ -10628,6 +10639,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb lb-add lb1 10.0.0.20:80 10.0.0.30:8080 check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute compute CHECK_NO_CHANGE_AFTER_RECOMPUTE(1) @@ -10637,6 +10649,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb ls-lb-del sw0 lb1 check_engine_stats lb_data norecompute compute check_engine_stats northd recompute nocompute +check_engine_stats lr_stateful recompute nocompute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute compute CHECK_NO_CHANGE_AFTER_RECOMPUTE(1) @@ -10647,6 +10660,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb ls-lb-add sw0 lb1 -- lsp-add sw0 sw0p1 check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute compute @@ -10667,6 +10681,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb lr-lb-add lr0 lb1 check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute compute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -10676,6 +10691,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb set load_balancer lb1 vips:'"10.0.0.10:80"'='"10.0.0.100:80"' check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute compute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -10685,6 +10701,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb clear load_Balancer lb1 vips check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute compute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -10694,6 +10711,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb lb-add lb1 10.0.0.10:80 10.0.0.3:80 check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute compute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -10703,6 +10721,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb lb-add lb1 10.0.0.20:80 10.0.0.30:8080 check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute compute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -10733,6 +10752,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb add load_balancer_group . load_Balancer $lb1_uuid check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute nocompute @@ -10740,6 +10760,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb clear load_balancer_group . load_Balancer check_engine_stats lb_data norecompute compute check_engine_stats northd recompute nocompute +check_engine_stats lr_stateful recompute nocompute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute nocompute @@ -10756,6 +10777,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb add logical_switch sw0 load_balancer_group $lbg1_uuid check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute compute @@ -10772,6 +10794,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb set load_balancer lb1 vips:'"10.0.0.10:80"'='"10.0.0.100:80"' check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute compute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -10781,6 +10804,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb clear load_Balancer lb1 vips check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute compute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -10790,6 +10814,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb lb-add lb1 10.0.0.10:80 10.0.0.3:80 check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute compute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -10799,6 +10824,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb lb-add lb1 10.0.0.20:80 10.0.0.30:8080 check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute compute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -10814,6 +10840,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl add logical_router lr0 load_balancer_group $lbg1_uuid check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute compute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -10823,6 +10850,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb set load_balancer lb1 vips:'"10.0.0.10:80"'='"10.0.0.100:80"' check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute compute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -10832,6 +10860,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb clear load_Balancer lb1 vips check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute compute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -10841,6 +10870,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb lb-add lb1 10.0.0.10:80 10.0.0.3:80 check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute compute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -10850,6 +10880,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb lb-add lb1 10.0.0.20:80 10.0.0.30:8080 check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute compute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -10858,6 +10889,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb clear logical_router lr0 load_balancer_group check_engine_stats lb_data norecompute compute check_engine_stats northd recompute nocompute +check_engine_stats lr_stateful recompute nocompute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute compute @@ -10866,6 +10898,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb add logical_switch sw0 load_balancer_group $lbg1_uuid check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute compute @@ -10874,6 +10907,7 @@ check ovn-nbctl --wait=sb clear logical_switch sw0 load_balancer_group -- \ destroy load_balancer_group $lbg1_uuid check_engine_stats lb_data norecompute compute check_engine_stats northd recompute nocompute +check_engine_stats lr_stateful recompute nocompute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb compute compute @@ -10897,6 +10931,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats lbg1_uuid=$(ovn-nbctl --wait=sb create load_balancer_group name=lbg1) check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow norecompute nocompute check_engine_stats sync_to_sb_lb norecompute nocompute @@ -10904,6 +10939,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb set load_balancer_group . load_balancer="$lb2_uuid,$lb3_uuid,$lb4_uuid" check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute nocompute @@ -10911,6 +10947,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb set logical_switch sw0 load_balancer_group=$lbg1_uuid check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute compute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -10919,6 +10956,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb set logical_router lr1 load_balancer_group=$lbg1_uuid check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute compute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -10927,6 +10965,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb ls-lb-add sw0 lb2 check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute nocompute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -10935,6 +10974,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb ls-lb-add sw0 lb3 check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute nocompute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -10944,6 +10984,7 @@ check ovn-nbctl --wait=sb lr-lb-add lr1 lb1 check ovn-nbctl --wait=sb lr-lb-add lr1 lb2 check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute compute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -10952,6 +10993,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb ls-lb-del sw0 lb2 check_engine_stats lb_data norecompute compute check_engine_stats northd recompute nocompute +check_engine_stats lr_stateful recompute nocompute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute nocompute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -10960,6 +11002,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb lr-lb-del lr1 lb2 check_engine_stats lb_data norecompute compute check_engine_stats northd recompute nocompute +check_engine_stats lr_stateful recompute nocompute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute nocompute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -10970,6 +11013,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb lb-del lb4 check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute compute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -10980,6 +11024,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb lb-del lb2 check_engine_stats lb_data norecompute compute check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute compute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -10988,6 +11033,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb remove load_balancer_group . load_balancer $lb3_uuid check_engine_stats lb_data norecompute compute check_engine_stats northd recompute nocompute +check_engine_stats lr_stateful recompute nocompute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_lb recompute compute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -11054,6 +11100,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb lr-add lr0 check_engine_stats northd recompute nocompute check_engine_stats lr_nat recompute nocompute +check_engine_stats lr_stateful recompute nocompute check_engine_stats sync_to_sb_pb recompute nocompute check_engine_stats lflow recompute nocompute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -11066,6 +11113,7 @@ check ovn-nbctl lrp-add lr0 lr0-sw0 00:00:00:00:ff:01 10.0.0.1/24 # for the SB port binding change. check_engine_stats northd recompute compute check_engine_stats lr_nat recompute nocompute +check_engine_stats lr_stateful recompute nocompute check_engine_stats sync_to_sb_pb recompute nocompute check_engine_stats lflow recompute nocompute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -11077,6 +11125,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb lsp-set-options sw0-lr0 router-port=lr0-sw0 check_engine_stats northd recompute compute check_engine_stats lr_nat recompute nocompute +check_engine_stats lr_stateful recompute nocompute check_engine_stats sync_to_sb_pb recompute nocompute check_engine_stats lflow recompute nocompute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -11102,6 +11151,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl set logical_router_port lr0-sw0 options:foo=bar check_engine_stats northd recompute nocompute check_engine_stats lr_nat recompute nocompute +check_engine_stats lr_stateful recompute nocompute check_engine_stats sync_to_sb_pb recompute nocompute check_engine_stats lflow recompute nocompute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -11131,6 +11181,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb set NAT . external_ip=172.168.0.120 check_engine_stats northd norecompute compute check_engine_stats lr_nat norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_pb recompute nocompute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -11140,6 +11191,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb set NAT . logical_ip=10.0.0.10 check_engine_stats northd norecompute compute check_engine_stats lr_nat norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_pb recompute nocompute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -11149,6 +11201,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb set NAT . type=snat check_engine_stats northd norecompute compute check_engine_stats lr_nat norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_pb recompute nocompute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -11158,6 +11211,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb lr-nat-add lr0 dnat_and_snat 172.168.0.110 10.0.0.4 sw0p1 30:54:00:00:00:03 check_engine_stats northd norecompute compute check_engine_stats lr_nat norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_pb recompute nocompute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -11168,6 +11222,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb set NAT $nat2_uuid external_mac='"30:54:00:00:00:04"' check_engine_stats northd norecompute compute check_engine_stats lr_nat norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_pb recompute nocompute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -11184,6 +11239,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb lr-nat-add lr0 dnat_and_snat 172.168.0.140 10.0.0.20 check_engine_stats northd norecompute compute check_engine_stats lr_nat norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats sync_to_sb_pb recompute nocompute check_engine_stats lflow recompute nocompute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -11192,6 +11248,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb lr-nat-add lr0 dnat_and_snat 172.168.0.150 10.0.0.41 check_engine_stats northd norecompute compute check_engine_stats lr_nat norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats sync_to_sb_pb recompute nocompute check_engine_stats lflow recompute nocompute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -11200,6 +11257,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb lr-nat-del lr0 dnat_and_snat 172.168.0.150 check_engine_stats northd norecompute compute check_engine_stats lr_nat norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats sync_to_sb_pb recompute nocompute check_engine_stats lflow recompute nocompute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -11208,6 +11266,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb lr-nat-del lr0 dnat_and_snat 172.168.0.140 check_engine_stats northd norecompute compute check_engine_stats lr_nat norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats sync_to_sb_pb recompute nocompute check_engine_stats lflow recompute nocompute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -11217,6 +11276,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb clear logical_router lr0 nat check_engine_stats northd norecompute compute check_engine_stats lr_nat norecompute compute +check_engine_stats lr_stateful norecompute compute check_engine_stats lflow recompute nocompute check_engine_stats sync_to_sb_pb recompute nocompute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -11226,6 +11286,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb lr-policy-add lr0 10 "ip4.src == 10.0.0.3" reroute 172.168.0.101,172.168.0.102 check_engine_stats northd recompute nocompute check_engine_stats lr_nat recompute nocompute +check_engine_stats lr_stateful recompute nocompute check_engine_stats sync_to_sb_pb recompute nocompute check_engine_stats lflow recompute nocompute CHECK_NO_CHANGE_AFTER_RECOMPUTE @@ -11234,6 +11295,7 @@ check as northd ovn-appctl -t NORTHD_TYPE inc-engine/clear-stats check ovn-nbctl --wait=sb lr-policy-del lr0 10 "ip4.src == 10.0.0.3" check_engine_stats northd recompute nocompute check_engine_stats lr_nat recompute nocompute +check_engine_stats lr_stateful recompute nocompute check_engine_stats sync_to_sb_pb recompute nocompute check_engine_stats lflow recompute nocompute CHECK_NO_CHANGE_AFTER_RECOMPUTE