diff mbox series

[ovs-dev,v3,1/4] northd: Optimize load balancer lookups for groups.

Message ID 20220909213223.824013-2-i.maximets@ovn.org
State Accepted
Headers show
Series northd: Optimize preparation of load balancers. | expand

Checks

Context Check Description
ovsrobot/apply-robot success apply and check: success
ovsrobot/github-robot-_Build_and_Test success github build: passed
ovsrobot/github-robot-_ovn-kubernetes fail github build: failed

Commit Message

Ilya Maximets Sept. 9, 2022, 9:32 p.m. UTC 
ovn_northd_lb_lookup() takes significant percent of northd runtime
in scenarios with large number of load balancers.  In many cases,
like in ovn-kubernetes, a lot of load balancers are actually groupped
and applied to most of the switches and routers.  So, instead of
looking up all the same load balancers from the group for each
datapath, we can look them up once and store as a group.  Later we
can lookup the entire group at once.

Acked-by: Dumitru Ceara <dceara@redhat.com>
Signed-off-by: Ilya Maximets <i.maximets@ovn.org>
---
 lib/lb.c           | 49 +++++++++++++++++++++++++++++++-
 lib/lb.h           | 21 ++++++++++++--
 northd/en-northd.c |  2 ++
 northd/northd.c    | 70 ++++++++++++++++++++++++++++------------------
 northd/northd.h    |  3 ++
 5 files changed, 115 insertions(+), 30 deletions(-)
diff mbox series

Patch

diff --git a/lib/lb.c b/lib/lb.c
index 7b0ed1abe..fca5f27da 100644
--- a/lib/lb.c
+++ b/lib/lb.c
@@ -226,7 +226,7 @@  ovn_northd_lb_create(const struct nbrec_load_balancer *nbrec_lb)
 }
 
 struct ovn_northd_lb *
-ovn_northd_lb_find(struct hmap *lbs, const struct uuid *uuid)
+ovn_northd_lb_find(const struct hmap *lbs, const struct uuid *uuid)
 {
     struct ovn_northd_lb *lb;
     size_t hash = uuid_hash(uuid);
@@ -275,6 +275,53 @@  ovn_northd_lb_destroy(struct ovn_northd_lb *lb)
     free(lb);
 }
 
+/* Constructs a new 'struct ovn_lb_group' object from the Nb LB Group record
+ * and a hash map of all existing 'struct ovn_northd_lb' objects. */
+struct ovn_lb_group *
+ovn_lb_group_create(const struct nbrec_load_balancer_group *nbrec_lb_group,
+                    const struct hmap *lbs)
+{
+    struct ovn_lb_group *lb_group;
+
+    lb_group = xzalloc(sizeof *lb_group);
+    lb_group->uuid = nbrec_lb_group->header_.uuid;
+    lb_group->n_lbs = nbrec_lb_group->n_load_balancer;
+    lb_group->lbs = xmalloc(lb_group->n_lbs * sizeof *lb_group->lbs);
+
+    for (size_t i = 0; i < nbrec_lb_group->n_load_balancer; i++) {
+        const struct uuid *lb_uuid =
+            &nbrec_lb_group->load_balancer[i]->header_.uuid;
+        lb_group->lbs[i] = ovn_northd_lb_find(lbs, lb_uuid);
+    }
+
+    return lb_group;
+}
+
+void
+ovn_lb_group_destroy(struct ovn_lb_group *lb_group)
+{
+    if (!lb_group) {
+        return;
+    }
+
+    free(lb_group->lbs);
+    free(lb_group);
+}
+
+struct ovn_lb_group *
+ovn_lb_group_find(const struct hmap *lb_groups, const struct uuid *uuid)
+{
+    struct ovn_lb_group *lb_group;
+    size_t hash = uuid_hash(uuid);
+
+    HMAP_FOR_EACH_WITH_HASH (lb_group, hmap_node, hash, lb_groups) {
+        if (uuid_equals(&lb_group->uuid, uuid)) {
+            return lb_group;
+        }
+    }
+    return NULL;
+}
+
 struct ovn_controller_lb *
 ovn_controller_lb_create(const struct sbrec_load_balancer *sbrec_lb)
 {
diff --git a/lib/lb.h b/lib/lb.h
index 832ed31fb..1c70bfb6e 100644
--- a/lib/lb.h
+++ b/lib/lb.h
@@ -20,10 +20,12 @@ 
 #include <sys/types.h>
 #include <netinet/in.h>
 #include "openvswitch/hmap.h"
-#include "sset.h"
 #include "ovn-util.h"
+#include "sset.h"
+#include "uuid.h"
 
 struct nbrec_load_balancer;
+struct nbrec_load_balancer_group;
 struct sbrec_load_balancer;
 struct sbrec_datapath_binding;
 struct ovn_port;
@@ -86,13 +88,28 @@  struct ovn_northd_lb_backend {
 };
 
 struct ovn_northd_lb *ovn_northd_lb_create(const struct nbrec_load_balancer *);
-struct ovn_northd_lb * ovn_northd_lb_find(struct hmap *, const struct uuid *);
+struct ovn_northd_lb *ovn_northd_lb_find(const struct hmap *,
+                                         const struct uuid *);
 void ovn_northd_lb_destroy(struct ovn_northd_lb *);
 void
 ovn_northd_lb_add_lr(struct ovn_northd_lb *lb, struct ovn_datapath *od);
 void
 ovn_northd_lb_add_ls(struct ovn_northd_lb *lb, struct ovn_datapath *od);
 
+struct ovn_lb_group {
+    struct hmap_node hmap_node;
+    struct uuid uuid;
+    size_t n_lbs;
+    struct ovn_northd_lb **lbs;
+};
+
+struct ovn_lb_group *ovn_lb_group_create(
+    const struct nbrec_load_balancer_group *,
+    const struct hmap *lbs);
+void ovn_lb_group_destroy(struct ovn_lb_group *lb_group);
+struct ovn_lb_group *ovn_lb_group_find(const struct hmap *lb_groups,
+                                       const struct uuid *);
+
 struct ovn_controller_lb {
     const struct sbrec_load_balancer *slb; /* May be NULL. */
 
diff --git a/northd/en-northd.c b/northd/en-northd.c
index 4907a1ff2..7fe83db64 100644
--- a/northd/en-northd.c
+++ b/northd/en-northd.c
@@ -68,6 +68,8 @@  void en_northd_run(struct engine_node *node, void *data)
         EN_OVSDB_GET(engine_get_input("NB_logical_router", node));
     input_data.nbrec_load_balancer_table =
         EN_OVSDB_GET(engine_get_input("NB_load_balancer", node));
+    input_data.nbrec_load_balancer_group_table =
+        EN_OVSDB_GET(engine_get_input("NB_load_balancer_group", node));
     input_data.nbrec_port_group_table =
         EN_OVSDB_GET(engine_get_input("NB_port_group", node));
     input_data.nbrec_address_set_table =
diff --git a/northd/northd.c b/northd/northd.c
index deda4a9d3..e00625e9c 100644
--- a/northd/northd.c
+++ b/northd/northd.c
@@ -3843,11 +3843,14 @@  build_lrouter_lb_ips(struct ovn_datapath *od, const struct ovn_northd_lb *lb)
 
 static void
 build_lbs(struct northd_input *input_data, struct hmap *datapaths,
-          struct hmap *lbs)
+          struct hmap *lbs, struct hmap *lb_groups)
 {
+    const struct nbrec_load_balancer_group *nbrec_lb_group;
+    struct ovn_lb_group *lb_group;
     struct ovn_northd_lb *lb;
 
     hmap_init(lbs);
+    hmap_init(lb_groups);
 
     const struct nbrec_load_balancer *nbrec_lb;
     NBREC_LOAD_BALANCER_TABLE_FOR_EACH (nbrec_lb,
@@ -3857,6 +3860,13 @@  build_lbs(struct northd_input *input_data, struct hmap *datapaths,
                     uuid_hash(&nbrec_lb->header_.uuid));
     }
 
+    NBREC_LOAD_BALANCER_GROUP_TABLE_FOR_EACH (nbrec_lb_group,
+                               input_data->nbrec_load_balancer_group_table) {
+        lb_group = ovn_lb_group_create(nbrec_lb_group, lbs);
+        hmap_insert(lb_groups, &lb_group->hmap_node,
+                    uuid_hash(&lb_group->uuid));
+    }
+
     struct ovn_datapath *od;
     HMAP_FOR_EACH (od, key_node, datapaths) {
         if (!od->nbs) {
@@ -3871,13 +3881,11 @@  build_lbs(struct northd_input *input_data, struct hmap *datapaths,
         }
 
         for (size_t i = 0; i < od->nbs->n_load_balancer_group; i++) {
-            const struct nbrec_load_balancer_group *lbg =
-                od->nbs->load_balancer_group[i];
-            for (size_t j = 0; j < lbg->n_load_balancer; j++) {
-                const struct uuid *lb_uuid =
-                    &lbg->load_balancer[j]->header_.uuid;
-                lb = ovn_northd_lb_find(lbs, lb_uuid);
-                ovn_northd_lb_add_ls(lb, od);
+            nbrec_lb_group = od->nbs->load_balancer_group[i];
+            lb_group = ovn_lb_group_find(lb_groups,
+                                         &nbrec_lb_group->header_.uuid);
+            for (size_t j = 0; j < lb_group->n_lbs; j++) {
+                ovn_northd_lb_add_ls(lb_group->lbs[j], od);
             }
         }
     }
@@ -3896,14 +3904,12 @@  build_lbs(struct northd_input *input_data, struct hmap *datapaths,
         }
 
         for (size_t i = 0; i < od->nbr->n_load_balancer_group; i++) {
-            const struct nbrec_load_balancer_group *lbg =
-                od->nbr->load_balancer_group[i];
-            for (size_t j = 0; j < lbg->n_load_balancer; j++) {
-                const struct uuid *lb_uuid =
-                    &lbg->load_balancer[j]->header_.uuid;
-                lb = ovn_northd_lb_find(lbs, lb_uuid);
-                ovn_northd_lb_add_lr(lb, od);
-                build_lrouter_lb_ips(od, lb);
+            nbrec_lb_group = od->nbr->load_balancer_group[i];
+            lb_group = ovn_lb_group_find(lb_groups,
+                                         &nbrec_lb_group->header_.uuid);
+            for (size_t j = 0; j < lb_group->n_lbs; j++) {
+                ovn_northd_lb_add_lr(lb_group->lbs[j], od);
+                build_lrouter_lb_ips(od, lb_group->lbs[j]);
             }
         }
     }
@@ -4021,7 +4027,8 @@  build_lrouter_lbs_check(const struct hmap *datapaths)
 }
 
 static void
-build_lrouter_lbs_reachable_ips(struct hmap *datapaths, struct hmap *lbs)
+build_lrouter_lbs_reachable_ips(struct hmap *datapaths, struct hmap *lbs,
+                                struct hmap *lb_groups)
 {
     struct ovn_datapath *od;
 
@@ -4038,13 +4045,14 @@  build_lrouter_lbs_reachable_ips(struct hmap *datapaths, struct hmap *lbs)
         }
 
         for (size_t i = 0; i < od->nbr->n_load_balancer_group; i++) {
-            const struct nbrec_load_balancer_group *lbg =
+            const struct nbrec_load_balancer_group *nbrec_lb_group =
                 od->nbr->load_balancer_group[i];
-            for (size_t j = 0; j < lbg->n_load_balancer; j++) {
-                struct ovn_northd_lb *lb =
-                    ovn_northd_lb_find(lbs,
-                                       &lbg->load_balancer[j]->header_.uuid);
-                build_lrouter_lb_reachable_ips(od, lb);
+            struct ovn_lb_group *lb_group;
+
+            lb_group = ovn_lb_group_find(lb_groups,
+                                         &nbrec_lb_group->header_.uuid);
+            for (size_t j = 0; j < lb_group->n_lbs; j++) {
+                build_lrouter_lb_reachable_ips(od, lb_group->lbs[j]);
             }
         }
     }
@@ -4105,11 +4113,12 @@  build_lswitch_lbs_from_lrouter(struct hmap *datapaths, struct hmap *lbs)
  */
 static void
 build_lb_port_related_data(struct hmap *datapaths, struct hmap *ports,
-                           struct hmap *lbs, struct northd_input *input_data,
+                           struct hmap *lbs, struct hmap *lb_groups,
+                           struct northd_input *input_data,
                            struct ovsdb_idl_txn *ovnsb_txn)
 {
     build_lrouter_lbs_check(datapaths);
-    build_lrouter_lbs_reachable_ips(datapaths, lbs);
+    build_lrouter_lbs_reachable_ips(datapaths, lbs, lb_groups);
     build_lb_svcs(input_data, ovnsb_txn, ports, lbs);
     build_lswitch_lbs_from_lrouter(datapaths, lbs);
 }
@@ -15373,6 +15382,7 @@  northd_init(struct northd_data *data)
     hmap_init(&data->port_groups);
     shash_init(&data->meter_groups);
     hmap_init(&data->lbs);
+    hmap_init(&data->lb_groups);
     hmap_init(&data->bfd_connections);
     ovs_list_init(&data->lr_list);
     data->features = (struct chassis_features) {
@@ -15391,6 +15401,12 @@  northd_destroy(struct northd_data *data)
     }
     hmap_destroy(&data->lbs);
 
+    struct ovn_lb_group *lb_group;
+    HMAP_FOR_EACH_POP (lb_group, hmap_node, &data->lb_groups) {
+        ovn_lb_group_destroy(lb_group);
+    }
+    hmap_destroy(&data->lb_groups);
+
     struct ovn_port_group *pg;
     HMAP_FOR_EACH_SAFE (pg, key_node, &data->port_groups) {
         ovn_port_group_destroy(&data->port_groups, pg);
@@ -15500,12 +15516,12 @@  ovnnb_db_run(struct northd_input *input_data,
 
     build_chassis_features(input_data, &data->features);
     build_datapaths(input_data, ovnsb_txn, &data->datapaths, &data->lr_list);
-    build_lbs(input_data, &data->datapaths, &data->lbs);
+    build_lbs(input_data, &data->datapaths, &data->lbs, &data->lb_groups);
     build_ports(input_data, ovnsb_txn, sbrec_chassis_by_name,
                 sbrec_chassis_by_hostname,
                 &data->datapaths, &data->ports);
     build_lb_port_related_data(&data->datapaths, &data->ports, &data->lbs,
-                               input_data, ovnsb_txn);
+                               &data->lb_groups, input_data, ovnsb_txn);
     build_ipam(&data->datapaths, &data->ports);
     build_port_group_lswitches(input_data, &data->port_groups, &data->ports);
     build_lrouter_groups(&data->ports, &data->lr_list);
diff --git a/northd/northd.h b/northd/northd.h
index d9856af97..8d299864f 100644
--- a/northd/northd.h
+++ b/northd/northd.h
@@ -28,6 +28,8 @@  struct northd_input {
     const struct nbrec_logical_switch_table *nbrec_logical_switch;
     const struct nbrec_logical_router_table *nbrec_logical_router;
     const struct nbrec_load_balancer_table *nbrec_load_balancer_table;
+    const struct nbrec_load_balancer_group_table
+        *nbrec_load_balancer_group_table;
     const struct nbrec_port_group_table *nbrec_port_group_table;
     const struct nbrec_address_set_table *nbrec_address_set_table;
     const struct nbrec_meter_table *nbrec_meter_table;
@@ -74,6 +76,7 @@  struct northd_data {
     struct hmap port_groups;
     struct shash meter_groups;
     struct hmap lbs;
+    struct hmap lb_groups;
     struct hmap bfd_connections;
     struct ovs_list lr_list;
     bool ovn_internal_version_changed;