From 0cd7984b912d520dbbc3a884dbadd8367c13d91e Mon Sep 17 00:00:00 2001
From: Jiakun Yan <jiakunyan1998@gmail.com>
Date: Sat, 29 Jun 2024 21:10:02 -0700
Subject: [PATCH] improve(ofi): one ofi domain per device

---
 lci/backend/ofi/server_ofi.c | 18 +++++++++---------
 lci/backend/ofi/server_ofi.h |  4 ++--
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/lci/backend/ofi/server_ofi.c b/lci/backend/ofi/server_ofi.c
index 258505c2..ff89b093 100644
--- a/lci/backend/ofi/server_ofi.c
+++ b/lci/backend/ofi/server_ofi.c
@@ -117,15 +117,11 @@ void LCISD_server_init(LCIS_server_t* s)
 
   // Create libfabric obj.
   FI_SAFECALL(fi_fabric(server->info->fabric_attr, &server->fabric, NULL));
-
-  // Create domain.
-  FI_SAFECALL(fi_domain(server->fabric, server->info, &server->domain, NULL));
 }
 
 void LCISD_server_fina(LCIS_server_t s)
 {
   LCISI_server_t* server = (LCISI_server_t*)s;
-  FI_SAFECALL(fi_close((struct fid*)&server->domain->fid));
   FI_SAFECALL(fi_close((struct fid*)&server->fabric->fid));
   fi_freeinfo(server->info);
   free(s);
@@ -139,10 +135,15 @@ void LCISD_endpoint_init(LCIS_server_t server_pp, LCIS_endpoint_t* endpoint_pp,
   *endpoint_pp = (LCIS_endpoint_t)endpoint_p;
   endpoint_p->server = (LCISI_server_t*)server_pp;
   endpoint_p->is_single_threaded = single_threaded;
+
+  // Create domain.
+  FI_SAFECALL(fi_domain(endpoint_p->server->fabric, endpoint_p->server->info,
+                        &endpoint_p->domain, NULL));
+
   // Create end-point;
   endpoint_p->server->info->tx_attr->size = LCI_SERVER_MAX_SENDS;
   endpoint_p->server->info->rx_attr->size = LCI_SERVER_MAX_RECVS;
-  FI_SAFECALL(fi_endpoint(endpoint_p->server->domain, endpoint_p->server->info,
+  FI_SAFECALL(fi_endpoint(endpoint_p->domain, endpoint_p->server->info,
                           &endpoint_p->ep, NULL));
 
   // Create cq.
@@ -150,8 +151,7 @@ void LCISD_endpoint_init(LCIS_server_t server_pp, LCIS_endpoint_t* endpoint_pp,
   memset(&cq_attr, 0, sizeof(struct fi_cq_attr));
   cq_attr.format = FI_CQ_FORMAT_DATA;
   cq_attr.size = LCI_SERVER_MAX_CQES;
-  FI_SAFECALL(
-      fi_cq_open(endpoint_p->server->domain, &cq_attr, &endpoint_p->cq, NULL));
+  FI_SAFECALL(fi_cq_open(endpoint_p->domain, &cq_attr, &endpoint_p->cq, NULL));
 
   // Bind my ep to cq.
   FI_SAFECALL(
@@ -160,8 +160,7 @@ void LCISD_endpoint_init(LCIS_server_t server_pp, LCIS_endpoint_t* endpoint_pp,
   struct fi_av_attr av_attr;
   memset(&av_attr, 0, sizeof(av_attr));
   av_attr.type = FI_AV_MAP;
-  FI_SAFECALL(
-      fi_av_open(endpoint_p->server->domain, &av_attr, &endpoint_p->av, NULL));
+  FI_SAFECALL(fi_av_open(endpoint_p->domain, &av_attr, &endpoint_p->av, NULL));
   FI_SAFECALL(fi_ep_bind(endpoint_p->ep, (fid_t)endpoint_p->av, 0));
   FI_SAFECALL(fi_enable(endpoint_p->ep));
 
@@ -214,4 +213,5 @@ void LCISD_endpoint_fina(LCIS_endpoint_t endpoint_pp)
   FI_SAFECALL(fi_close((struct fid*)&endpoint_p->ep->fid));
   FI_SAFECALL(fi_close((struct fid*)&endpoint_p->cq->fid));
   FI_SAFECALL(fi_close((struct fid*)&endpoint_p->av->fid));
+  FI_SAFECALL(fi_close((struct fid*)&endpoint_p->domain->fid));
 }
\ No newline at end of file
diff --git a/lci/backend/ofi/server_ofi.h b/lci/backend/ofi/server_ofi.h
index 1ae28f73..af39a654 100644
--- a/lci/backend/ofi/server_ofi.h
+++ b/lci/backend/ofi/server_ofi.h
@@ -38,12 +38,12 @@ struct LCISI_endpoint_t;
 typedef struct __attribute__((aligned(LCI_CACHE_LINE))) LCISI_server_t {
   struct fi_info* info;
   struct fid_fabric* fabric;
-  struct fid_domain* domain;
 } LCISI_server_t;
 
 typedef struct __attribute__((aligned(LCI_CACHE_LINE))) LCISI_endpoint_t {
   struct LCISI_endpoint_super_t super;
   LCISI_server_t* server;
+  struct fid_domain* domain;
   struct fid_ep* ep;
   struct fid_cq* cq;
   struct fid_av* av;
@@ -65,7 +65,7 @@ static inline void* LCISI_real_server_reg(LCIS_endpoint_t endpoint_pp,
     rdma_key = __sync_fetch_and_add(&g_next_rdma_key, 1);
   }
   struct fid_mr* mr;
-  FI_SAFECALL(fi_mr_reg(server->domain, buf, size,
+  FI_SAFECALL(fi_mr_reg(endpoint_p->domain, buf, size,
                         FI_READ | FI_WRITE | FI_REMOTE_WRITE, 0, rdma_key, 0,
                         &mr, 0));
   if (server->info->domain_attr->mr_mode & FI_MR_ENDPOINT) {