diff --git a/lci/backend/ofi/server_ofi.c b/lci/backend/ofi/server_ofi.c index 258505c2..742f31e1 100644 --- a/lci/backend/ofi/server_ofi.c +++ b/lci/backend/ofi/server_ofi.c @@ -117,15 +117,11 @@ void LCISD_server_init(LCIS_server_t* s) // Create libfabric obj. FI_SAFECALL(fi_fabric(server->info->fabric_attr, &server->fabric, NULL)); - - // Create domain. - FI_SAFECALL(fi_domain(server->fabric, server->info, &server->domain, NULL)); } void LCISD_server_fina(LCIS_server_t s) { LCISI_server_t* server = (LCISI_server_t*)s; - FI_SAFECALL(fi_close((struct fid*)&server->domain->fid)); FI_SAFECALL(fi_close((struct fid*)&server->fabric->fid)); fi_freeinfo(server->info); free(s); @@ -139,10 +135,14 @@ void LCISD_endpoint_init(LCIS_server_t server_pp, LCIS_endpoint_t* endpoint_pp, *endpoint_pp = (LCIS_endpoint_t)endpoint_p; endpoint_p->server = (LCISI_server_t*)server_pp; endpoint_p->is_single_threaded = single_threaded; + + // Create domain. + FI_SAFECALL(fi_domain(endpoint_p->server->fabric, endpoint_p->server->info, &endpoint_p->domain, NULL)); + // Create end-point; endpoint_p->server->info->tx_attr->size = LCI_SERVER_MAX_SENDS; endpoint_p->server->info->rx_attr->size = LCI_SERVER_MAX_RECVS; - FI_SAFECALL(fi_endpoint(endpoint_p->server->domain, endpoint_p->server->info, + FI_SAFECALL(fi_endpoint(endpoint_p->domain, endpoint_p->server->info, &endpoint_p->ep, NULL)); // Create cq. @@ -151,7 +151,7 @@ void LCISD_endpoint_init(LCIS_server_t server_pp, LCIS_endpoint_t* endpoint_pp, cq_attr.format = FI_CQ_FORMAT_DATA; cq_attr.size = LCI_SERVER_MAX_CQES; FI_SAFECALL( - fi_cq_open(endpoint_p->server->domain, &cq_attr, &endpoint_p->cq, NULL)); + fi_cq_open(endpoint_p->domain, &cq_attr, &endpoint_p->cq, NULL)); // Bind my ep to cq. FI_SAFECALL( @@ -161,7 +161,7 @@ void LCISD_endpoint_init(LCIS_server_t server_pp, LCIS_endpoint_t* endpoint_pp, memset(&av_attr, 0, sizeof(av_attr)); av_attr.type = FI_AV_MAP; FI_SAFECALL( - fi_av_open(endpoint_p->server->domain, &av_attr, &endpoint_p->av, NULL)); + fi_av_open(endpoint_p->domain, &av_attr, &endpoint_p->av, NULL)); FI_SAFECALL(fi_ep_bind(endpoint_p->ep, (fid_t)endpoint_p->av, 0)); FI_SAFECALL(fi_enable(endpoint_p->ep)); @@ -214,4 +214,5 @@ void LCISD_endpoint_fina(LCIS_endpoint_t endpoint_pp) FI_SAFECALL(fi_close((struct fid*)&endpoint_p->ep->fid)); FI_SAFECALL(fi_close((struct fid*)&endpoint_p->cq->fid)); FI_SAFECALL(fi_close((struct fid*)&endpoint_p->av->fid)); + FI_SAFECALL(fi_close((struct fid*)&endpoint_p->domain->fid)); } \ No newline at end of file diff --git a/lci/backend/ofi/server_ofi.h b/lci/backend/ofi/server_ofi.h index 1ae28f73..af39a654 100644 --- a/lci/backend/ofi/server_ofi.h +++ b/lci/backend/ofi/server_ofi.h @@ -38,12 +38,12 @@ struct LCISI_endpoint_t; typedef struct __attribute__((aligned(LCI_CACHE_LINE))) LCISI_server_t { struct fi_info* info; struct fid_fabric* fabric; - struct fid_domain* domain; } LCISI_server_t; typedef struct __attribute__((aligned(LCI_CACHE_LINE))) LCISI_endpoint_t { struct LCISI_endpoint_super_t super; LCISI_server_t* server; + struct fid_domain* domain; struct fid_ep* ep; struct fid_cq* cq; struct fid_av* av; @@ -65,7 +65,7 @@ static inline void* LCISI_real_server_reg(LCIS_endpoint_t endpoint_pp, rdma_key = __sync_fetch_and_add(&g_next_rdma_key, 1); } struct fid_mr* mr; - FI_SAFECALL(fi_mr_reg(server->domain, buf, size, + FI_SAFECALL(fi_mr_reg(endpoint_p->domain, buf, size, FI_READ | FI_WRITE | FI_REMOTE_WRITE, 0, rdma_key, 0, &mr, 0)); if (server->info->domain_attr->mr_mode & FI_MR_ENDPOINT) {