add net_cq_poll related counters to measure the thread contention in …

…progress engine
uiuc-hpc · Jul 1, 2024 · e132671 · e132671
1 parent 0cd7984
commit e132671
Show file tree

Hide file tree

Showing 4 changed files with 8 additions and 3 deletions.
diff --git a/lci/backend/ibv/server_ibv.h b/lci/backend/ibv/server_ibv.h
@@ -140,7 +140,8 @@ static inline int LCISD_poll_cq(LCIS_endpoint_t endpoint_pp,
 #ifdef LCI_ENABLE_MULTITHREAD_PROGRESS
   LCIU_release_spinlock(&endpoint_p->cq_lock);
 #endif
-  if (ne > 0) LCII_PCOUNTER_ADD(net_poll_cq_num, ne);
+  LCII_PCOUNTER_ADD(net_poll_cq_calls, 1);
+  if (ne > 0) LCII_PCOUNTER_ADD(net_poll_cq_entry_count, ne);
   for (int i = 0; i < ne; i++) {
     LCI_DBG_Assert(
         wc[i].status == IBV_WC_SUCCESS, "Failed status %s (%d) for wr_id %d\n",

diff --git a/lci/backend/ofi/server_ofi.h b/lci/backend/ofi/server_ofi.h
@@ -120,8 +120,9 @@ static inline int LCISD_poll_cq(LCIS_endpoint_t endpoint_pp,
   ne = fi_cq_read(endpoint_p->cq, &fi_entry, LCI_CQ_MAX_POLL);
   LCISI_OFI_CS_EXIT(endpoint_p, LCI_BACKEND_TRY_LOCK_POLL)
   ret = ne;
+  LCII_PCOUNTER_ADD(net_poll_cq_calls, 1);
   if (ne > 0) {
-    LCII_PCOUNTER_ADD(net_poll_cq_num, ne);
+    LCII_PCOUNTER_ADD(net_poll_cq_entry_count, ne);
     // Got an entry here
     for (int i = 0; i < ne; i++) {
       if (fi_entry[i].flags & FI_RECV) {

diff --git a/lci/backend/server.h b/lci/backend/server.h
@@ -165,6 +165,7 @@ static inline void LCIS_endpoint_fina(LCIS_endpoint_t endpoint_pp)
 static inline int LCIS_poll_cq(LCIS_endpoint_t endpoint_pp,
                                LCIS_cq_entry_t* entry)
 {
+  LCII_PCOUNTER_ADD(net_poll_cq_attempts, 1);
   LCISI_CS_ENTER(endpoint_pp, 0);
   int ret = LCISD_poll_cq(endpoint_pp, entry);
   LCISI_CS_EXIT(endpoint_pp);

diff --git a/lci/profile/performance_counter.h b/lci/profile/performance_counter.h
@@ -30,7 +30,9 @@ extern LCT_pcounter_ctx_t LCII_pcounter_ctx;
     _macro(net_send_failed_lock)             \
     _macro(net_send_failed_nomem)            \
     _macro(net_recv_failed_nopacket)         \
-    _macro(net_poll_cq_num)                  \
+    _macro(net_poll_cq_attempts)             \
+    _macro(net_poll_cq_calls)                \
+    _macro(net_poll_cq_entry_count)          \
     _macro(progress_call)                    \
     _macro(packet_get)                       \
     _macro(packet_put)                       \