diff --git a/prov/efa/src/efa.h b/prov/efa/src/efa.h index dc8ebfe9f0a..7e85962cbee 100644 --- a/prov/efa/src/efa.h +++ b/prov/efa/src/efa.h @@ -93,21 +93,6 @@ #define EFA_DEFAULT_INTER_MIN_READ_WRITE_SIZE (65536) #define EFA_DEFAULT_INTRA_MAX_GDRCOPY_FROM_DEV_SIZE (3072) -/* - * The default memory alignment - */ -#define EFA_RDM_DEFAULT_MEMORY_ALIGNMENT (8) - -/* - * The CUDA memory alignment - */ -#define EFA_RDM_CUDA_MEMORY_ALIGNMENT (64) - -/* - * The alignment to support in-order aligned ops. - */ -#define EFA_RDM_IN_ORDER_ALIGNMENT (128) - /* * Set alignment to x86 cache line size. */ diff --git a/prov/efa/src/rdm/efa_rdm_atomic.c b/prov/efa/src/rdm/efa_rdm_atomic.c index 1850f665899..ac28e983b84 100644 --- a/prov/efa/src/rdm/efa_rdm_atomic.c +++ b/prov/efa/src/rdm/efa_rdm_atomic.c @@ -58,10 +58,6 @@ efa_rdm_atomic_alloc_txe(struct efa_rdm_ep *efa_rdm_ep, return NULL; } - efa_domain_ope_list_lock(efa_rdm_ep_domain(efa_rdm_ep)); - dlist_insert_tail(&txe->ep_entry, &efa_rdm_ep->txe_list); - efa_domain_ope_list_unlock(efa_rdm_ep_domain(efa_rdm_ep)); - ofi_ioc_to_iov(msg_atomic->msg_iov, iov, msg_atomic->iov_count, datatype_size); msg.addr = msg_atomic->addr; msg.msg_iov = iov; diff --git a/prov/efa/src/rdm/efa_rdm_ep.h b/prov/efa/src/rdm/efa_rdm_ep.h index 6008c59a389..eb5cf941900 100644 --- a/prov/efa/src/rdm/efa_rdm_ep.h +++ b/prov/efa/src/rdm/efa_rdm_ep.h @@ -29,6 +29,21 @@ struct efa_rdm_ep_queued_copy { #define EFA_RDM_MAX_QUEUED_COPY (8) +/* + * The default memory alignment + */ +#define EFA_RDM_EP_DEFAULT_MEMORY_ALIGNMENT (8) + +/* + * The CUDA memory alignment + */ +#define EFA_RDM_EP_CUDA_MEMORY_ALIGNMENT (64) + +/* + * The alignment to support in-order aligned ops. + */ +#define EFA_RDM_EP_IN_ORDER_ALIGNMENT (128) + /** * Max number of opes queued before handshake is made * with their peers. This cnt is per EP. @@ -195,7 +210,8 @@ struct efa_rdm_ep { struct efa_rdm_pke **pke_vec; /* Work arrays for efa_rdm_ope_post_send to avoid stack allocation */ struct efa_rdm_pke **send_pkt_entry_vec; - int *send_pkt_entry_size_vec; + size_t *send_pkt_entry_vec_data_sizes; + size_t send_pkt_entry_vec_size; struct dlist_entry entry; /* the count of opes queued before handshake is made with their peers */ size_t ope_queued_before_handshake_cnt; @@ -220,13 +236,6 @@ struct efa_rdm_peer *efa_rdm_ep_get_peer_explicit(struct efa_rdm_ep *ep, fi_addr int32_t efa_rdm_ep_get_peer_ahn(struct efa_rdm_ep *ep, fi_addr_t addr); struct efa_rdm_peer *efa_rdm_ep_get_peer_implicit(struct efa_rdm_ep *ep, fi_addr_t addr); -struct efa_rdm_ope *efa_rdm_ep_alloc_txe(struct efa_rdm_ep *efa_rdm_ep, - struct efa_rdm_peer *peer, - const struct fi_msg *msg, - uint32_t op, - uint64_t tag, - uint64_t flags); - struct efa_rdm_ope *efa_rdm_ep_alloc_rxe(struct efa_rdm_ep *ep, struct efa_rdm_peer *peer, uint32_t op); @@ -253,7 +262,26 @@ void efa_rdm_ep_queue_rnr_pkt(struct efa_rdm_ep *ep, struct efa_rdm_pke *pkt_ent ssize_t efa_rdm_ep_post_queued_pkts(struct efa_rdm_ep *ep, struct dlist_entry *pkts); -size_t efa_rdm_ep_get_memory_alignment(struct efa_rdm_ep *ep, enum fi_hmem_iface iface); +/** + * @brief Get memory alignment for given ep and hmem iface + * + * @param ep efa rdm ep + * @param iface hmem iface + * @return size_t the memory alignment + */ +static inline size_t efa_rdm_ep_get_memory_alignment(struct efa_rdm_ep *ep, + enum fi_hmem_iface iface) +{ + size_t memory_alignment = EFA_RDM_EP_DEFAULT_MEMORY_ALIGNMENT; + + if (ep->sendrecv_in_order_aligned_128_bytes) { + memory_alignment = EFA_RDM_EP_IN_ORDER_ALIGNMENT; + } else if (iface == FI_HMEM_CUDA) { + memory_alignment = EFA_RDM_EP_CUDA_MEMORY_ALIGNMENT; + } + + return memory_alignment; +} static inline struct efa_domain *efa_rdm_ep_domain(struct efa_rdm_ep *ep) @@ -547,4 +575,10 @@ fi_addr_t efa_rdm_ep_get_explicit_shm_fi_addr(struct efa_rdm_ep *ep, fi_addr_t a conn = efa_av_addr_to_conn(ep->base_ep.av, addr); return conn ? conn->shm_fi_addr : FI_ADDR_NOTAVAIL; } + +static inline size_t efa_rdm_ep_get_available_tx_pkts(struct efa_rdm_ep *ep) +{ + return ep->efa_max_outstanding_tx_ops - ep->efa_outstanding_tx_ops - + ep->efa_rnr_queued_pkt_cnt; +} #endif diff --git a/prov/efa/src/rdm/efa_rdm_ep_fiops.c b/prov/efa/src/rdm/efa_rdm_ep_fiops.c index e3c003002a1..faa35860b20 100644 --- a/prov/efa/src/rdm/efa_rdm_ep_fiops.c +++ b/prov/efa/src/rdm/efa_rdm_ep_fiops.c @@ -220,7 +220,7 @@ int efa_rdm_ep_create_buffer_pools(struct efa_rdm_ep *ep) true, /* need memory registration */ efa_env.readcopy_pool_size, efa_env.readcopy_pool_size, /* max_cnt==chunk_cnt means pool is not allowed to grow */ - EFA_RDM_IN_ORDER_ALIGNMENT, /* support in-order aligned send/recv */ + EFA_RDM_EP_IN_ORDER_ALIGNMENT, /* support in-order aligned send/recv */ 0, &ep->rx_readcopy_pkt_pool); if (ret) @@ -680,9 +680,13 @@ int efa_rdm_ep_open(struct fid_domain *domain, struct fi_info *info, goto err_free_pke_vec; } - efa_rdm_ep->send_pkt_entry_size_vec = calloc(sizeof(int), efa_base_ep_get_tx_pool_size(&efa_rdm_ep->base_ep)); - if (!efa_rdm_ep->send_pkt_entry_size_vec) { - EFA_WARN(FI_LOG_EP_CTRL, "cannot alloc memory for efa_rdm_ep->send_pkt_entry_size_vec!\n"); + efa_rdm_ep->send_pkt_entry_vec_data_sizes = + calloc(sizeof(size_t), + efa_base_ep_get_tx_pool_size(&efa_rdm_ep->base_ep)); + if (!efa_rdm_ep->send_pkt_entry_vec_data_sizes) { + EFA_WARN(FI_LOG_EP_CTRL, + "cannot alloc memory for " + "efa_rdm_ep->send_pkt_entry_vec_data_sizes!\n"); ret = -FI_ENOMEM; goto err_free_send_pkt_entry_vec; } @@ -1190,8 +1194,8 @@ static int efa_rdm_ep_close(struct fid *fid) free(efa_rdm_ep->pke_vec); if (efa_rdm_ep->send_pkt_entry_vec) free(efa_rdm_ep->send_pkt_entry_vec); - if (efa_rdm_ep->send_pkt_entry_size_vec) - free(efa_rdm_ep->send_pkt_entry_size_vec); + if (efa_rdm_ep->send_pkt_entry_vec_data_sizes) + free(efa_rdm_ep->send_pkt_entry_vec_data_sizes); ofi_genlock_unlock(&domain->srx_lock); diff --git a/prov/efa/src/rdm/efa_rdm_ep_utils.c b/prov/efa/src/rdm/efa_rdm_ep_utils.c index 9c2132710db..cb754815551 100644 --- a/prov/efa/src/rdm/efa_rdm_ep_utils.c +++ b/prov/efa/src/rdm/efa_rdm_ep_utils.c @@ -317,36 +317,6 @@ int efa_rdm_ep_post_user_recv_buf(struct efa_rdm_ep *ep, struct efa_rdm_ope *rxe return err; } - - -/* create a new txe */ -struct efa_rdm_ope *efa_rdm_ep_alloc_txe(struct efa_rdm_ep *efa_rdm_ep, - struct efa_rdm_peer *peer, - const struct fi_msg *msg, - uint32_t op, - uint64_t tag, - uint64_t flags) -{ - struct efa_rdm_ope *txe; - - txe = ofi_buf_alloc(efa_rdm_ep->ope_pool); - if (OFI_UNLIKELY(!txe)) { - EFA_DBG(FI_LOG_EP_CTRL, "TX entries exhausted.\n"); - return NULL; - } - - efa_rdm_txe_construct(txe, efa_rdm_ep, peer, msg, op, flags); - if (op == ofi_op_tagged) { - txe->cq_entry.tag = tag; - txe->tag = tag; - } - - efa_domain_ope_list_lock(efa_rdm_ep_domain(efa_rdm_ep)); - dlist_insert_tail(&txe->ep_entry, &efa_rdm_ep->txe_list); - efa_domain_ope_list_unlock(efa_rdm_ep_domain(efa_rdm_ep)); - return txe; -} - /** * @brief record the event that a TX op has been submitted * @@ -674,15 +644,18 @@ static ssize_t efa_rdm_ep_handshake_common(struct efa_rdm_ep *ep, struct efa_rdm msg.addr = peer->conn->fi_addr; - txe = efa_rdm_ep_alloc_txe(ep, peer, &msg, ofi_op_write, 0, 0); - + txe = ofi_buf_alloc(ep->ope_pool); if (OFI_UNLIKELY(!txe)) { EFA_WARN(FI_LOG_EP_CTRL, "TX entries exhausted.\n"); return -FI_EAGAIN; } - /* efa_rdm_ep_alloc_txe() joins ep->base_ep.util_ep.tx_op_flags and passed in flags, - * reset to desired flags (remove things like FI_DELIVERY_COMPLETE, and FI_COMPLETION) + efa_rdm_txe_construct(txe, ep, peer, &msg, ofi_op_write, 0); + + /* + * efa_rdm_txe_construct() joins ep->base_ep.util_ep.tx_op_flags and + * passed in flags, reset to desired flags (remove things like + * FI_DELIVERY_COMPLETE, and FI_COMPLETION) */ txe->fi_flags = EFA_RDM_TXE_NO_COMPLETION | EFA_RDM_TXE_NO_COUNTER; txe->internal_flags |= EFA_RDM_OPE_INTERNAL; @@ -1068,26 +1041,6 @@ void efa_rdm_ep_post_internal_rx_pkts(struct efa_rdm_ep *ep) efa_base_ep_write_eq_error(&ep->base_ep, err, FI_EFA_ERR_INTERNAL_RX_BUF_POST); } -/** - * @brief Get memory alignment for given ep and hmem iface - * - * @param ep efa rdm ep - * @param iface hmem iface - * @return size_t the memory alignment - */ -size_t efa_rdm_ep_get_memory_alignment(struct efa_rdm_ep *ep, enum fi_hmem_iface iface) -{ - size_t memory_alignment = EFA_RDM_DEFAULT_MEMORY_ALIGNMENT; - - if (ep->sendrecv_in_order_aligned_128_bytes) { - memory_alignment = EFA_RDM_IN_ORDER_ALIGNMENT; - } else if (iface == FI_HMEM_CUDA) { - memory_alignment = EFA_RDM_CUDA_MEMORY_ALIGNMENT; - } - - return memory_alignment; -} - /** * @brief Enforce a handshake to made for given txe. * It will trigger a handshake with peer and choose to diff --git a/prov/efa/src/rdm/efa_rdm_msg.c b/prov/efa/src/rdm/efa_rdm_msg.c index 13844764156..dd01ff48ec7 100644 --- a/prov/efa/src/rdm/efa_rdm_msg.c +++ b/prov/efa/src/rdm/efa_rdm_msg.c @@ -162,6 +162,7 @@ ssize_t efa_rdm_msg_generic_send(struct efa_rdm_ep *ep, const struct fi_msg *msg ssize_t err; struct efa_rdm_ope *txe; struct efa_rdm_peer *peer; + size_t available_tx_pkts; efa_rdm_tracepoint(send_begin_msg_context, (size_t) msg->context, (size_t) msg->addr); @@ -177,12 +178,25 @@ ssize_t efa_rdm_msg_generic_send(struct efa_rdm_ep *ep, const struct fi_msg *msg goto out; } - txe = efa_rdm_ep_alloc_txe(ep, peer, msg, op, tag, flags); + // Handle case when there are no TX packets available + available_tx_pkts = efa_rdm_ep_get_available_tx_pkts(ep); + if (OFI_UNLIKELY(available_tx_pkts == 0)) { + err = -FI_EAGAIN; + goto out; + } + + txe = ofi_buf_alloc(ep->ope_pool); if (OFI_UNLIKELY(!txe)) { err = -FI_EAGAIN; goto out; } + efa_rdm_txe_construct(txe, ep, peer, msg, op, flags); + if (op == ofi_op_tagged) { + txe->cq_entry.tag = tag; + txe->tag = tag; + } + assert(txe->op == ofi_op_msg || txe->op == ofi_op_tagged); txe->msg_id = peer->next_msg_id++; diff --git a/prov/efa/src/rdm/efa_rdm_ope.c b/prov/efa/src/rdm/efa_rdm_ope.c index 98406f70222..224becaa401 100644 --- a/prov/efa/src/rdm/efa_rdm_ope.c +++ b/prov/efa/src/rdm/efa_rdm_ope.c @@ -94,6 +94,10 @@ void efa_rdm_txe_construct(struct efa_rdm_ope *txe, EFA_WARN(FI_LOG_CQ, "invalid operation type\n"); assert(0); } + + efa_domain_ope_list_lock(efa_rdm_ep_domain(ep)); + dlist_insert_tail(&txe->ep_entry, &ep->txe_list); + efa_domain_ope_list_unlock(efa_rdm_ep_domain(ep)); } void efa_rdm_txe_release(struct efa_rdm_ope *txe) @@ -458,25 +462,22 @@ size_t efa_rdm_txe_max_req_data_capacity(struct efa_rdm_ep *ep, struct efa_rdm_o * On success, return 0 * If there is not enough available packet entry in TX packet pool, return -FI_EAGAIN */ -ssize_t efa_rdm_ope_prepare_to_post_send(struct efa_rdm_ope *ope, - int pkt_type, - int *pkt_entry_cnt, - int *pkt_entry_data_size_vec) +ssize_t efa_rdm_ope_prepare_to_post_send(struct efa_rdm_ope *ope, int pkt_type, + size_t *pkt_entry_cnt, + size_t *pkt_entry_data_size_vec) { struct efa_rdm_ep *ep; size_t total_pkt_entry_data_size; /* total number of bytes send via packet entry's payload */ size_t single_pkt_entry_data_size; size_t single_pkt_entry_max_data_size; int i, memory_alignment, remainder, iface; - int available_tx_pkts; + size_t available_tx_pkts; ep = ope->ep; assert(ep->efa_max_outstanding_tx_ops >= ep->efa_outstanding_tx_ops + ep->efa_rnr_queued_pkt_cnt); - available_tx_pkts = ep->efa_max_outstanding_tx_ops - - ep->efa_outstanding_tx_ops - - ep->efa_rnr_queued_pkt_cnt; + available_tx_pkts = efa_rdm_ep_get_available_tx_pkts(ep); if (available_tx_pkts == 0) return -FI_EAGAIN; @@ -1314,7 +1315,7 @@ ssize_t efa_rdm_txe_prepare_local_read_pkt_entry(struct efa_rdm_ope *txe) assert(pkt_entry_copy->mr); txe->local_read_pkt_entry = pkt_entry_copy; /* pkt from read-copy pool only stores actual application data in wiredata */ - assert(ofi_is_addr_aligned((void *)pkt_entry_copy->wiredata, EFA_RDM_IN_ORDER_ALIGNMENT)); + assert(ofi_is_addr_aligned((void *)pkt_entry_copy->wiredata, EFA_RDM_EP_IN_ORDER_ALIGNMENT)); txe->rma_iov[0].addr = (uint64_t)pkt_entry_copy->wiredata; txe->rma_iov[0].key = fi_mr_key(pkt_entry_copy->mr); return 0; @@ -1785,20 +1786,23 @@ ssize_t efa_rdm_ope_post_send(struct efa_rdm_ope *ope, int pkt_type) struct efa_rdm_ep *ep; ssize_t err; int64_t segment_offset; - int pkt_entry_cnt, pkt_entry_cnt_allocated = 0; + int pkt_entry_cnt_allocated = 0; int i; uint64_t flags = 0; ep = ope->ep; assert(ep); - err = efa_rdm_ope_prepare_to_post_send(ope, pkt_type, &pkt_entry_cnt, ep->send_pkt_entry_size_vec); + err = efa_rdm_ope_prepare_to_post_send(ope, pkt_type, + &ep->send_pkt_entry_vec_size, + ep->send_pkt_entry_vec_data_sizes); if (err) return err; - assert(pkt_entry_cnt <= efa_base_ep_get_tx_pool_size(&ep->base_ep)); + assert(ep->send_pkt_entry_vec_size <= + efa_base_ep_get_tx_pool_size(&ep->base_ep)); segment_offset = efa_rdm_pkt_type_contains_data(pkt_type) ? (int64_t) ope->bytes_sent : -1; - for (i = 0; i < pkt_entry_cnt; ++i) { + for (i = 0; i < ep->send_pkt_entry_vec_size; ++i) { ep->send_pkt_entry_vec[i] = efa_rdm_pke_alloc(ep, ep->efa_tx_pkt_pool, EFA_RDM_PKE_FROM_EFA_TX_POOL); if (OFI_UNLIKELY(!ep->send_pkt_entry_vec[i])) { @@ -1808,21 +1812,19 @@ ssize_t efa_rdm_ope_post_send(struct efa_rdm_ope *ope, int pkt_type) pkt_entry_cnt_allocated++; - err = efa_rdm_pke_fill_data(ep->send_pkt_entry_vec[i], - pkt_type, - ope, - segment_offset, - ep->send_pkt_entry_size_vec[i]); + err = efa_rdm_pke_fill_data(ep->send_pkt_entry_vec[i], pkt_type, + ope, segment_offset, + ep->send_pkt_entry_vec_data_sizes[i]); if (err) goto handle_err; - if (segment_offset != -1 && pkt_entry_cnt > 1) { - assert(ep->send_pkt_entry_size_vec[i] > 0); - segment_offset += ep->send_pkt_entry_size_vec[i]; + if (segment_offset != -1 && ep->send_pkt_entry_vec_size > 1) { + assert(ep->send_pkt_entry_vec_data_sizes[i] > 0); + segment_offset += ep->send_pkt_entry_vec_data_sizes[i]; } } - assert(pkt_entry_cnt == pkt_entry_cnt_allocated); + assert(ep->send_pkt_entry_vec_size == pkt_entry_cnt_allocated); /** * We currently respect FI_MORE only for eager pkt type because @@ -1838,12 +1840,13 @@ ssize_t efa_rdm_ope_post_send(struct efa_rdm_ope *ope, int pkt_type) if (ope->fi_flags & FI_MORE && efa_rdm_pkt_type_is_eager(pkt_type)) flags |= FI_MORE; - err = efa_rdm_pke_sendv(ep->send_pkt_entry_vec, pkt_entry_cnt, flags); + err = efa_rdm_pke_sendv(ep->send_pkt_entry_vec, + ep->send_pkt_entry_vec_size, flags); if (err) goto handle_err; ope->peer->flags |= EFA_RDM_PEER_REQ_SENT; - for (i = 0; i < pkt_entry_cnt; ++i) + for (i = 0; i < ep->send_pkt_entry_vec_size; ++i) efa_rdm_pke_handle_sent(ep->send_pkt_entry_vec[i], pkt_type, ope->peer); return FI_SUCCESS; diff --git a/prov/efa/src/rdm/efa_rdm_ope.h b/prov/efa/src/rdm/efa_rdm_ope.h index 61c2c74883d..6451acaeecb 100644 --- a/prov/efa/src/rdm/efa_rdm_ope.h +++ b/prov/efa/src/rdm/efa_rdm_ope.h @@ -352,10 +352,9 @@ int efa_rdm_rxe_post_local_read_or_queue(struct efa_rdm_ope *rxe, struct efa_rdm_pke *pkt_entry, char *pkt_data, size_t data_size); -ssize_t efa_rdm_ope_prepare_to_post_send(struct efa_rdm_ope *ope, - int pkt_type, - int *pkt_entry_cnt, - int *pkt_entry_data_size_vec); +ssize_t efa_rdm_ope_prepare_to_post_send(struct efa_rdm_ope *ope, int pkt_type, + size_t *pkt_entry_cnt, + size_t *pkt_entry_data_size_vec); ssize_t efa_rdm_ope_post_send(struct efa_rdm_ope *ope, int pkt_type); diff --git a/prov/efa/src/rdm/efa_rdm_pke_rtm.c b/prov/efa/src/rdm/efa_rdm_pke_rtm.c index 96fa541dd32..5e0a7e5b3de 100644 --- a/prov/efa/src/rdm/efa_rdm_pke_rtm.c +++ b/prov/efa/src/rdm/efa_rdm_pke_rtm.c @@ -120,9 +120,9 @@ ssize_t efa_rdm_pke_init_rtm_with_payload(struct efa_rdm_pke *pkt_entry, if (data_size + segment_offset < txe->total_len) { if (efa_mr_is_cuda(txe->desc[0])) { if (txe->ep->sendrecv_in_order_aligned_128_bytes) - data_size &= ~(EFA_RDM_IN_ORDER_ALIGNMENT - 1); + data_size &= ~(EFA_RDM_EP_IN_ORDER_ALIGNMENT - 1); else - data_size &= ~(EFA_RDM_CUDA_MEMORY_ALIGNMENT -1); + data_size &= ~(EFA_RDM_EP_CUDA_MEMORY_ALIGNMENT -1); } } } diff --git a/prov/efa/src/rdm/efa_rdm_rma.c b/prov/efa/src/rdm/efa_rdm_rma.c index 9d9434014a2..d361dd564b5 100644 --- a/prov/efa/src/rdm/efa_rdm_rma.c +++ b/prov/efa/src/rdm/efa_rdm_rma.c @@ -74,9 +74,6 @@ efa_rdm_rma_alloc_txe(struct efa_rdm_ep *efa_rdm_ep, memcpy(txe->rma_iov, msg_rma->rma_iov, sizeof(struct fi_rma_iov) * msg_rma->rma_iov_count); - efa_domain_ope_list_lock(efa_rdm_ep_domain(efa_rdm_ep)); - dlist_insert_tail(&txe->ep_entry, &efa_rdm_ep->txe_list); - efa_domain_ope_list_unlock(efa_rdm_ep_domain(efa_rdm_ep)); return txe; } diff --git a/prov/efa/test/efa_unit_test_common.c b/prov/efa/test/efa_unit_test_common.c index 563ac92e925..2f2c131b0a5 100644 --- a/prov/efa/test/efa_unit_test_common.c +++ b/prov/efa/test/efa_unit_test_common.c @@ -403,6 +403,7 @@ struct efa_rdm_ope *efa_unit_test_alloc_txe(struct efa_resource *resource, uint3 struct efa_rdm_peer *peer; struct fi_msg msg = {0}; struct efa_rdm_ep *efa_rdm_ep; + struct efa_rdm_ope *txe; efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid); @@ -415,7 +416,12 @@ struct efa_rdm_ope *efa_unit_test_alloc_txe(struct efa_resource *resource, uint3 peer = efa_rdm_ep_get_peer(efa_rdm_ep, peer_addr); - return efa_rdm_ep_alloc_txe(efa_rdm_ep, peer, &msg, op, 0, 0); + txe = ofi_buf_alloc(efa_rdm_ep->ope_pool); + if (!txe) + return NULL; + + efa_rdm_txe_construct(txe, efa_rdm_ep, peer, &msg, op, 0); + return txe; } struct efa_rdm_ope *efa_unit_test_alloc_rxe(struct efa_resource *resource, uint32_t op) diff --git a/prov/efa/test/efa_unit_test_ep.c b/prov/efa/test/efa_unit_test_ep.c index 6033778591b..ec9daf55b50 100644 --- a/prov/efa/test/efa_unit_test_ep.c +++ b/prov/efa/test/efa_unit_test_ep.c @@ -364,7 +364,7 @@ void test_efa_rdm_read_copy_pkt_pool_128_alignment(struct efa_resource **state) assert_non_null(pkt_entry); efa_rdm_ep->rx_readcopy_pkt_pool_used++; assert(ofi_is_addr_aligned((void *) pkt_entry->wiredata, - EFA_RDM_IN_ORDER_ALIGNMENT)); + EFA_RDM_EP_IN_ORDER_ALIGNMENT)); efa_rdm_pke_release_rx(pkt_entry); fi_close(&ep->fid); diff --git a/prov/efa/test/efa_unit_test_ope.c b/prov/efa/test/efa_unit_test_ope.c index 6bca9813103..f4fb1c1db8d 100644 --- a/prov/efa/test/efa_unit_test_ope.c +++ b/prov/efa/test/efa_unit_test_ope.c @@ -12,7 +12,7 @@ void test_efa_rdm_ope_prepare_to_post_send_impl(struct efa_resource *resource, size_t total_len, int expected_ret, int expected_pkt_entry_cnt, - int *expected_pkt_entry_data_size_vec) + size_t *expected_pkt_entry_data_size_vec) { struct efa_ep_addr raw_addr; struct efa_rdm_mr mock_mr; @@ -20,7 +20,8 @@ void test_efa_rdm_ope_prepare_to_post_send_impl(struct efa_resource *resource, struct efa_rdm_peer mock_peer; size_t raw_addr_len = sizeof(raw_addr); fi_addr_t addr; - int pkt_entry_cnt, pkt_entry_data_size_vec[1024]; + size_t pkt_entry_cnt; + size_t pkt_entry_data_size_vec[1024]; int i, err, ret; ret = fi_getname(&resource->ep->fid, &raw_addr, &raw_addr_len); @@ -93,7 +94,7 @@ void test_efa_rdm_ope_prepare_to_post_send_host_memory(struct efa_resource **sta struct efa_resource *resource = *state; size_t msg_length; int expected_pkt_entry_cnt; - int expected_pkt_entry_data_size_vec[1024]; + size_t expected_pkt_entry_data_size_vec[1024]; efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME); @@ -142,7 +143,7 @@ void test_efa_rdm_ope_prepare_to_post_send_host_memory_align128(struct efa_resou struct efa_rdm_ep *efa_rdm_ep; size_t msg_length; int expected_pkt_entry_cnt; - int expected_pkt_entry_data_size_vec[1024]; + size_t expected_pkt_entry_data_size_vec[1024]; efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME); efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid); @@ -191,7 +192,7 @@ void test_efa_rdm_ope_prepare_to_post_send_cuda_memory(struct efa_resource **sta struct efa_resource *resource = *state; size_t msg_length; int expected_pkt_entry_cnt; - int expected_pkt_entry_data_size_vec[1024]; + size_t expected_pkt_entry_data_size_vec[1024]; efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME); @@ -216,7 +217,7 @@ void test_efa_rdm_ope_prepare_to_post_send_cuda_memory_align128(struct efa_resou struct efa_rdm_ep *efa_rdm_ep; size_t msg_length; int expected_pkt_entry_cnt; - int expected_pkt_entry_data_size_vec[1024]; + size_t expected_pkt_entry_data_size_vec[1024]; efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME); efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid); @@ -599,8 +600,9 @@ void test_efa_rdm_txe_prepare_local_read_pkt_entry(struct efa_resource **state) assert_int_equal(fi_endpoint(resource->domain, resource->info, &ep, NULL), 0); efa_rdm_ep = container_of(ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid); - txe = efa_rdm_ep_alloc_txe(efa_rdm_ep, NULL, &msg, ofi_op_msg, 0, 0); + txe = ofi_buf_alloc(efa_rdm_ep->ope_pool); assert_non_null(txe); + efa_rdm_txe_construct(txe, efa_rdm_ep, NULL, &msg, ofi_op_msg, 0); /* Use ooo rx pkt because it doesn't have mr so a read_copy pkt clone is enforced. */ pkt_entry = efa_rdm_pke_alloc(efa_rdm_ep, efa_rdm_ep->rx_ooo_pkt_pool, EFA_RDM_PKE_FROM_OOO_POOL); diff --git a/prov/efa/test/efa_unit_test_pke.c b/prov/efa/test/efa_unit_test_pke.c index e5eb24dfa3a..fa16882fcc1 100644 --- a/prov/efa/test/efa_unit_test_pke.c +++ b/prov/efa/test/efa_unit_test_pke.c @@ -47,8 +47,9 @@ void test_efa_rdm_pke_handle_longcts_rtm_send_completion(struct efa_resource **s msg.iov_count = 1; msg.msg_iov = &iov; msg.desc = NULL; - txe = efa_rdm_ep_alloc_txe(efa_rdm_ep, peer, &msg, ofi_op_msg, 0, 0); + txe = ofi_buf_alloc(efa_rdm_ep->ope_pool); assert_non_null(txe); + efa_rdm_txe_construct(txe, efa_rdm_ep, peer, &msg, ofi_op_msg, 0); txe->internal_flags |= EFA_RDM_OPE_READ_NACK; /* construct a fallback long cts rtm pkt */ @@ -322,8 +323,9 @@ void test_efa_rdm_pke_flag_tracking(struct efa_resource **state) msg.iov_count = 1; msg.msg_iov = &iov; msg.desc = NULL; - txe = efa_rdm_ep_alloc_txe(efa_rdm_ep, peer, &msg, ofi_op_msg, 0, 0); + txe = ofi_buf_alloc(efa_rdm_ep->ope_pool); assert_non_null(txe); + efa_rdm_txe_construct(txe, efa_rdm_ep, peer, &msg, ofi_op_msg, 0); /* Allocate a packet entry */ pkt_entry = efa_rdm_pke_alloc(efa_rdm_ep, efa_rdm_ep->efa_tx_pkt_pool, EFA_RDM_PKE_FROM_EFA_TX_POOL);