This patch series reorganizes the main datastructure for each queue, struct ionic_qcq. Its constituent struct ionic_queue and struct ionic_cq are stripped down first. Then the generic struct ionic_qcq is stripped down, and a unique struct is created for each queue type. The adminq code is consolidated into ionic_main.c as part of the cleanup. Next comes some minor performance fixups related to queue posting and doorbells. Finally, two minor fixes to Tx packet prep and LIF init. Signed-off-by: Andrew Boyer <aboyer@pensando.io> Andrew Boyer (14): net/ionic: cut down completion queue structure net/ionic: consolidate adminq code net/ionic: convert info array to generic pointers net/ionic: remove unused field from queue structure net/ionic: remove unused interrupt free function net/ionic: cut down queue structure net/ionic: split up queue-completion queue structure net/ionic: use the socket id passed in for Rx and Tx queues net/ionic: log queue counters when tearing down net/ionic: break up queue post function net/ionic: ring doorbell once at the end of each burst net/ionic: send as many packets as possible net/ionic: fix Tx fragment limit check net/ionic: fix code around lif init devcmd drivers/net/ionic/ionic.h | 13 ++ drivers/net/ionic/ionic_dev.c | 132 +------------- drivers/net/ionic/ionic_dev.h | 81 +++------ drivers/net/ionic/ionic_lif.c | 242 +++++++++++++------------ drivers/net/ionic/ionic_lif.h | 77 +++++--- drivers/net/ionic/ionic_main.c | 93 +++++++++- drivers/net/ionic/ionic_rx_filter.c | 1 + drivers/net/ionic/ionic_rxtx.c | 265 ++++++++++++++++------------ 8 files changed, 458 insertions(+), 446 deletions(-) -- 2.17.1
Add Q_NEXT_TO_POST() and Q_NEXT_TO_SRVC() macros. Use a precomputed size mask. This will conserve resources. Signed-off-by: Andrew Boyer <aboyer@pensando.io> --- drivers/net/ionic/ionic_dev.c | 14 +++----------- drivers/net/ionic/ionic_dev.h | 14 +++++++------- drivers/net/ionic/ionic_lif.c | 5 +++-- drivers/net/ionic/ionic_lif.h | 3 ++- drivers/net/ionic/ionic_rxtx.c | 24 ++++++++++++------------ 5 files changed, 27 insertions(+), 33 deletions(-) diff --git a/drivers/net/ionic/ionic_dev.c b/drivers/net/ionic/ionic_dev.c index a9f9e2faf9..f837817a0a 100644 --- a/drivers/net/ionic/ionic_dev.c +++ b/drivers/net/ionic/ionic_dev.c @@ -358,14 +358,8 @@ ionic_dev_cmd_adminq_init(struct ionic_dev *idev, struct ionic_qcq *qcq) } int -ionic_cq_init(struct ionic_lif *lif, struct ionic_cq *cq, - uint32_t num_descs, size_t desc_size) +ionic_cq_init(struct ionic_cq *cq, uint16_t num_descs) { - if (desc_size == 0) { - IONIC_PRINT(ERR, "Descriptor size is %zu", desc_size); - return -EINVAL; - } - if (!rte_is_power_of_2(num_descs) || num_descs < IONIC_MIN_RING_DESC || num_descs > IONIC_MAX_RING_DESC) { @@ -374,9 +368,8 @@ ionic_cq_init(struct ionic_lif *lif, struct ionic_cq *cq, return -EINVAL; } - cq->lif = lif; cq->num_descs = num_descs; - cq->desc_size = desc_size; + cq->size_mask = num_descs - 1; cq->tail_idx = 0; cq->done_color = 1; @@ -393,7 +386,6 @@ ionic_cq_map(struct ionic_cq *cq, void *base, rte_iova_t base_pa) void ionic_cq_bind(struct ionic_cq *cq, struct ionic_queue *q) { - cq->bound_q = q; q->bound_cq = cq; } @@ -407,7 +399,7 @@ ionic_cq_service(struct ionic_cq *cq, uint32_t work_to_do, return 0; while (cb(cq, cq->tail_idx, cb_arg)) { - cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1); + cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1); if (cq->tail_idx == 0) cq->done_color = !cq->done_color; diff --git a/drivers/net/ionic/ionic_dev.h b/drivers/net/ionic/ionic_dev.h index bacbe3f053..b29bfd13be 100644 --- a/drivers/net/ionic/ionic_dev.h +++ b/drivers/net/ionic/ionic_dev.h @@ -142,6 +142,9 @@ struct ionic_desc_info { void *cb_arg; }; +#define Q_NEXT_TO_POST(_q, _n) (((_q)->head_idx + (_n)) & ((_q)->size_mask)) +#define Q_NEXT_TO_SRVC(_q, _n) (((_q)->tail_idx + (_n)) & ((_q)->size_mask)) + struct ionic_queue { struct ionic_dev *idev; struct ionic_lif *lif; @@ -174,11 +177,9 @@ struct ionic_intr_info { }; struct ionic_cq { - struct ionic_lif *lif; - struct ionic_queue *bound_q; - uint32_t tail_idx; - uint32_t num_descs; - uint32_t desc_size; + uint16_t tail_idx; + uint16_t num_descs; + uint16_t size_mask; bool done_color; void *base; rte_iova_t base_pa; @@ -240,8 +241,7 @@ void ionic_dev_cmd_adminq_init(struct ionic_dev *idev, struct ionic_qcq *qcq); struct ionic_doorbell __iomem *ionic_db_map(struct ionic_lif *lif, struct ionic_queue *q); -int ionic_cq_init(struct ionic_lif *lif, struct ionic_cq *cq, - uint32_t num_descs, size_t desc_size); +int ionic_cq_init(struct ionic_cq *cq, uint16_t num_descs); void ionic_cq_map(struct ionic_cq *cq, void *base, rte_iova_t base_pa); void ionic_cq_bind(struct ionic_cq *cq, struct ionic_queue *q); typedef bool (*ionic_cq_cb)(struct ionic_cq *cq, uint32_t cq_desc_index, diff --git a/drivers/net/ionic/ionic_lif.c b/drivers/net/ionic/ionic_lif.c index e083f92417..a9fe34c0f2 100644 --- a/drivers/net/ionic/ionic_lif.c +++ b/drivers/net/ionic/ionic_lif.c @@ -656,7 +656,7 @@ ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type, goto err_out_free_info; } - err = ionic_cq_init(lif, &new->cq, num_descs, cq_desc_size); + err = ionic_cq_init(&new->cq, num_descs); if (err) { IONIC_PRINT(ERR, "Completion queue initialization failed"); goto err_out_free_info; @@ -1169,11 +1169,12 @@ ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index, { struct ionic_admin_comp *cq_desc_base = cq->base; struct ionic_admin_comp *cq_desc = &cq_desc_base[cq_desc_index]; + struct ionic_qcq *qcq = IONIC_CQ_TO_QCQ(cq); if (!color_match(cq_desc->color, cq->done_color)) return false; - ionic_q_service(cq->bound_q, cq_desc_index, cq_desc->comp_index, NULL); + ionic_q_service(&qcq->q, cq_desc_index, cq_desc->comp_index, NULL); return true; } diff --git a/drivers/net/ionic/ionic_lif.h b/drivers/net/ionic/ionic_lif.h index 8f01aefd60..1a898a0ef9 100644 --- a/drivers/net/ionic/ionic_lif.h +++ b/drivers/net/ionic/ionic_lif.h @@ -71,7 +71,8 @@ struct ionic_qcq { struct ionic_intr_info intr; }; -#define IONIC_Q_TO_QCQ(q) container_of(q, struct ionic_qcq, q) +#define IONIC_Q_TO_QCQ(_q) container_of(_q, struct ionic_qcq, q) +#define IONIC_CQ_TO_QCQ(_cq) container_of(_cq, struct ionic_qcq, cq) #define IONIC_Q_TO_TX_STATS(q) (&IONIC_Q_TO_QCQ(q)->stats.tx) #define IONIC_Q_TO_RX_STATS(q) (&IONIC_Q_TO_QCQ(q)->stats.rx) diff --git a/drivers/net/ionic/ionic_rxtx.c b/drivers/net/ionic/ionic_rxtx.c index 5ae9ecf400..1ad95d8e39 100644 --- a/drivers/net/ionic/ionic_rxtx.c +++ b/drivers/net/ionic/ionic_rxtx.c @@ -68,9 +68,10 @@ ionic_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, } static __rte_always_inline void -ionic_tx_flush(struct ionic_cq *cq) +ionic_tx_flush(struct ionic_qcq *txq) { - struct ionic_queue *q = cq->bound_q; + struct ionic_cq *cq = &txq->cq; + struct ionic_queue *q = &txq->q; struct ionic_desc_info *q_desc_info; struct rte_mbuf *txm, *next; struct ionic_txq_comp *cq_desc_base = cq->base; @@ -79,7 +80,7 @@ ionic_tx_flush(struct ionic_cq *cq) cq_desc = &cq_desc_base[cq->tail_idx]; while (color_match(cq_desc->color, cq->done_color)) { - cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1); + cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1); /* Prefetch the next 4 descriptors (not really useful here) */ if ((cq->tail_idx & 0x3) == 0) @@ -149,7 +150,7 @@ ionic_dev_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id) ionic_qcq_disable(txq); - ionic_tx_flush(&txq->cq); + ionic_tx_flush(txq); return 0; } @@ -521,7 +522,6 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, { struct ionic_qcq *txq = (struct ionic_qcq *)tx_queue; struct ionic_queue *q = &txq->q; - struct ionic_cq *cq = &txq->cq; struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q); uint32_t next_q_head_idx; uint32_t bytes_tx = 0; @@ -530,7 +530,7 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, bool last; /* Cleaning old buffers */ - ionic_tx_flush(cq); + ionic_tx_flush(txq); if (unlikely(ionic_q_space_avail(q) < nb_pkts)) { stats->stop += nb_pkts; @@ -1018,10 +1018,11 @@ ionic_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) } static __rte_always_inline void -ionic_rxq_service(struct ionic_cq *cq, uint32_t work_to_do, +ionic_rxq_service(struct ionic_qcq *rxq, uint32_t work_to_do, void *service_cb_arg) { - struct ionic_queue *q = cq->bound_q; + struct ionic_cq *cq = &rxq->cq; + struct ionic_queue *q = &rxq->q; struct ionic_desc_info *q_desc_info; struct ionic_rxq_comp *cq_desc_base = cq->base; struct ionic_rxq_comp *cq_desc; @@ -1035,7 +1036,7 @@ ionic_rxq_service(struct ionic_cq *cq, uint32_t work_to_do, cq_desc = &cq_desc_base[cq->tail_idx]; while (color_match(cq_desc->pkt_type_color, cq->done_color)) { curr_cq_tail_idx = cq->tail_idx; - cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1); + cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1); if (cq->tail_idx == 0) cq->done_color = !cq->done_color; @@ -1087,7 +1088,7 @@ ionic_dev_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) ionic_qcq_disable(rxq); /* Flush */ - ionic_rxq_service(&rxq->cq, -1, NULL); + ionic_rxq_service(rxq, -1, NULL); return 0; } @@ -1099,14 +1100,13 @@ ionic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, struct ionic_qcq *rxq = (struct ionic_qcq *)rx_queue; uint32_t frame_size = rxq->lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; - struct ionic_cq *cq = &rxq->cq; struct ionic_rx_service service_cb_arg; service_cb_arg.rx_pkts = rx_pkts; service_cb_arg.nb_pkts = nb_pkts; service_cb_arg.nb_rx = 0; - ionic_rxq_service(cq, nb_pkts, &service_cb_arg); + ionic_rxq_service(rxq, nb_pkts, &service_cb_arg); ionic_rx_fill(rxq, frame_size); -- 2.17.1
The adminq is the only caller of ionic_q_service(), so absorb it into ionic_adminq_service(). Move all of the adminq code together into ionic_main.c. Signed-off-by: Andrew Boyer <aboyer@pensando.io> --- drivers/net/ionic/ionic.h | 13 +++++ drivers/net/ionic/ionic_dev.c | 85 ----------------------------- drivers/net/ionic/ionic_dev.h | 15 ----- drivers/net/ionic/ionic_lif.c | 16 ------ drivers/net/ionic/ionic_lif.h | 2 - drivers/net/ionic/ionic_main.c | 85 +++++++++++++++++++++++++++-- drivers/net/ionic/ionic_rx_filter.c | 1 + 7 files changed, 93 insertions(+), 124 deletions(-) diff --git a/drivers/net/ionic/ionic.h b/drivers/net/ionic/ionic.h index 49d1fc003f..fe75c191f5 100644 --- a/drivers/net/ionic/ionic.h +++ b/drivers/net/ionic/ionic.h @@ -65,8 +65,21 @@ struct ionic_adapter { LIST_ENTRY(ionic_adapter) pci_adapters; }; +/** ionic_admin_ctx - Admin command context. + * @pending_work: Flag that indicates a completion. + * @cmd: Admin command (64B) to be copied to the queue. + * @comp: Admin completion (16B) copied from the queue. + */ +struct ionic_admin_ctx { + bool pending_work; + union ionic_adminq_cmd cmd; + union ionic_adminq_comp comp; +}; + int ionic_adminq_check_err(struct ionic_admin_ctx *ctx, bool timeout); +int ionic_adminq_post(struct ionic_lif *lif, struct ionic_admin_ctx *ctx); int ionic_adminq_post_wait(struct ionic_lif *lif, struct ionic_admin_ctx *ctx); + int ionic_dev_cmd_wait_check(struct ionic_dev *idev, unsigned long max_wait); int ionic_setup(struct ionic_adapter *adapter); diff --git a/drivers/net/ionic/ionic_dev.c b/drivers/net/ionic/ionic_dev.c index f837817a0a..0eb9f6f21a 100644 --- a/drivers/net/ionic/ionic_dev.c +++ b/drivers/net/ionic/ionic_dev.c @@ -465,88 +465,3 @@ ionic_q_post(struct ionic_queue *q, bool ring_doorbell, desc_cb cb, if (ring_doorbell) ionic_q_flush(q); } - -void -ionic_q_service(struct ionic_queue *q, uint32_t cq_desc_index, - uint32_t stop_index, void *service_cb_arg) -{ - struct ionic_desc_info *desc_info; - uint32_t curr_q_tail_idx; - - do { - desc_info = &q->info[q->tail_idx]; - - if (desc_info->cb) - desc_info->cb(q, q->tail_idx, cq_desc_index, - desc_info->cb_arg, service_cb_arg); - - desc_info->cb = NULL; - desc_info->cb_arg = NULL; - - curr_q_tail_idx = q->tail_idx; - q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); - - } while (curr_q_tail_idx != stop_index); -} - -static void -ionic_adminq_cb(struct ionic_queue *q, - uint32_t q_desc_index, uint32_t cq_desc_index, - void *cb_arg, void *service_cb_arg __rte_unused) -{ - struct ionic_admin_ctx *ctx = cb_arg; - struct ionic_admin_comp *cq_desc_base = q->bound_cq->base; - struct ionic_admin_comp *cq_desc = &cq_desc_base[cq_desc_index]; - uint16_t comp_index; - - if (!ctx) - return; - - comp_index = rte_le_to_cpu_16(cq_desc->comp_index); - if (unlikely(comp_index != q_desc_index)) { - IONIC_WARN_ON(comp_index != q_desc_index); - return; - } - - memcpy(&ctx->comp, cq_desc, sizeof(*cq_desc)); - - ctx->pending_work = false; /* done */ -} - -/** ionic_adminq_post - Post an admin command. - * @lif: Handle to lif. - * @cmd_ctx: Api admin command context. - * - * Post the command to an admin queue in the ethernet driver. If this command - * succeeds, then the command has been posted, but that does not indicate a - * completion. If this command returns success, then the completion callback - * will eventually be called. - * - * Return: zero or negative error status. - */ -int -ionic_adminq_post(struct ionic_lif *lif, struct ionic_admin_ctx *ctx) -{ - struct ionic_queue *adminq = &lif->adminqcq->q; - struct ionic_admin_cmd *q_desc_base = adminq->base; - struct ionic_admin_cmd *q_desc; - int err = 0; - - rte_spinlock_lock(&lif->adminq_lock); - - if (ionic_q_space_avail(adminq) < 1) { - err = -ENOSPC; - goto err_out; - } - - q_desc = &q_desc_base[adminq->head_idx]; - - memcpy(q_desc, &ctx->cmd, sizeof(ctx->cmd)); - - ionic_q_post(adminq, true, ionic_adminq_cb, ctx); - -err_out: - rte_spinlock_unlock(&lif->adminq_lock); - - return err; -} diff --git a/drivers/net/ionic/ionic_dev.h b/drivers/net/ionic/ionic_dev.h index b29bfd13be..70d95af6bb 100644 --- a/drivers/net/ionic/ionic_dev.h +++ b/drivers/net/ionic/ionic_dev.h @@ -185,17 +185,6 @@ struct ionic_cq { rte_iova_t base_pa; }; -/** ionic_admin_ctx - Admin command context. - * @pending_work: Flag that indicates a completion. - * @cmd: Admin command (64B) to be copied to the queue. - * @comp: Admin completion (16B) copied from the queue. - */ -struct ionic_admin_ctx { - bool pending_work; - union ionic_adminq_cmd cmd; - union ionic_adminq_comp comp; -}; - struct ionic_lif; struct ionic_adapter; struct ionic_qcq; @@ -256,8 +245,6 @@ void ionic_q_map(struct ionic_queue *q, void *base, rte_iova_t base_pa); void ionic_q_sg_map(struct ionic_queue *q, void *base, rte_iova_t base_pa); void ionic_q_post(struct ionic_queue *q, bool ring_doorbell, desc_cb cb, void *cb_arg); -void ionic_q_service(struct ionic_queue *q, uint32_t cq_desc_index, - uint32_t stop_index, void *service_cb_arg); static inline uint32_t ionic_q_space_avail(struct ionic_queue *q) @@ -280,6 +267,4 @@ ionic_q_flush(struct ionic_queue *q) rte_write64(rte_cpu_to_le_64(val), q->db); } -int ionic_adminq_post(struct ionic_lif *lif, struct ionic_admin_ctx *ctx); - #endif /* _IONIC_DEV_H_ */ diff --git a/drivers/net/ionic/ionic_lif.c b/drivers/net/ionic/ionic_lif.c index a9fe34c0f2..52bc0b178d 100644 --- a/drivers/net/ionic/ionic_lif.c +++ b/drivers/net/ionic/ionic_lif.c @@ -1163,22 +1163,6 @@ ionic_lif_notifyq_deinit(struct ionic_lif *lif) nqcq->flags &= ~IONIC_QCQ_F_INITED; } -bool -ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index, - void *cb_arg __rte_unused) -{ - struct ionic_admin_comp *cq_desc_base = cq->base; - struct ionic_admin_comp *cq_desc = &cq_desc_base[cq_desc_index]; - struct ionic_qcq *qcq = IONIC_CQ_TO_QCQ(cq); - - if (!color_match(cq_desc->color, cq->done_color)) - return false; - - ionic_q_service(&qcq->q, cq_desc_index, cq_desc->comp_index, NULL); - - return true; -} - /* This acts like ionic_napi */ int ionic_qcq_service(struct ionic_qcq *qcq, int budget, ionic_cq_cb cb, diff --git a/drivers/net/ionic/ionic_lif.h b/drivers/net/ionic/ionic_lif.h index 1a898a0ef9..c4cb7514fb 100644 --- a/drivers/net/ionic/ionic_lif.h +++ b/drivers/net/ionic/ionic_lif.h @@ -154,8 +154,6 @@ void ionic_lif_reset(struct ionic_lif *lif); int ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr); void ionic_intr_free(struct ionic_lif *lif, struct ionic_intr_info *intr); -bool ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index, - void *cb_arg); int ionic_qcq_service(struct ionic_qcq *qcq, int budget, ionic_cq_cb cb, void *cb_arg); diff --git a/drivers/net/ionic/ionic_main.c b/drivers/net/ionic/ionic_main.c index 3f1a764888..acc8b87b64 100644 --- a/drivers/net/ionic/ionic_main.c +++ b/drivers/net/ionic/ionic_main.c @@ -145,8 +145,81 @@ ionic_adminq_check_err(struct ionic_admin_ctx *ctx, bool timeout) return 0; } +static bool +ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index, + void *cb_arg __rte_unused) +{ + struct ionic_admin_comp *cq_desc_base = cq->base; + struct ionic_admin_comp *cq_desc = &cq_desc_base[cq_desc_index]; + struct ionic_qcq *qcq = IONIC_CQ_TO_QCQ(cq); + struct ionic_queue *q = &qcq->q; + struct ionic_admin_ctx *ctx; + struct ionic_desc_info *desc_info; + uint16_t curr_q_tail_idx; + uint16_t stop_index; + + if (!color_match(cq_desc->color, cq->done_color)) + return false; + + stop_index = rte_le_to_cpu_16(cq_desc->comp_index); + + do { + desc_info = &q->info[q->tail_idx]; + + ctx = desc_info->cb_arg; + if (ctx) { + memcpy(&ctx->comp, cq_desc, sizeof(*cq_desc)); + + ctx->pending_work = false; /* done */ + } + + curr_q_tail_idx = q->tail_idx; + q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); + } while (curr_q_tail_idx != stop_index); + + return true; +} + +/** ionic_adminq_post - Post an admin command. + * @lif: Handle to lif. + * @cmd_ctx: Api admin command context. + * + * Post the command to an admin queue in the ethernet driver. If this command + * succeeds, then the command has been posted, but that does not indicate a + * completion. If this command returns success, then the completion callback + * will eventually be called. + * + * Return: zero or negative error status. + */ +int +ionic_adminq_post(struct ionic_lif *lif, struct ionic_admin_ctx *ctx) +{ + struct ionic_queue *q = &lif->adminqcq->q; + struct ionic_admin_cmd *q_desc_base = q->base; + struct ionic_admin_cmd *q_desc; + int err = 0; + + rte_spinlock_lock(&lif->adminq_lock); + + if (ionic_q_space_avail(q) < 1) { + err = -ENOSPC; + goto err_out; + } + + q_desc = &q_desc_base[q->head_idx]; + + memcpy(q_desc, &ctx->cmd, sizeof(ctx->cmd)); + + ionic_q_post(q, true, NULL, ctx); + +err_out: + rte_spinlock_unlock(&lif->adminq_lock); + + return err; +} + static int -ionic_wait_ctx_for_completion(struct ionic_lif *lif, struct ionic_qcq *qcq, +ionic_adminq_wait_for_completion(struct ionic_lif *lif, struct ionic_admin_ctx *ctx, unsigned long max_wait) { unsigned long step_usec = IONIC_DEVCMD_CHECK_PERIOD_US; @@ -156,12 +229,13 @@ ionic_wait_ctx_for_completion(struct ionic_lif *lif, struct ionic_qcq *qcq, while (ctx->pending_work && elapsed_usec < max_wait_usec) { /* - * Locking here as adminq is served inline (this could be called - * from multiple places) + * Locking here as adminq is served inline and could be + * called from multiple places */ rte_spinlock_lock(&lif->adminq_service_lock); - ionic_qcq_service(qcq, budget, ionic_adminq_service, NULL); + ionic_qcq_service(lif->adminqcq, budget, + ionic_adminq_service, NULL); rte_spinlock_unlock(&lif->adminq_service_lock); @@ -175,7 +249,6 @@ ionic_wait_ctx_for_completion(struct ionic_lif *lif, struct ionic_qcq *qcq, int ionic_adminq_post_wait(struct ionic_lif *lif, struct ionic_admin_ctx *ctx) { - struct ionic_qcq *qcq = lif->adminqcq; bool done; int err; @@ -189,7 +262,7 @@ ionic_adminq_post_wait(struct ionic_lif *lif, struct ionic_admin_ctx *ctx) return err; } - done = ionic_wait_ctx_for_completion(lif, qcq, ctx, + done = ionic_adminq_wait_for_completion(lif, ctx, IONIC_DEVCMD_TIMEOUT); return ionic_adminq_check_err(ctx, !done /* timed out */); diff --git a/drivers/net/ionic/ionic_rx_filter.c b/drivers/net/ionic/ionic_rx_filter.c index 320b9019b3..02a8e04e4e 100644 --- a/drivers/net/ionic/ionic_rx_filter.c +++ b/drivers/net/ionic/ionic_rx_filter.c @@ -7,6 +7,7 @@ #include <rte_malloc.h> +#include "ionic.h" #include "ionic_lif.h" #include "ionic_rx_filter.h" -- 2.17.1
Drop the callback part of the object and store only the pointers. This saves a bit of space and simplifies the code. Signed-off-by: Andrew Boyer <aboyer@pensando.io> --- drivers/net/ionic/ionic_dev.c | 8 ++------ drivers/net/ionic/ionic_dev.h | 21 +++++-------------- drivers/net/ionic/ionic_main.c | 8 ++++---- drivers/net/ionic/ionic_rxtx.c | 37 ++++++++++++++++++---------------- 4 files changed, 31 insertions(+), 43 deletions(-) diff --git a/drivers/net/ionic/ionic_dev.c b/drivers/net/ionic/ionic_dev.c index 0eb9f6f21a..74ac2e67a5 100644 --- a/drivers/net/ionic/ionic_dev.c +++ b/drivers/net/ionic/ionic_dev.c @@ -452,13 +452,9 @@ ionic_q_sg_map(struct ionic_queue *q, void *base, rte_iova_t base_pa) } void -ionic_q_post(struct ionic_queue *q, bool ring_doorbell, desc_cb cb, - void *cb_arg) +ionic_q_post(struct ionic_queue *q, bool ring_doorbell, void *cb_arg) { - struct ionic_desc_info *head = &q->info[q->head_idx]; - - head->cb = cb; - head->cb_arg = cb_arg; + q->info[q->head_idx] = cb_arg; q->head_idx = (q->head_idx + 1) & (q->num_descs - 1); diff --git a/drivers/net/ionic/ionic_dev.h b/drivers/net/ionic/ionic_dev.h index 70d95af6bb..1a43ccc61f 100644 --- a/drivers/net/ionic/ionic_dev.h +++ b/drivers/net/ionic/ionic_dev.h @@ -129,22 +129,12 @@ struct ionic_dev { uint32_t port_info_sz; }; -struct ionic_queue; -struct ionic_desc_info; - -typedef void (*desc_cb)(struct ionic_queue *q, - uint32_t q_desc_index, - uint32_t cq_desc_index, - void *cb_arg, void *service_cb_arg); - -struct ionic_desc_info { - desc_cb cb; - void *cb_arg; -}; - #define Q_NEXT_TO_POST(_q, _n) (((_q)->head_idx + (_n)) & ((_q)->size_mask)) #define Q_NEXT_TO_SRVC(_q, _n) (((_q)->tail_idx + (_n)) & ((_q)->size_mask)) +#define IONIC_INFO_IDX(_q, _i) (_i) +#define IONIC_INFO_PTR(_q, _i) (&(_q)->info[IONIC_INFO_IDX((_q), _i)]) + struct ionic_queue { struct ionic_dev *idev; struct ionic_lif *lif; @@ -155,9 +145,9 @@ struct ionic_queue { uint32_t hw_type; void *base; void *sg_base; + void **info; rte_iova_t base_pa; rte_iova_t sg_base_pa; - struct ionic_desc_info *info; uint32_t tail_idx; uint32_t head_idx; uint32_t num_descs; @@ -243,8 +233,7 @@ int ionic_q_init(struct ionic_lif *lif, struct ionic_dev *idev, size_t desc_size, size_t sg_desc_size); void ionic_q_map(struct ionic_queue *q, void *base, rte_iova_t base_pa); void ionic_q_sg_map(struct ionic_queue *q, void *base, rte_iova_t base_pa); -void ionic_q_post(struct ionic_queue *q, bool ring_doorbell, desc_cb cb, - void *cb_arg); +void ionic_q_post(struct ionic_queue *q, bool ring_doorbell, void *cb_arg); static inline uint32_t ionic_q_space_avail(struct ionic_queue *q) diff --git a/drivers/net/ionic/ionic_main.c b/drivers/net/ionic/ionic_main.c index acc8b87b64..c9c0123d6a 100644 --- a/drivers/net/ionic/ionic_main.c +++ b/drivers/net/ionic/ionic_main.c @@ -154,9 +154,9 @@ ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index, struct ionic_qcq *qcq = IONIC_CQ_TO_QCQ(cq); struct ionic_queue *q = &qcq->q; struct ionic_admin_ctx *ctx; - struct ionic_desc_info *desc_info; uint16_t curr_q_tail_idx; uint16_t stop_index; + void **info; if (!color_match(cq_desc->color, cq->done_color)) return false; @@ -164,9 +164,9 @@ ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index, stop_index = rte_le_to_cpu_16(cq_desc->comp_index); do { - desc_info = &q->info[q->tail_idx]; + info = IONIC_INFO_PTR(q, q->tail_idx); - ctx = desc_info->cb_arg; + ctx = info[0]; if (ctx) { memcpy(&ctx->comp, cq_desc, sizeof(*cq_desc)); @@ -210,7 +210,7 @@ ionic_adminq_post(struct ionic_lif *lif, struct ionic_admin_ctx *ctx) memcpy(q_desc, &ctx->cmd, sizeof(ctx->cmd)); - ionic_q_post(q, true, NULL, ctx); + ionic_q_post(q, true, ctx); err_out: rte_spinlock_unlock(&lif->adminq_lock); diff --git a/drivers/net/ionic/ionic_rxtx.c b/drivers/net/ionic/ionic_rxtx.c index 1ad95d8e39..81c9ff8ba8 100644 --- a/drivers/net/ionic/ionic_rxtx.c +++ b/drivers/net/ionic/ionic_rxtx.c @@ -72,10 +72,10 @@ ionic_tx_flush(struct ionic_qcq *txq) { struct ionic_cq *cq = &txq->cq; struct ionic_queue *q = &txq->q; - struct ionic_desc_info *q_desc_info; struct rte_mbuf *txm, *next; struct ionic_txq_comp *cq_desc_base = cq->base; struct ionic_txq_comp *cq_desc; + void **info; u_int32_t comp_index = (u_int32_t)-1; cq_desc = &cq_desc_base[cq->tail_idx]; @@ -96,7 +96,7 @@ ionic_tx_flush(struct ionic_qcq *txq) if (comp_index != (u_int32_t)-1) { while (q->tail_idx != comp_index) { - q_desc_info = &q->info[q->tail_idx]; + info = IONIC_INFO_PTR(q, q->tail_idx); q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); @@ -109,7 +109,7 @@ ionic_tx_flush(struct ionic_qcq *txq) * Note: you can just use rte_pktmbuf_free, * but this loop is faster */ - txm = q_desc_info->cb_arg; + txm = info[0]; while (txm != NULL) { next = txm->next; rte_pktmbuf_free_seg(txm); @@ -311,7 +311,7 @@ ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc, desc->hdr_len = hdrlen; desc->mss = mss; - ionic_q_post(q, done, NULL, done ? txm : NULL); + ionic_q_post(q, done, done ? txm : NULL); } static struct ionic_txq_desc * @@ -511,7 +511,7 @@ ionic_tx(struct ionic_qcq *txq, struct rte_mbuf *txm, txm_seg = txm_seg->next; } - ionic_q_post(q, not_xmit_more, NULL, txm); + ionic_q_post(q, not_xmit_more, txm); return 0; } @@ -639,12 +639,12 @@ static void __rte_cold ionic_rx_empty(struct ionic_queue *q) { struct ionic_qcq *rxq = IONIC_Q_TO_QCQ(q); - struct ionic_desc_info *cur; struct rte_mbuf *mbuf; + void **info; while (q->tail_idx != q->head_idx) { - cur = &q->info[q->tail_idx]; - mbuf = cur->cb_arg; + info = IONIC_INFO_PTR(q, q->tail_idx); + mbuf = info[0]; rte_mempool_put(rxq->mb_pool, mbuf); q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); @@ -743,12 +743,11 @@ ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, static __rte_always_inline void ionic_rx_clean(struct ionic_queue *q, uint32_t q_desc_index, uint32_t cq_desc_index, - void *cb_arg, void *service_cb_arg) + void *service_cb_arg) { struct ionic_rxq_comp *cq_desc_base = q->bound_cq->base; struct ionic_rxq_comp *cq_desc = &cq_desc_base[cq_desc_index]; - struct rte_mbuf *rxm = cb_arg; - struct rte_mbuf *rxm_seg; + struct rte_mbuf *rxm, *rxm_seg; struct ionic_qcq *rxq = IONIC_Q_TO_QCQ(q); uint32_t max_frame_size = rxq->lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; @@ -761,6 +760,13 @@ ionic_rx_clean(struct ionic_queue *q, (rte_pktmbuf_data_room_size(rxq->mb_pool) - RTE_PKTMBUF_HEADROOM); uint32_t left; + void **info; + + assert(q_desc_index == cq_desc->comp_index); + + info = IONIC_INFO_PTR(q, cq_desc->comp_index); + + rxm = info[0]; if (!recv_args) { stats->no_cb_arg++; @@ -898,7 +904,7 @@ ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index, new->addr = old->addr; new->len = old->len; - ionic_q_post(q, true, ionic_rx_clean, mbuf); + ionic_q_post(q, true, mbuf); } static __rte_always_inline int @@ -969,7 +975,7 @@ ionic_rx_fill(struct ionic_qcq *rxq, uint32_t len) ring_doorbell = ((q->head_idx + 1) & IONIC_RX_RING_DOORBELL_STRIDE) == 0; - ionic_q_post(q, ring_doorbell, ionic_rx_clean, rxm); + ionic_q_post(q, ring_doorbell, rxm); } return 0; @@ -1023,7 +1029,6 @@ ionic_rxq_service(struct ionic_qcq *rxq, uint32_t work_to_do, { struct ionic_cq *cq = &rxq->cq; struct ionic_queue *q = &rxq->q; - struct ionic_desc_info *q_desc_info; struct ionic_rxq_comp *cq_desc_base = cq->base; struct ionic_rxq_comp *cq_desc; bool more; @@ -1048,8 +1053,6 @@ ionic_rxq_service(struct ionic_qcq *rxq, uint32_t work_to_do, do { more = (q->tail_idx != cq_desc->comp_index); - q_desc_info = &q->info[q->tail_idx]; - curr_q_tail_idx = q->tail_idx; q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); @@ -1059,7 +1062,7 @@ ionic_rxq_service(struct ionic_qcq *rxq, uint32_t work_to_do, rte_prefetch0(&q->info[q->tail_idx]); ionic_rx_clean(q, curr_q_tail_idx, curr_cq_tail_idx, - q_desc_info->cb_arg, service_cb_arg); + service_cb_arg); } while (more); -- 2.17.1
This will conserve resources. Signed-off-by: Andrew Boyer <aboyer@pensando.io> --- drivers/net/ionic/ionic_dev.c | 6 ------ drivers/net/ionic/ionic_dev.h | 2 -- drivers/net/ionic/ionic_lif.c | 1 - drivers/net/ionic/ionic_rxtx.c | 9 +++++---- 4 files changed, 5 insertions(+), 13 deletions(-) diff --git a/drivers/net/ionic/ionic_dev.c b/drivers/net/ionic/ionic_dev.c index 74ac2e67a5..97fb8acf9b 100644 --- a/drivers/net/ionic/ionic_dev.c +++ b/drivers/net/ionic/ionic_dev.c @@ -383,12 +383,6 @@ ionic_cq_map(struct ionic_cq *cq, void *base, rte_iova_t base_pa) cq->base_pa = base_pa; } -void -ionic_cq_bind(struct ionic_cq *cq, struct ionic_queue *q) -{ - q->bound_cq = cq; -} - uint32_t ionic_cq_service(struct ionic_cq *cq, uint32_t work_to_do, ionic_cq_cb cb, void *cb_arg) diff --git a/drivers/net/ionic/ionic_dev.h b/drivers/net/ionic/ionic_dev.h index 1a43ccc61f..511d0bc7aa 100644 --- a/drivers/net/ionic/ionic_dev.h +++ b/drivers/net/ionic/ionic_dev.h @@ -138,7 +138,6 @@ struct ionic_dev { struct ionic_queue { struct ionic_dev *idev; struct ionic_lif *lif; - struct ionic_cq *bound_cq; uint32_t index; uint32_t type; uint32_t hw_index; @@ -222,7 +221,6 @@ struct ionic_doorbell __iomem *ionic_db_map(struct ionic_lif *lif, int ionic_cq_init(struct ionic_cq *cq, uint16_t num_descs); void ionic_cq_map(struct ionic_cq *cq, void *base, rte_iova_t base_pa); -void ionic_cq_bind(struct ionic_cq *cq, struct ionic_queue *q); typedef bool (*ionic_cq_cb)(struct ionic_cq *cq, uint32_t cq_desc_index, void *cb_arg); uint32_t ionic_cq_service(struct ionic_cq *cq, uint32_t work_to_do, diff --git a/drivers/net/ionic/ionic_lif.c b/drivers/net/ionic/ionic_lif.c index 52bc0b178d..1ca6e050ad 100644 --- a/drivers/net/ionic/ionic_lif.c +++ b/drivers/net/ionic/ionic_lif.c @@ -695,7 +695,6 @@ ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type, ionic_q_map(&new->q, q_base, q_base_pa); ionic_cq_map(&new->cq, cq_base, cq_base_pa); - ionic_cq_bind(&new->cq, &new->q); *qcq = new; diff --git a/drivers/net/ionic/ionic_rxtx.c b/drivers/net/ionic/ionic_rxtx.c index 81c9ff8ba8..107b1cb091 100644 --- a/drivers/net/ionic/ionic_rxtx.c +++ b/drivers/net/ionic/ionic_rxtx.c @@ -741,14 +741,15 @@ ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, } static __rte_always_inline void -ionic_rx_clean(struct ionic_queue *q, +ionic_rx_clean(struct ionic_qcq *rxq, uint32_t q_desc_index, uint32_t cq_desc_index, void *service_cb_arg) { - struct ionic_rxq_comp *cq_desc_base = q->bound_cq->base; + struct ionic_queue *q = &rxq->q; + struct ionic_cq *cq = &rxq->cq; + struct ionic_rxq_comp *cq_desc_base = cq->base; struct ionic_rxq_comp *cq_desc = &cq_desc_base[cq_desc_index]; struct rte_mbuf *rxm, *rxm_seg; - struct ionic_qcq *rxq = IONIC_Q_TO_QCQ(q); uint32_t max_frame_size = rxq->lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; uint64_t pkt_flags = 0; @@ -1061,7 +1062,7 @@ ionic_rxq_service(struct ionic_qcq *rxq, uint32_t work_to_do, /* q desc info */ rte_prefetch0(&q->info[q->tail_idx]); - ionic_rx_clean(q, curr_q_tail_idx, curr_cq_tail_idx, + ionic_rx_clean(rxq, curr_q_tail_idx, curr_cq_tail_idx, service_cb_arg); } while (more); -- 2.17.1
This will conserve resources. Signed-off-by: Andrew Boyer <aboyer@pensando.io> --- drivers/net/ionic/ionic_lif.c | 7 ------- drivers/net/ionic/ionic_lif.h | 1 - 2 files changed, 8 deletions(-) diff --git a/drivers/net/ionic/ionic_lif.c b/drivers/net/ionic/ionic_lif.c index 1ca6e050ad..462a526935 100644 --- a/drivers/net/ionic/ionic_lif.c +++ b/drivers/net/ionic/ionic_lif.c @@ -584,13 +584,6 @@ ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr) return 0; } -void -ionic_intr_free(struct ionic_lif *lif, struct ionic_intr_info *intr) -{ - if (intr->index != IONIC_INTR_NONE) - lif->adapter->intrs[intr->index] = false; -} - static int ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type, uint32_t index, diff --git a/drivers/net/ionic/ionic_lif.h b/drivers/net/ionic/ionic_lif.h index c4cb7514fb..c850a9c08d 100644 --- a/drivers/net/ionic/ionic_lif.h +++ b/drivers/net/ionic/ionic_lif.h @@ -152,7 +152,6 @@ void ionic_lif_configure_vlan_offload(struct ionic_lif *lif, int mask); void ionic_lif_reset(struct ionic_lif *lif); int ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr); -void ionic_intr_free(struct ionic_lif *lif, struct ionic_intr_info *intr); int ionic_qcq_service(struct ionic_qcq *qcq, int budget, ionic_cq_cb cb, void *cb_arg); -- 2.17.1
This will conserve resources. Rename ionic_qcq_alloc() arg from 'base' to 'type_name' for clarity. Signed-off-by: Andrew Boyer <aboyer@pensando.io> --- drivers/net/ionic/ionic_dev.c | 14 +++------- drivers/net/ionic/ionic_dev.h | 34 ++++++++++------------- drivers/net/ionic/ionic_lif.c | 49 ++++++++++++++++++++++------------ drivers/net/ionic/ionic_main.c | 4 +-- drivers/net/ionic/ionic_rxtx.c | 8 +++--- 5 files changed, 56 insertions(+), 53 deletions(-) diff --git a/drivers/net/ionic/ionic_dev.c b/drivers/net/ionic/ionic_dev.c index 97fb8acf9b..cfaf4abc23 100644 --- a/drivers/net/ionic/ionic_dev.c +++ b/drivers/net/ionic/ionic_dev.c @@ -405,26 +405,20 @@ ionic_cq_service(struct ionic_cq *cq, uint32_t work_to_do, } int -ionic_q_init(struct ionic_lif *lif, struct ionic_dev *idev, - struct ionic_queue *q, uint32_t index, uint32_t num_descs, - size_t desc_size, size_t sg_desc_size) +ionic_q_init(struct ionic_queue *q, uint32_t index, uint16_t num_descs) { uint32_t ring_size; - if (desc_size == 0 || !rte_is_power_of_2(num_descs)) + if (!rte_is_power_of_2(num_descs)) return -EINVAL; ring_size = rte_log2_u32(num_descs); - if (ring_size < 2 || ring_size > 16) return -EINVAL; - q->lif = lif; - q->idev = idev; q->index = index; q->num_descs = num_descs; - q->desc_size = desc_size; - q->sg_desc_size = sg_desc_size; + q->size_mask = num_descs - 1; q->head_idx = 0; q->tail_idx = 0; @@ -450,7 +444,7 @@ ionic_q_post(struct ionic_queue *q, bool ring_doorbell, void *cb_arg) { q->info[q->head_idx] = cb_arg; - q->head_idx = (q->head_idx + 1) & (q->num_descs - 1); + q->head_idx = Q_NEXT_TO_POST(q, 1); if (ring_doorbell) ionic_q_flush(q); diff --git a/drivers/net/ionic/ionic_dev.h b/drivers/net/ionic/ionic_dev.h index 511d0bc7aa..0d1aff776b 100644 --- a/drivers/net/ionic/ionic_dev.h +++ b/drivers/net/ionic/ionic_dev.h @@ -136,25 +136,21 @@ struct ionic_dev { #define IONIC_INFO_PTR(_q, _i) (&(_q)->info[IONIC_INFO_IDX((_q), _i)]) struct ionic_queue { - struct ionic_dev *idev; - struct ionic_lif *lif; - uint32_t index; - uint32_t type; - uint32_t hw_index; - uint32_t hw_type; + uint16_t num_descs; + uint16_t head_idx; + uint16_t tail_idx; + uint16_t size_mask; + uint8_t type; + uint8_t hw_type; void *base; void *sg_base; + struct ionic_doorbell __iomem *db; void **info; + + uint32_t index; + uint32_t hw_index; rte_iova_t base_pa; rte_iova_t sg_base_pa; - uint32_t tail_idx; - uint32_t head_idx; - uint32_t num_descs; - uint32_t desc_size; - uint32_t sg_desc_size; - uint32_t qid; - uint32_t qtype; - struct ionic_doorbell __iomem *db; }; #define IONIC_INTR_NONE (-1) @@ -221,22 +217,20 @@ struct ionic_doorbell __iomem *ionic_db_map(struct ionic_lif *lif, int ionic_cq_init(struct ionic_cq *cq, uint16_t num_descs); void ionic_cq_map(struct ionic_cq *cq, void *base, rte_iova_t base_pa); -typedef bool (*ionic_cq_cb)(struct ionic_cq *cq, uint32_t cq_desc_index, +typedef bool (*ionic_cq_cb)(struct ionic_cq *cq, uint16_t cq_desc_index, void *cb_arg); uint32_t ionic_cq_service(struct ionic_cq *cq, uint32_t work_to_do, ionic_cq_cb cb, void *cb_arg); -int ionic_q_init(struct ionic_lif *lif, struct ionic_dev *idev, - struct ionic_queue *q, uint32_t index, uint32_t num_descs, - size_t desc_size, size_t sg_desc_size); +int ionic_q_init(struct ionic_queue *q, uint32_t index, uint16_t num_descs); void ionic_q_map(struct ionic_queue *q, void *base, rte_iova_t base_pa); void ionic_q_sg_map(struct ionic_queue *q, void *base, rte_iova_t base_pa); void ionic_q_post(struct ionic_queue *q, bool ring_doorbell, void *cb_arg); -static inline uint32_t +static inline uint16_t ionic_q_space_avail(struct ionic_queue *q) { - uint32_t avail = q->tail_idx; + uint16_t avail = q->tail_idx; if (q->head_idx >= avail) avail += q->num_descs - q->head_idx - 1; diff --git a/drivers/net/ionic/ionic_lif.c b/drivers/net/ionic/ionic_lif.c index 462a526935..fcd6fca9a3 100644 --- a/drivers/net/ionic/ionic_lif.c +++ b/drivers/net/ionic/ionic_lif.c @@ -34,7 +34,7 @@ int ionic_qcq_enable(struct ionic_qcq *qcq) { struct ionic_queue *q = &qcq->q; - struct ionic_lif *lif = q->lif; + struct ionic_lif *lif = qcq->lif; struct ionic_admin_ctx ctx = { .pending_work = true, .cmd.q_control = { @@ -52,7 +52,7 @@ int ionic_qcq_disable(struct ionic_qcq *qcq) { struct ionic_queue *q = &qcq->q; - struct ionic_lif *lif = q->lif; + struct ionic_lif *lif = qcq->lif; struct ionic_admin_ctx ctx = { .pending_work = true, .cmd.q_control = { @@ -585,16 +585,17 @@ ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr) } static int -ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type, +ionic_qcq_alloc(struct ionic_lif *lif, + uint8_t type, uint32_t index, - const char *base, uint32_t flags, - uint32_t num_descs, - uint32_t desc_size, - uint32_t cq_desc_size, - uint32_t sg_desc_size, + const char *type_name, + uint16_t flags, + uint16_t num_descs, + uint16_t desc_size, + uint16_t cq_desc_size, + uint16_t sg_desc_size, struct ionic_qcq **qcq) { - struct ionic_dev *idev = &lif->adapter->idev; struct ionic_qcq *new; uint32_t q_size, cq_size, sg_size, total_size; void *q_base, *cq_base, *sg_base; @@ -642,8 +643,7 @@ ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type, new->q.type = type; - err = ionic_q_init(lif, idev, &new->q, index, num_descs, - desc_size, sg_desc_size); + err = ionic_q_init(&new->q, index, num_descs); if (err) { IONIC_PRINT(ERR, "Queue initialization failed"); goto err_out_free_info; @@ -656,7 +656,7 @@ ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type, } new->base_z = rte_eth_dma_zone_reserve(lif->eth_dev, - base /* name */, index /* queue_idx */, + type_name, index /* queue_idx */, total_size, IONIC_ALIGN, socket_id); if (!new->base_z) { @@ -727,7 +727,11 @@ ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t nrxq_descs, int err = -ENOMEM; flags = IONIC_QCQ_F_SG; - err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, index, "rx", flags, + err = ionic_qcq_alloc(lif, + IONIC_QTYPE_RXQ, + index, + "rx", + flags, nrxq_descs, sizeof(struct ionic_rxq_desc), sizeof(struct ionic_rxq_comp), @@ -749,7 +753,11 @@ ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t ntxq_descs, int err = -ENOMEM; flags = IONIC_QCQ_F_SG; - err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, index, "tx", flags, + err = ionic_qcq_alloc(lif, + IONIC_QTYPE_TXQ, + index, + "tx", + flags, ntxq_descs, sizeof(struct ionic_txq_desc), sizeof(struct ionic_txq_comp), @@ -770,7 +778,11 @@ ionic_admin_qcq_alloc(struct ionic_lif *lif) int err = -ENOMEM; flags = 0; - err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags, + err = ionic_qcq_alloc(lif, + IONIC_QTYPE_ADMINQ, + 0, + "admin", + flags, IONIC_ADMINQ_LENGTH, sizeof(struct ionic_admin_cmd), sizeof(struct ionic_admin_comp), @@ -790,7 +802,10 @@ ionic_notify_qcq_alloc(struct ionic_lif *lif) uint32_t flags = 0; int err = -ENOMEM; - err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notify", + err = ionic_qcq_alloc(lif, + IONIC_QTYPE_NOTIFYQ, + 0, + "notify", flags, IONIC_NOTIFYQ_LENGTH, sizeof(struct ionic_notifyq_cmd), @@ -1216,7 +1231,7 @@ ionic_lif_handle_fw_down(struct ionic_lif *lif) } static bool -ionic_notifyq_cb(struct ionic_cq *cq, uint32_t cq_desc_index, void *cb_arg) +ionic_notifyq_cb(struct ionic_cq *cq, uint16_t cq_desc_index, void *cb_arg) { union ionic_notifyq_comp *cq_desc_base = cq->base; union ionic_notifyq_comp *cq_desc = &cq_desc_base[cq_desc_index]; diff --git a/drivers/net/ionic/ionic_main.c b/drivers/net/ionic/ionic_main.c index c9c0123d6a..b358a76f06 100644 --- a/drivers/net/ionic/ionic_main.c +++ b/drivers/net/ionic/ionic_main.c @@ -146,7 +146,7 @@ ionic_adminq_check_err(struct ionic_admin_ctx *ctx, bool timeout) } static bool -ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index, +ionic_adminq_service(struct ionic_cq *cq, uint16_t cq_desc_index, void *cb_arg __rte_unused) { struct ionic_admin_comp *cq_desc_base = cq->base; @@ -174,7 +174,7 @@ ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index, } curr_q_tail_idx = q->tail_idx; - q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); + q->tail_idx = Q_NEXT_TO_SRVC(q, 1); } while (curr_q_tail_idx != stop_index); return true; diff --git a/drivers/net/ionic/ionic_rxtx.c b/drivers/net/ionic/ionic_rxtx.c index 107b1cb091..ce38651559 100644 --- a/drivers/net/ionic/ionic_rxtx.c +++ b/drivers/net/ionic/ionic_rxtx.c @@ -98,7 +98,7 @@ ionic_tx_flush(struct ionic_qcq *txq) while (q->tail_idx != comp_index) { info = IONIC_INFO_PTR(q, q->tail_idx); - q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); + q->tail_idx = Q_NEXT_TO_SRVC(q, 1); /* Prefetch the next 4 descriptors */ if ((q->tail_idx & 0x3) == 0) @@ -540,7 +540,7 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, while (nb_tx < nb_pkts) { last = (nb_tx == (nb_pkts - 1)); - next_q_head_idx = (q->head_idx + 1) & (q->num_descs - 1); + next_q_head_idx = Q_NEXT_TO_POST(q, 1); if ((next_q_head_idx & 0x3) == 0) { struct ionic_txq_desc *desc_base = q->base; rte_prefetch0(&desc_base[next_q_head_idx]); @@ -647,7 +647,7 @@ ionic_rx_empty(struct ionic_queue *q) mbuf = info[0]; rte_mempool_put(rxq->mb_pool, mbuf); - q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); + q->tail_idx = Q_NEXT_TO_SRVC(q, 1); } } @@ -1055,7 +1055,7 @@ ionic_rxq_service(struct ionic_qcq *rxq, uint32_t work_to_do, more = (q->tail_idx != cq_desc->comp_index); curr_q_tail_idx = q->tail_idx; - q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); + q->tail_idx = Q_NEXT_TO_SRVC(q, 1); /* Prefetch the next 4 descriptors */ if ((q->tail_idx & 0x3) == 0) -- 2.17.1
Create a unique Q-CQ struct for adminq, notifyq, rxq, and txq to reduce the size of each object. Minimize the size of each field to squeeze into as few cachelines as possible. Signed-off-by: Andrew Boyer <aboyer@pensando.io> --- drivers/net/ionic/ionic_lif.c | 153 ++++++++++++++++++--------------- drivers/net/ionic/ionic_lif.h | 62 +++++++++---- drivers/net/ionic/ionic_main.c | 4 +- drivers/net/ionic/ionic_rxtx.c | 129 +++++++++++++-------------- 4 files changed, 195 insertions(+), 153 deletions(-) diff --git a/drivers/net/ionic/ionic_lif.c b/drivers/net/ionic/ionic_lif.c index fcd6fca9a3..87579a09e5 100644 --- a/drivers/net/ionic/ionic_lif.c +++ b/drivers/net/ionic/ionic_lif.c @@ -76,13 +76,13 @@ ionic_lif_stop(struct ionic_lif *lif) lif->state &= ~IONIC_LIF_F_UP; for (i = 0; i < lif->nrxqcqs; i++) { - struct ionic_qcq *rxq = lif->rxqcqs[i]; + struct ionic_rx_qcq *rxq = lif->rxqcqs[i]; if (rxq->flags & IONIC_QCQ_F_INITED) (void)ionic_dev_rx_queue_stop(lif->eth_dev, i); } for (i = 0; i < lif->ntxqcqs; i++) { - struct ionic_qcq *txq = lif->txqcqs[i]; + struct ionic_tx_qcq *txq = lif->txqcqs[i]; if (txq->flags & IONIC_QCQ_F_INITED) (void)ionic_dev_tx_queue_stop(lif->eth_dev, i); } @@ -131,7 +131,7 @@ ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats ls->rx_bcast_bytes; for (i = 0; i < lif->nrxqcqs; i++) { - struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx; + struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats; stats->imissed += rx_stats->no_cb_arg + rx_stats->bad_cq_status + @@ -152,7 +152,7 @@ ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats ls->rx_desc_data_error; for (i = 0; i < num_rx_q_counters; i++) { - struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx; + struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats; stats->q_ipackets[i] = rx_stats->packets; stats->q_ibytes[i] = rx_stats->bytes; stats->q_errors[i] = @@ -173,7 +173,7 @@ ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats ls->tx_bcast_bytes; for (i = 0; i < lif->ntxqcqs; i++) { - struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx; + struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats; stats->oerrors += tx_stats->drop; } @@ -189,7 +189,7 @@ ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats ls->tx_desc_data_error; for (i = 0; i < num_tx_q_counters; i++) { - struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx; + struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats; stats->q_opackets[i] = tx_stats->packets; stats->q_obytes[i] = tx_stats->bytes; } @@ -217,9 +217,9 @@ ionic_lif_reset_stats(struct ionic_lif *lif) uint32_t i; for (i = 0; i < lif->nrxqcqs; i++) { - memset(&lif->rxqcqs[i]->stats.rx, 0, + memset(&lif->rxqcqs[i]->stats, 0, sizeof(struct ionic_rx_stats)); - memset(&lif->txqcqs[i]->stats.tx, 0, + memset(&lif->txqcqs[i]->stats, 0, sizeof(struct ionic_tx_stats)); } @@ -587,6 +587,7 @@ ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr) static int ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type, + size_t struct_size, uint32_t index, const char *type_name, uint16_t flags, @@ -625,16 +626,17 @@ ionic_qcq_alloc(struct ionic_lif *lif, total_size += PAGE_SIZE; } - new = rte_zmalloc("ionic", sizeof(*new), 0); + new = rte_zmalloc("ionic", struct_size, 0); if (!new) { IONIC_PRINT(ERR, "Cannot allocate queue structure"); return -ENOMEM; } new->lif = lif; - new->flags = flags; - new->q.info = rte_zmalloc("ionic", sizeof(*new->q.info) * num_descs, 0); + new->q.info = rte_calloc_socket("ionic", + num_descs, sizeof(void *), + PAGE_SIZE, socket_id); if (!new->q.info) { IONIC_PRINT(ERR, "Cannot allocate queue info"); err = -ENOMEM; @@ -667,7 +669,6 @@ ionic_qcq_alloc(struct ionic_lif *lif, new->base = new->base_z->addr; new->base_pa = new->base_z->iova; - new->total_size = total_size; q_base = new->base; q_base_pa = new->base_pa; @@ -720,15 +721,17 @@ ionic_qcq_free(struct ionic_qcq *qcq) } int -ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t nrxq_descs, - struct ionic_qcq **qcq) +ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, + uint16_t nrxq_descs, struct ionic_rx_qcq **rxq_out) { - uint32_t flags; - int err = -ENOMEM; + struct ionic_rx_qcq *rxq; + uint16_t flags; + int err; flags = IONIC_QCQ_F_SG; err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, + sizeof(struct ionic_rx_qcq), index, "rx", flags, @@ -736,25 +739,30 @@ ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t nrxq_descs, sizeof(struct ionic_rxq_desc), sizeof(struct ionic_rxq_comp), sizeof(struct ionic_rxq_sg_desc), - &lif->rxqcqs[index]); + (struct ionic_qcq **)&rxq); if (err) return err; - *qcq = lif->rxqcqs[index]; + rxq->flags = flags; + + lif->rxqcqs[index] = rxq; + *rxq_out = rxq; return 0; } int -ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t ntxq_descs, - struct ionic_qcq **qcq) +ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, + uint16_t ntxq_descs, struct ionic_tx_qcq **txq_out) { - uint32_t flags; - int err = -ENOMEM; + struct ionic_tx_qcq *txq; + uint16_t flags; + int err; flags = IONIC_QCQ_F_SG; err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, + sizeof(struct ionic_tx_qcq), index, "tx", flags, @@ -762,11 +770,14 @@ ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t ntxq_descs, sizeof(struct ionic_txq_desc), sizeof(struct ionic_txq_comp), sizeof(struct ionic_txq_sg_desc_v1), - &lif->txqcqs[index]); + (struct ionic_qcq **)&txq); if (err) return err; - *qcq = lif->txqcqs[index]; + txq->flags = flags; + + lif->txqcqs[index] = txq; + *txq_out = txq; return 0; } @@ -774,12 +785,12 @@ ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t ntxq_descs, static int ionic_admin_qcq_alloc(struct ionic_lif *lif) { - uint32_t flags; - int err = -ENOMEM; + uint16_t flags = 0; + int err; - flags = 0; err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, + sizeof(struct ionic_admin_qcq), 0, "admin", flags, @@ -787,7 +798,7 @@ ionic_admin_qcq_alloc(struct ionic_lif *lif) sizeof(struct ionic_admin_cmd), sizeof(struct ionic_admin_comp), 0, - &lif->adminqcq); + (struct ionic_qcq **)&lif->adminqcq); if (err) return err; @@ -797,13 +808,14 @@ ionic_admin_qcq_alloc(struct ionic_lif *lif) static int ionic_notify_qcq_alloc(struct ionic_lif *lif) { - struct ionic_qcq *nqcq; + struct ionic_notify_qcq *nqcq; struct ionic_dev *idev = &lif->adapter->idev; - uint32_t flags = 0; - int err = -ENOMEM; + uint16_t flags = 0; + int err; err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, + sizeof(struct ionic_notify_qcq), 0, "notify", flags, @@ -811,13 +823,13 @@ ionic_notify_qcq_alloc(struct ionic_lif *lif) sizeof(struct ionic_notifyq_cmd), sizeof(union ionic_notifyq_comp), 0, - &nqcq); + (struct ionic_qcq **)&nqcq); if (err) return err; err = ionic_intr_alloc(lif, &nqcq->intr); if (err) { - ionic_qcq_free(nqcq); + ionic_qcq_free(&nqcq->qcq); return err; } @@ -1002,12 +1014,12 @@ void ionic_lif_free(struct ionic_lif *lif) { if (lif->notifyqcq) { - ionic_qcq_free(lif->notifyqcq); + ionic_qcq_free(&lif->notifyqcq->qcq); lif->notifyqcq = NULL; } if (lif->adminqcq) { - ionic_qcq_free(lif->adminqcq); + ionic_qcq_free(&lif->adminqcq->qcq); lif->adminqcq = NULL; } @@ -1137,28 +1149,28 @@ ionic_lif_rss_teardown(struct ionic_lif *lif) } } -static void -ionic_lif_qcq_deinit(struct ionic_qcq *qcq) +void +ionic_lif_txq_deinit(struct ionic_tx_qcq *txq) { - qcq->flags &= ~IONIC_QCQ_F_INITED; + txq->flags &= ~IONIC_QCQ_F_INITED; } void -ionic_lif_txq_deinit(struct ionic_qcq *qcq) +ionic_lif_rxq_deinit(struct ionic_rx_qcq *rxq) { - ionic_lif_qcq_deinit(qcq); + rxq->flags &= ~IONIC_QCQ_F_INITED; } -void -ionic_lif_rxq_deinit(struct ionic_qcq *qcq) +static void +ionic_lif_adminq_deinit(struct ionic_lif *lif) { - ionic_lif_qcq_deinit(qcq); + lif->adminqcq->flags &= ~IONIC_QCQ_F_INITED; } static void ionic_lif_notifyq_deinit(struct ionic_lif *lif) { - struct ionic_qcq *nqcq = lif->notifyqcq; + struct ionic_notify_qcq *nqcq = lif->notifyqcq; struct ionic_dev *idev = &lif->adapter->idev; if (!(nqcq->flags & IONIC_QCQ_F_INITED)) @@ -1283,26 +1295,27 @@ int ionic_notifyq_handler(struct ionic_lif *lif, int budget) { struct ionic_dev *idev = &lif->adapter->idev; - struct ionic_qcq *qcq = lif->notifyqcq; + struct ionic_notify_qcq *nqcq = lif->notifyqcq; uint32_t work_done; - if (!(qcq->flags & IONIC_QCQ_F_INITED)) { + if (!(nqcq->flags & IONIC_QCQ_F_INITED)) { IONIC_PRINT(DEBUG, "Notifyq not yet initialized"); return -1; } - ionic_intr_mask(idev->intr_ctrl, qcq->intr.index, + ionic_intr_mask(idev->intr_ctrl, nqcq->intr.index, IONIC_INTR_MASK_SET); - work_done = ionic_qcq_service(qcq, budget, ionic_notifyq_cb, lif); + work_done = ionic_qcq_service(&nqcq->qcq, budget, + ionic_notifyq_cb, lif); if (lif->state & IONIC_LIF_F_LINK_CHECK_NEEDED) ionic_link_status_check(lif); - ionic_intr_credits(idev->intr_ctrl, qcq->intr.index, + ionic_intr_credits(idev->intr_ctrl, nqcq->intr.index, work_done, IONIC_INTR_CRED_RESET_COALESCE); - ionic_intr_mask(idev->intr_ctrl, qcq->intr.index, + ionic_intr_mask(idev->intr_ctrl, nqcq->intr.index, IONIC_INTR_MASK_CLEAR); return 0; @@ -1312,12 +1325,12 @@ static int ionic_lif_adminq_init(struct ionic_lif *lif) { struct ionic_dev *idev = &lif->adapter->idev; - struct ionic_qcq *qcq = lif->adminqcq; - struct ionic_queue *q = &qcq->q; + struct ionic_admin_qcq *aqcq = lif->adminqcq; + struct ionic_queue *q = &aqcq->qcq.q; struct ionic_q_init_comp comp; int err; - ionic_dev_cmd_adminq_init(idev, qcq); + ionic_dev_cmd_adminq_init(idev, &aqcq->qcq); err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT); if (err) return err; @@ -1332,7 +1345,7 @@ ionic_lif_adminq_init(struct ionic_lif *lif) IONIC_PRINT(DEBUG, "adminq->hw_index %d", q->hw_index); IONIC_PRINT(DEBUG, "adminq->db %p", q->db); - qcq->flags |= IONIC_QCQ_F_INITED; + aqcq->flags |= IONIC_QCQ_F_INITED; return 0; } @@ -1341,8 +1354,8 @@ static int ionic_lif_notifyq_init(struct ionic_lif *lif) { struct ionic_dev *idev = &lif->adapter->idev; - struct ionic_qcq *qcq = lif->notifyqcq; - struct ionic_queue *q = &qcq->q; + struct ionic_notify_qcq *nqcq = lif->notifyqcq; + struct ionic_queue *q = &nqcq->qcq.q; int err; struct ionic_admin_ctx ctx = { @@ -1352,7 +1365,7 @@ ionic_lif_notifyq_init(struct ionic_lif *lif) .type = q->type, .ver = lif->qtype_info[q->type].version, .index = rte_cpu_to_le_32(q->index), - .intr_index = rte_cpu_to_le_16(qcq->intr.index), + .intr_index = rte_cpu_to_le_16(nqcq->intr.index), .flags = rte_cpu_to_le_16(IONIC_QINIT_F_IRQ | IONIC_QINIT_F_ENA), .ring_size = rte_log2_u32(q->num_descs), @@ -1378,10 +1391,10 @@ ionic_lif_notifyq_init(struct ionic_lif *lif) IONIC_PRINT(DEBUG, "notifyq->hw_index %d", q->hw_index); IONIC_PRINT(DEBUG, "notifyq->db %p", q->db); - ionic_intr_mask(idev->intr_ctrl, qcq->intr.index, + ionic_intr_mask(idev->intr_ctrl, nqcq->intr.index, IONIC_INTR_MASK_CLEAR); - qcq->flags |= IONIC_QCQ_F_INITED; + nqcq->flags |= IONIC_QCQ_F_INITED; return 0; } @@ -1445,8 +1458,9 @@ ionic_lif_set_features(struct ionic_lif *lif) } int -ionic_lif_txq_init(struct ionic_qcq *qcq) +ionic_lif_txq_init(struct ionic_tx_qcq *txq) { + struct ionic_qcq *qcq = &txq->qcq; struct ionic_queue *q = &qcq->q; struct ionic_lif *lif = qcq->lif; struct ionic_cq *cq = &qcq->cq; @@ -1474,7 +1488,7 @@ ionic_lif_txq_init(struct ionic_qcq *qcq) ctx.cmd.q_init.ring_size); IONIC_PRINT(DEBUG, "txq_init.ver %u", ctx.cmd.q_init.ver); - err = ionic_adminq_post_wait(qcq->lif, &ctx); + err = ionic_adminq_post_wait(lif, &ctx); if (err) return err; @@ -1486,14 +1500,15 @@ ionic_lif_txq_init(struct ionic_qcq *qcq) IONIC_PRINT(DEBUG, "txq->hw_index %d", q->hw_index); IONIC_PRINT(DEBUG, "txq->db %p", q->db); - qcq->flags |= IONIC_QCQ_F_INITED; + txq->flags |= IONIC_QCQ_F_INITED; return 0; } int -ionic_lif_rxq_init(struct ionic_qcq *qcq) +ionic_lif_rxq_init(struct ionic_rx_qcq *rxq) { + struct ionic_qcq *qcq = &rxq->qcq; struct ionic_queue *q = &qcq->q; struct ionic_lif *lif = qcq->lif; struct ionic_cq *cq = &qcq->cq; @@ -1521,7 +1536,7 @@ ionic_lif_rxq_init(struct ionic_qcq *qcq) ctx.cmd.q_init.ring_size); IONIC_PRINT(DEBUG, "rxq_init.ver %u", ctx.cmd.q_init.ver); - err = ionic_adminq_post_wait(qcq->lif, &ctx); + err = ionic_adminq_post_wait(lif, &ctx); if (err) return err; @@ -1529,7 +1544,7 @@ ionic_lif_rxq_init(struct ionic_qcq *qcq) q->hw_index = rte_le_to_cpu_32(ctx.comp.q_init.hw_index); q->db = ionic_db_map(lif, q); - qcq->flags |= IONIC_QCQ_F_INITED; + rxq->flags |= IONIC_QCQ_F_INITED; IONIC_PRINT(DEBUG, "rxq->hw_type %d", q->hw_type); IONIC_PRINT(DEBUG, "rxq->hw_index %d", q->hw_index); @@ -1634,7 +1649,7 @@ ionic_lif_init(struct ionic_lif *lif) ionic_lif_notifyq_deinit(lif); err_out_adminq_deinit: - ionic_lif_qcq_deinit(lif->adminqcq); + ionic_lif_adminq_deinit(lif); return err; } @@ -1648,7 +1663,7 @@ ionic_lif_deinit(struct ionic_lif *lif) ionic_rx_filters_deinit(lif); ionic_lif_rss_teardown(lif); ionic_lif_notifyq_deinit(lif); - ionic_lif_qcq_deinit(lif->adminqcq); + ionic_lif_adminq_deinit(lif); lif->state &= ~IONIC_LIF_F_INITED; } @@ -1788,7 +1803,7 @@ ionic_lif_start(struct ionic_lif *lif) lif->nrxqcqs, lif->ntxqcqs, lif->port_id); for (i = 0; i < lif->nrxqcqs; i++) { - struct ionic_qcq *rxq = lif->rxqcqs[i]; + struct ionic_rx_qcq *rxq = lif->rxqcqs[i]; if (!(rxq->flags & IONIC_QCQ_F_DEFERRED)) { err = ionic_dev_rx_queue_start(lif->eth_dev, i); @@ -1798,7 +1813,7 @@ ionic_lif_start(struct ionic_lif *lif) } for (i = 0; i < lif->ntxqcqs; i++) { - struct ionic_qcq *txq = lif->txqcqs[i]; + struct ionic_tx_qcq *txq = lif->txqcqs[i]; if (!(txq->flags & IONIC_QCQ_F_DEFERRED)) { err = ionic_dev_tx_queue_start(lif->eth_dev, i); diff --git a/drivers/net/ionic/ionic_lif.h b/drivers/net/ionic/ionic_lif.h index c850a9c08d..a8243ebf21 100644 --- a/drivers/net/ionic/ionic_lif.h +++ b/drivers/net/ionic/ionic_lif.h @@ -58,23 +58,47 @@ struct ionic_qcq { struct ionic_queue q; /**< Queue */ struct ionic_cq cq; /**< Completion Queue */ struct ionic_lif *lif; /**< LIF */ - struct rte_mempool *mb_pool; /**< mbuf pool to populate the RX ring */ - union { - struct ionic_tx_stats tx; - struct ionic_rx_stats rx; - } stats; const struct rte_memzone *base_z; void *base; rte_iova_t base_pa; - uint32_t total_size; - uint32_t flags; +}; + +struct ionic_admin_qcq { + struct ionic_qcq qcq; + uint16_t flags; +}; + +struct ionic_notify_qcq { + struct ionic_qcq qcq; + uint16_t flags; + struct ionic_intr_info intr; }; +struct ionic_rx_qcq { + /* cacheline0, cacheline1 */ + struct ionic_qcq qcq; + + /* cacheline2 */ + struct rte_mempool *mb_pool; + uint16_t flags; + + /* cacheline3 (inside stats) */ + struct ionic_rx_stats stats; +}; + +struct ionic_tx_qcq { + /* cacheline0, cacheline1 */ + struct ionic_qcq qcq; + + /* cacheline2 */ + uint16_t flags; + + struct ionic_tx_stats stats; +}; + #define IONIC_Q_TO_QCQ(_q) container_of(_q, struct ionic_qcq, q) #define IONIC_CQ_TO_QCQ(_cq) container_of(_cq, struct ionic_qcq, cq) -#define IONIC_Q_TO_TX_STATS(q) (&IONIC_Q_TO_QCQ(q)->stats.tx) -#define IONIC_Q_TO_RX_STATS(q) (&IONIC_Q_TO_QCQ(q)->stats.rx) struct ionic_qtype_info { uint8_t version; @@ -104,10 +128,10 @@ struct ionic_lif { uint32_t nrxqcqs; rte_spinlock_t adminq_lock; rte_spinlock_t adminq_service_lock; - struct ionic_qcq *adminqcq; - struct ionic_qcq *notifyqcq; - struct ionic_qcq **txqcqs; - struct ionic_qcq **rxqcqs; + struct ionic_admin_qcq *adminqcq; + struct ionic_notify_qcq *notifyqcq; + struct ionic_tx_qcq **txqcqs; + struct ionic_rx_qcq **rxqcqs; struct ionic_rx_filters rx_filters; struct ionic_doorbell __iomem *kern_dbpage; uint64_t last_eid; @@ -173,19 +197,19 @@ int ionic_dev_allmulticast_enable(struct rte_eth_dev *dev); int ionic_dev_allmulticast_disable(struct rte_eth_dev *dev); int ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, - uint16_t nrxq_descs, struct ionic_qcq **qcq); + uint16_t nrxq_descs, struct ionic_rx_qcq **qcq_out); int ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, - uint16_t ntxq_descs, struct ionic_qcq **qcq); + uint16_t ntxq_descs, struct ionic_tx_qcq **qcq_out); void ionic_qcq_free(struct ionic_qcq *qcq); int ionic_qcq_enable(struct ionic_qcq *qcq); int ionic_qcq_disable(struct ionic_qcq *qcq); -int ionic_lif_rxq_init(struct ionic_qcq *qcq); -void ionic_lif_rxq_deinit(struct ionic_qcq *qcq); +int ionic_lif_rxq_init(struct ionic_rx_qcq *rxq); +void ionic_lif_rxq_deinit(struct ionic_rx_qcq *rxq); -int ionic_lif_txq_init(struct ionic_qcq *qcq); -void ionic_lif_txq_deinit(struct ionic_qcq *qcq); +int ionic_lif_txq_init(struct ionic_tx_qcq *txq); +void ionic_lif_txq_deinit(struct ionic_tx_qcq *txq); int ionic_lif_rss_config(struct ionic_lif *lif, const uint16_t types, const uint8_t *key, const uint32_t *indir); diff --git a/drivers/net/ionic/ionic_main.c b/drivers/net/ionic/ionic_main.c index b358a76f06..05dc72e834 100644 --- a/drivers/net/ionic/ionic_main.c +++ b/drivers/net/ionic/ionic_main.c @@ -194,7 +194,7 @@ ionic_adminq_service(struct ionic_cq *cq, uint16_t cq_desc_index, int ionic_adminq_post(struct ionic_lif *lif, struct ionic_admin_ctx *ctx) { - struct ionic_queue *q = &lif->adminqcq->q; + struct ionic_queue *q = &lif->adminqcq->qcq.q; struct ionic_admin_cmd *q_desc_base = q->base; struct ionic_admin_cmd *q_desc; int err = 0; @@ -234,7 +234,7 @@ ionic_adminq_wait_for_completion(struct ionic_lif *lif, */ rte_spinlock_lock(&lif->adminq_service_lock); - ionic_qcq_service(lif->adminqcq, budget, + ionic_qcq_service(&lif->adminqcq->qcq, budget, ionic_adminq_service, NULL); rte_spinlock_unlock(&lif->adminq_service_lock); diff --git a/drivers/net/ionic/ionic_rxtx.c b/drivers/net/ionic/ionic_rxtx.c index ce38651559..102cfa4331 100644 --- a/drivers/net/ionic/ionic_rxtx.c +++ b/drivers/net/ionic/ionic_rxtx.c @@ -59,8 +59,8 @@ void ionic_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, struct rte_eth_txq_info *qinfo) { - struct ionic_qcq *txq = dev->data->tx_queues[queue_id]; - struct ionic_queue *q = &txq->q; + struct ionic_tx_qcq *txq = dev->data->tx_queues[queue_id]; + struct ionic_queue *q = &txq->qcq.q; qinfo->nb_desc = q->num_descs; qinfo->conf.offloads = dev->data->dev_conf.txmode.offloads; @@ -68,10 +68,10 @@ ionic_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, } static __rte_always_inline void -ionic_tx_flush(struct ionic_qcq *txq) +ionic_tx_flush(struct ionic_tx_qcq *txq) { - struct ionic_cq *cq = &txq->cq; - struct ionic_queue *q = &txq->q; + struct ionic_cq *cq = &txq->qcq.cq; + struct ionic_queue *q = &txq->qcq.q; struct rte_mbuf *txm, *next; struct ionic_txq_comp *cq_desc_base = cq->base; struct ionic_txq_comp *cq_desc; @@ -122,19 +122,19 @@ ionic_tx_flush(struct ionic_qcq *txq) void __rte_cold ionic_dev_tx_queue_release(void *tx_queue) { - struct ionic_qcq *txq = (struct ionic_qcq *)tx_queue; + struct ionic_tx_qcq *txq = tx_queue; IONIC_PRINT_CALL(); ionic_lif_txq_deinit(txq); - ionic_qcq_free(txq); + ionic_qcq_free(&txq->qcq); } int __rte_cold ionic_dev_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id) { - struct ionic_qcq *txq; + struct ionic_tx_qcq *txq; IONIC_PRINT(DEBUG, "Stopping TX queue %u", tx_queue_id); @@ -148,7 +148,7 @@ ionic_dev_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id) * before disabling Tx queue */ - ionic_qcq_disable(txq); + ionic_qcq_disable(&txq->qcq); ionic_tx_flush(txq); @@ -161,7 +161,7 @@ ionic_dev_tx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id, const struct rte_eth_txconf *tx_conf) { struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); - struct ionic_qcq *txq; + struct ionic_tx_qcq *txq; uint64_t offloads; int err; @@ -221,7 +221,7 @@ int __rte_cold ionic_dev_tx_queue_start(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id) { uint8_t *tx_queue_state = eth_dev->data->tx_queue_state; - struct ionic_qcq *txq; + struct ionic_tx_qcq *txq; int err; if (tx_queue_state[tx_queue_id] == RTE_ETH_QUEUE_STATE_STARTED) { @@ -233,14 +233,14 @@ ionic_dev_tx_queue_start(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id) txq = eth_dev->data->tx_queues[tx_queue_id]; IONIC_PRINT(DEBUG, "Starting TX queue %u, %u descs", - tx_queue_id, txq->q.num_descs); + tx_queue_id, txq->qcq.q.num_descs); if (!(txq->flags & IONIC_QCQ_F_INITED)) { err = ionic_lif_txq_init(txq); if (err) return err; } else { - ionic_qcq_enable(txq); + ionic_qcq_enable(&txq->qcq); } tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; @@ -315,8 +315,9 @@ ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc, } static struct ionic_txq_desc * -ionic_tx_tso_next(struct ionic_queue *q, struct ionic_txq_sg_elem **elem) +ionic_tx_tso_next(struct ionic_tx_qcq *txq, struct ionic_txq_sg_elem **elem) { + struct ionic_queue *q = &txq->qcq.q; struct ionic_txq_desc *desc_base = q->base; struct ionic_txq_sg_desc_v1 *sg_desc_base = q->sg_base; struct ionic_txq_desc *desc = &desc_base[q->head_idx]; @@ -327,11 +328,11 @@ ionic_tx_tso_next(struct ionic_queue *q, struct ionic_txq_sg_elem **elem) } static int -ionic_tx_tso(struct ionic_qcq *txq, struct rte_mbuf *txm, +ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, bool not_xmit_more) { - struct ionic_queue *q = &txq->q; - struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q); + struct ionic_queue *q = &txq->qcq.q; + struct ionic_tx_stats *stats = &txq->stats; struct ionic_txq_desc *desc; struct ionic_txq_sg_elem *elem; struct rte_mbuf *txm_seg; @@ -375,7 +376,7 @@ ionic_tx_tso(struct ionic_qcq *txq, struct rte_mbuf *txm, left = txm->data_len; data_iova = rte_mbuf_data_iova(txm); - desc = ionic_tx_tso_next(q, &elem); + desc = ionic_tx_tso_next(txq, &elem); start = true; /* Chop data up into desc segments */ @@ -397,7 +398,7 @@ ionic_tx_tso(struct ionic_qcq *txq, struct rte_mbuf *txm, encap, vlan_tci, has_vlan, start, done && not_xmit_more); - desc = ionic_tx_tso_next(q, &elem); + desc = ionic_tx_tso_next(txq, &elem); start = false; seglen = mss; } @@ -439,7 +440,7 @@ ionic_tx_tso(struct ionic_qcq *txq, struct rte_mbuf *txm, encap, vlan_tci, has_vlan, start, done && not_xmit_more); - desc = ionic_tx_tso_next(q, &elem); + desc = ionic_tx_tso_next(txq, &elem); start = false; } @@ -452,16 +453,14 @@ ionic_tx_tso(struct ionic_qcq *txq, struct rte_mbuf *txm, } static __rte_always_inline int -ionic_tx(struct ionic_qcq *txq, struct rte_mbuf *txm, +ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, bool not_xmit_more) { - struct ionic_queue *q = &txq->q; - struct ionic_txq_desc *desc_base = q->base; + struct ionic_queue *q = &txq->qcq.q; + struct ionic_txq_desc *desc, *desc_base = q->base; struct ionic_txq_sg_desc_v1 *sg_desc_base = q->sg_base; - struct ionic_txq_desc *desc = &desc_base[q->head_idx]; - struct ionic_txq_sg_desc_v1 *sg_desc = &sg_desc_base[q->head_idx]; - struct ionic_txq_sg_elem *elem = sg_desc->elems; - struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q); + struct ionic_txq_sg_elem *elem; + struct ionic_tx_stats *stats = &txq->stats; struct rte_mbuf *txm_seg; bool encap; bool has_vlan; @@ -470,6 +469,8 @@ ionic_tx(struct ionic_qcq *txq, struct rte_mbuf *txm, uint8_t opcode = IONIC_TXQ_DESC_OPCODE_CSUM_NONE; uint8_t flags = 0; + desc = &desc_base[q->head_idx]; + if ((ol_flags & PKT_TX_IP_CKSUM) && (txq->flags & IONIC_QCQ_F_CSUM_L3)) { opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW; @@ -502,6 +503,7 @@ ionic_tx(struct ionic_qcq *txq, struct rte_mbuf *txm, desc->len = txm->data_len; desc->vlan_tci = txm->vlan_tci; + elem = sg_desc_base[q->head_idx].elems; txm_seg = txm->next; while (txm_seg != NULL) { elem->len = txm_seg->data_len; @@ -520,9 +522,9 @@ uint16_t ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { - struct ionic_qcq *txq = (struct ionic_qcq *)tx_queue; - struct ionic_queue *q = &txq->q; - struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q); + struct ionic_tx_qcq *txq = tx_queue; + struct ionic_queue *q = &txq->qcq.q; + struct ionic_tx_stats *stats = &txq->stats; uint32_t next_q_head_idx; uint32_t bytes_tx = 0; uint16_t nb_tx = 0; @@ -625,8 +627,8 @@ void ionic_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, struct rte_eth_rxq_info *qinfo) { - struct ionic_qcq *rxq = dev->data->rx_queues[queue_id]; - struct ionic_queue *q = &rxq->q; + struct ionic_rx_qcq *rxq = dev->data->rx_queues[queue_id]; + struct ionic_queue *q = &rxq->qcq.q; qinfo->mp = rxq->mb_pool; qinfo->scattered_rx = dev->data->scattered_rx; @@ -636,9 +638,9 @@ ionic_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, } static void __rte_cold -ionic_rx_empty(struct ionic_queue *q) +ionic_rx_empty(struct ionic_rx_qcq *rxq) { - struct ionic_qcq *rxq = IONIC_Q_TO_QCQ(q); + struct ionic_queue *q = &rxq->qcq.q; struct rte_mbuf *mbuf; void **info; @@ -654,15 +656,18 @@ ionic_rx_empty(struct ionic_queue *q) void __rte_cold ionic_dev_rx_queue_release(void *rx_queue) { - struct ionic_qcq *rxq = (struct ionic_qcq *)rx_queue; + struct ionic_rx_qcq *rxq = rx_queue; + + if (!rxq) + return; IONIC_PRINT_CALL(); - ionic_rx_empty(&rxq->q); + ionic_rx_empty(rxq); ionic_lif_rxq_deinit(rxq); - ionic_qcq_free(rxq); + ionic_qcq_free(&rxq->qcq); } int __rte_cold @@ -674,7 +679,7 @@ ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, struct rte_mempool *mp) { struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); - struct ionic_qcq *rxq; + struct ionic_rx_qcq *rxq; uint64_t offloads; int err; @@ -713,7 +718,8 @@ ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, eth_dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; - err = ionic_rx_qcq_alloc(lif, rx_queue_id, nb_desc, &rxq); + err = ionic_rx_qcq_alloc(lif, rx_queue_id, nb_desc, + &rxq); if (err) { IONIC_PRINT(ERR, "Queue %d allocation failure", rx_queue_id); return -EINVAL; @@ -741,20 +747,20 @@ ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, } static __rte_always_inline void -ionic_rx_clean(struct ionic_qcq *rxq, +ionic_rx_clean(struct ionic_rx_qcq *rxq, uint32_t q_desc_index, uint32_t cq_desc_index, void *service_cb_arg) { - struct ionic_queue *q = &rxq->q; - struct ionic_cq *cq = &rxq->cq; + struct ionic_queue *q = &rxq->qcq.q; + struct ionic_cq *cq = &rxq->qcq.cq; struct ionic_rxq_comp *cq_desc_base = cq->base; struct ionic_rxq_comp *cq_desc = &cq_desc_base[cq_desc_index]; struct rte_mbuf *rxm, *rxm_seg; uint32_t max_frame_size = - rxq->lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; + rxq->qcq.lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; uint64_t pkt_flags = 0; uint32_t pkt_type; - struct ionic_rx_stats *stats = IONIC_Q_TO_RX_STATS(q); + struct ionic_rx_stats *stats = &rxq->stats; struct ionic_rx_service *recv_args = (struct ionic_rx_service *) service_cb_arg; uint32_t buf_size = (uint16_t) @@ -803,7 +809,7 @@ ionic_rx_clean(struct ionic_qcq *rxq, rte_prefetch1((char *)rxm->buf_addr + rxm->data_off); rxm->nb_segs = 1; /* cq_desc->num_sg_elems */ rxm->pkt_len = cq_desc->len; - rxm->port = rxq->lif->port_id; + rxm->port = rxq->qcq.lif->port_id; left = cq_desc->len; @@ -909,13 +915,11 @@ ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index, } static __rte_always_inline int -ionic_rx_fill(struct ionic_qcq *rxq, uint32_t len) +ionic_rx_fill(struct ionic_rx_qcq *rxq, uint32_t len) { - struct ionic_queue *q = &rxq->q; - struct ionic_rxq_desc *desc_base = q->base; - struct ionic_rxq_sg_desc *sg_desc_base = q->sg_base; - struct ionic_rxq_desc *desc; - struct ionic_rxq_sg_desc *sg_desc; + struct ionic_queue *q = &rxq->qcq.q; + struct ionic_rxq_desc *desc, *desc_base = q->base; + struct ionic_rxq_sg_desc *sg_desc, *sg_desc_base = q->sg_base; struct ionic_rxq_sg_elem *elem; rte_iova_t dma_addr; uint32_t i, j, nsegs, buf_size, size; @@ -990,7 +994,7 @@ ionic_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) { uint32_t frame_size = eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; uint8_t *rx_queue_state = eth_dev->data->rx_queue_state; - struct ionic_qcq *rxq; + struct ionic_rx_qcq *rxq; int err; if (rx_queue_state[rx_queue_id] == RTE_ETH_QUEUE_STATE_STARTED) { @@ -1002,14 +1006,14 @@ ionic_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) rxq = eth_dev->data->rx_queues[rx_queue_id]; IONIC_PRINT(DEBUG, "Starting RX queue %u, %u descs (size: %u)", - rx_queue_id, rxq->q.num_descs, frame_size); + rx_queue_id, rxq->qcq.q.num_descs, frame_size); if (!(rxq->flags & IONIC_QCQ_F_INITED)) { err = ionic_lif_rxq_init(rxq); if (err) return err; } else { - ionic_qcq_enable(rxq); + ionic_qcq_enable(&rxq->qcq); } /* Allocate buffers for descriptor rings */ @@ -1025,13 +1029,12 @@ ionic_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) } static __rte_always_inline void -ionic_rxq_service(struct ionic_qcq *rxq, uint32_t work_to_do, +ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do, void *service_cb_arg) { - struct ionic_cq *cq = &rxq->cq; - struct ionic_queue *q = &rxq->q; - struct ionic_rxq_comp *cq_desc_base = cq->base; - struct ionic_rxq_comp *cq_desc; + struct ionic_cq *cq = &rxq->qcq.cq; + struct ionic_queue *q = &rxq->qcq.q; + struct ionic_rxq_comp *cq_desc, *cq_desc_base = cq->base; bool more; uint32_t curr_q_tail_idx, curr_cq_tail_idx; uint32_t work_done = 0; @@ -1080,7 +1083,7 @@ ionic_rxq_service(struct ionic_qcq *rxq, uint32_t work_to_do, int __rte_cold ionic_dev_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) { - struct ionic_qcq *rxq; + struct ionic_rx_qcq *rxq; IONIC_PRINT(DEBUG, "Stopping RX queue %u", rx_queue_id); @@ -1089,7 +1092,7 @@ ionic_dev_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) eth_dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; - ionic_qcq_disable(rxq); + ionic_qcq_disable(&rxq->qcq); /* Flush */ ionic_rxq_service(rxq, -1, NULL); @@ -1101,9 +1104,9 @@ uint16_t ionic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { - struct ionic_qcq *rxq = (struct ionic_qcq *)rx_queue; + struct ionic_rx_qcq *rxq = rx_queue; uint32_t frame_size = - rxq->lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; + rxq->qcq.lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; struct ionic_rx_service service_cb_arg; service_cb_arg.rx_pkts = rx_pkts; -- 2.17.1
Pipe the value from the queue setup routines through to ionic_qcq_alloc(). Signed-off-by: Andrew Boyer <aboyer@pensando.io> --- drivers/net/ionic/ionic_lif.c | 10 +++++++--- drivers/net/ionic/ionic_lif.h | 10 ++++++---- drivers/net/ionic/ionic_rxtx.c | 4 ++-- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/drivers/net/ionic/ionic_lif.c b/drivers/net/ionic/ionic_lif.c index 87579a09e5..dd79068948 100644 --- a/drivers/net/ionic/ionic_lif.c +++ b/drivers/net/ionic/ionic_lif.c @@ -588,6 +588,7 @@ static int ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type, size_t struct_size, + uint32_t socket_id, uint32_t index, const char *type_name, uint16_t flags, @@ -603,7 +604,6 @@ ionic_qcq_alloc(struct ionic_lif *lif, rte_iova_t q_base_pa = 0; rte_iova_t cq_base_pa = 0; rte_iova_t sg_base_pa = 0; - uint32_t socket_id = rte_socket_id(); int err; *qcq = NULL; @@ -721,7 +721,7 @@ ionic_qcq_free(struct ionic_qcq *qcq) } int -ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, +ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t socket_id, uint32_t index, uint16_t nrxq_descs, struct ionic_rx_qcq **rxq_out) { struct ionic_rx_qcq *rxq; @@ -732,6 +732,7 @@ ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, sizeof(struct ionic_rx_qcq), + socket_id, index, "rx", flags, @@ -752,7 +753,7 @@ ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, } int -ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, +ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t socket_id, uint32_t index, uint16_t ntxq_descs, struct ionic_tx_qcq **txq_out) { struct ionic_tx_qcq *txq; @@ -763,6 +764,7 @@ ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, sizeof(struct ionic_tx_qcq), + socket_id, index, "tx", flags, @@ -791,6 +793,7 @@ ionic_admin_qcq_alloc(struct ionic_lif *lif) err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, sizeof(struct ionic_admin_qcq), + rte_socket_id(), 0, "admin", flags, @@ -816,6 +819,7 @@ ionic_notify_qcq_alloc(struct ionic_lif *lif) err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, sizeof(struct ionic_notify_qcq), + rte_socket_id(), 0, "notify", flags, diff --git a/drivers/net/ionic/ionic_lif.h b/drivers/net/ionic/ionic_lif.h index a8243ebf21..ba1471b6e9 100644 --- a/drivers/net/ionic/ionic_lif.h +++ b/drivers/net/ionic/ionic_lif.h @@ -196,10 +196,12 @@ int ionic_dev_promiscuous_disable(struct rte_eth_dev *dev); int ionic_dev_allmulticast_enable(struct rte_eth_dev *dev); int ionic_dev_allmulticast_disable(struct rte_eth_dev *dev); -int ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, - uint16_t nrxq_descs, struct ionic_rx_qcq **qcq_out); -int ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, - uint16_t ntxq_descs, struct ionic_tx_qcq **qcq_out); +int ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t socket_id, + uint32_t index, uint16_t nrxq_descs, + struct ionic_rx_qcq **qcq_out); +int ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t socket_id, + uint32_t index, uint16_t ntxq_descs, + struct ionic_tx_qcq **qcq_out); void ionic_qcq_free(struct ionic_qcq *qcq); int ionic_qcq_enable(struct ionic_qcq *qcq); diff --git a/drivers/net/ionic/ionic_rxtx.c b/drivers/net/ionic/ionic_rxtx.c index 102cfa4331..0565cea603 100644 --- a/drivers/net/ionic/ionic_rxtx.c +++ b/drivers/net/ionic/ionic_rxtx.c @@ -191,7 +191,7 @@ ionic_dev_tx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id, eth_dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; - err = ionic_tx_qcq_alloc(lif, tx_queue_id, nb_desc, &txq); + err = ionic_tx_qcq_alloc(lif, socket_id, tx_queue_id, nb_desc, &txq); if (err) { IONIC_PRINT(DEBUG, "Queue allocation failure"); return -EINVAL; @@ -718,7 +718,7 @@ ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, eth_dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; - err = ionic_rx_qcq_alloc(lif, rx_queue_id, nb_desc, + err = ionic_rx_qcq_alloc(lif, socket_id, rx_queue_id, nb_desc, &rxq); if (err) { IONIC_PRINT(ERR, "Queue %d allocation failure", rx_queue_id); -- 2.17.1
This improves debuggability. To see the logs, use EAL arg: --log-level=pmd.net.ionic,debug While here, stop counting fragments, but start counting mtods. Signed-off-by: Andrew Boyer <aboyer@pensando.io> --- drivers/net/ionic/ionic_lif.h | 2 +- drivers/net/ionic/ionic_rxtx.c | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/drivers/net/ionic/ionic_lif.h b/drivers/net/ionic/ionic_lif.h index ba1471b6e9..5885aa1546 100644 --- a/drivers/net/ionic/ionic_lif.h +++ b/drivers/net/ionic/ionic_lif.h @@ -34,7 +34,6 @@ struct ionic_tx_stats { uint64_t stop; uint64_t no_csum; uint64_t tso; - uint64_t frags; }; struct ionic_rx_stats { @@ -44,6 +43,7 @@ struct ionic_rx_stats { uint64_t bad_cq_status; uint64_t no_room; uint64_t bad_len; + uint64_t mtods; }; #define IONIC_QCQ_F_INITED BIT(0) diff --git a/drivers/net/ionic/ionic_rxtx.c b/drivers/net/ionic/ionic_rxtx.c index 0565cea603..9d0df2a098 100644 --- a/drivers/net/ionic/ionic_rxtx.c +++ b/drivers/net/ionic/ionic_rxtx.c @@ -123,9 +123,13 @@ void __rte_cold ionic_dev_tx_queue_release(void *tx_queue) { struct ionic_tx_qcq *txq = tx_queue; + struct ionic_tx_stats *stats = &txq->stats; IONIC_PRINT_CALL(); + IONIC_PRINT(DEBUG, "TX queue %u pkts %ju tso %ju", + txq->qcq.q.index, stats->packets, stats->tso); + ionic_lif_txq_deinit(txq); ionic_qcq_free(&txq->qcq); @@ -410,7 +414,6 @@ ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, offset = 0; data_iova = rte_mbuf_data_iova(txm_seg); left = txm_seg->data_len; - stats->frags++; while (left > 0) { next_addr = rte_cpu_to_le_64(data_iova + offset); @@ -508,7 +511,6 @@ ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, while (txm_seg != NULL) { elem->len = txm_seg->data_len; elem->addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm_seg)); - stats->frags++; elem++; txm_seg = txm_seg->next; } @@ -657,12 +659,18 @@ void __rte_cold ionic_dev_rx_queue_release(void *rx_queue) { struct ionic_rx_qcq *rxq = rx_queue; + struct ionic_rx_stats *stats; if (!rxq) return; IONIC_PRINT_CALL(); + stats = &rxq->stats; + + IONIC_PRINT(DEBUG, "RX queue %u pkts %ju mtod %ju", + rxq->qcq.q.index, stats->packets, stats->mtods); + ionic_rx_empty(rxq); ionic_lif_rxq_deinit(rxq); @@ -887,6 +895,7 @@ ionic_rx_clean(struct ionic_rx_qcq *rxq, pkt_type = RTE_PTYPE_L2_ETHER_ARP; else pkt_type = RTE_PTYPE_UNKNOWN; + stats->mtods++; break; } } -- 2.17.1
Break it up rather than inlining it, so that we can remove branches from the hot path. Signed-off-by: Andrew Boyer <aboyer@pensando.io> --- drivers/net/ionic/ionic_dev.c | 11 ---------- drivers/net/ionic/ionic_dev.h | 1 - drivers/net/ionic/ionic_main.c | 10 ++++++++- drivers/net/ionic/ionic_rxtx.c | 37 ++++++++++++++++++++++++++++++---- 4 files changed, 42 insertions(+), 17 deletions(-) diff --git a/drivers/net/ionic/ionic_dev.c b/drivers/net/ionic/ionic_dev.c index cfaf4abc23..43e9ca3de3 100644 --- a/drivers/net/ionic/ionic_dev.c +++ b/drivers/net/ionic/ionic_dev.c @@ -438,14 +438,3 @@ ionic_q_sg_map(struct ionic_queue *q, void *base, rte_iova_t base_pa) q->sg_base = base; q->sg_base_pa = base_pa; } - -void -ionic_q_post(struct ionic_queue *q, bool ring_doorbell, void *cb_arg) -{ - q->info[q->head_idx] = cb_arg; - - q->head_idx = Q_NEXT_TO_POST(q, 1); - - if (ring_doorbell) - ionic_q_flush(q); -} diff --git a/drivers/net/ionic/ionic_dev.h b/drivers/net/ionic/ionic_dev.h index 0d1aff776b..4c7f6f647b 100644 --- a/drivers/net/ionic/ionic_dev.h +++ b/drivers/net/ionic/ionic_dev.h @@ -225,7 +225,6 @@ uint32_t ionic_cq_service(struct ionic_cq *cq, uint32_t work_to_do, int ionic_q_init(struct ionic_queue *q, uint32_t index, uint16_t num_descs); void ionic_q_map(struct ionic_queue *q, void *base, rte_iova_t base_pa); void ionic_q_sg_map(struct ionic_queue *q, void *base, rte_iova_t base_pa); -void ionic_q_post(struct ionic_queue *q, bool ring_doorbell, void *cb_arg); static inline uint16_t ionic_q_space_avail(struct ionic_queue *q) diff --git a/drivers/net/ionic/ionic_main.c b/drivers/net/ionic/ionic_main.c index 05dc72e834..b65f538fe5 100644 --- a/drivers/net/ionic/ionic_main.c +++ b/drivers/net/ionic/ionic_main.c @@ -197,6 +197,7 @@ ionic_adminq_post(struct ionic_lif *lif, struct ionic_admin_ctx *ctx) struct ionic_queue *q = &lif->adminqcq->qcq.q; struct ionic_admin_cmd *q_desc_base = q->base; struct ionic_admin_cmd *q_desc; + void **info; int err = 0; rte_spinlock_lock(&lif->adminq_lock); @@ -210,7 +211,14 @@ ionic_adminq_post(struct ionic_lif *lif, struct ionic_admin_ctx *ctx) memcpy(q_desc, &ctx->cmd, sizeof(ctx->cmd)); - ionic_q_post(q, true, ctx); + info = IONIC_INFO_PTR(q, q->head_idx); + info[0] = ctx; + + q->head_idx = Q_NEXT_TO_POST(q, 1); + + /* Ring doorbell */ + rte_wmb(); + ionic_q_flush(q); err_out: rte_spinlock_unlock(&lif->adminq_lock); diff --git a/drivers/net/ionic/ionic_rxtx.c b/drivers/net/ionic/ionic_rxtx.c index 9d0df2a098..c671b34e94 100644 --- a/drivers/net/ionic/ionic_rxtx.c +++ b/drivers/net/ionic/ionic_rxtx.c @@ -302,6 +302,7 @@ ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc, uint16_t vlan_tci, bool has_vlan, bool start, bool done) { + void **info; uint8_t flags = 0; flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; @@ -315,7 +316,15 @@ ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc, desc->hdr_len = hdrlen; desc->mss = mss; - ionic_q_post(q, done, done ? txm : NULL); + if (done) { + info = IONIC_INFO_PTR(q, q->head_idx); + info[0] = txm; + } + + q->head_idx = Q_NEXT_TO_POST(q, 1); + + if (done) + ionic_q_flush(q); } static struct ionic_txq_desc * @@ -465,6 +474,7 @@ ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, struct ionic_txq_sg_elem *elem; struct ionic_tx_stats *stats = &txq->stats; struct rte_mbuf *txm_seg; + void **info; bool encap; bool has_vlan; uint64_t ol_flags = txm->ol_flags; @@ -473,6 +483,7 @@ ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, uint8_t flags = 0; desc = &desc_base[q->head_idx]; + info = IONIC_INFO_PTR(q, q->head_idx); if ((ol_flags & PKT_TX_IP_CKSUM) && (txq->flags & IONIC_QCQ_F_CSUM_L3)) { @@ -506,7 +517,10 @@ ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, desc->len = txm->data_len; desc->vlan_tci = txm->vlan_tci; + info[0] = txm; + elem = sg_desc_base[q->head_idx].elems; + txm_seg = txm->next; while (txm_seg != NULL) { elem->len = txm_seg->data_len; @@ -515,7 +529,10 @@ ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, txm_seg = txm_seg->next; } - ionic_q_post(q, not_xmit_more, txm); + q->head_idx = Q_NEXT_TO_POST(q, 1); + + if (not_xmit_more) + ionic_q_flush(q); return 0; } @@ -920,7 +937,11 @@ ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index, new->addr = old->addr; new->len = old->len; - ionic_q_post(q, true, mbuf); + q->info[q->head_idx] = mbuf; + + q->head_idx = Q_NEXT_TO_POST(q, 1); + + ionic_q_flush(q); } static __rte_always_inline int @@ -930,6 +951,7 @@ ionic_rx_fill(struct ionic_rx_qcq *rxq, uint32_t len) struct ionic_rxq_desc *desc, *desc_base = q->base; struct ionic_rxq_sg_desc *sg_desc, *sg_desc_base = q->sg_base; struct ionic_rxq_sg_elem *elem; + void **info; rte_iova_t dma_addr; uint32_t i, j, nsegs, buf_size, size; bool ring_doorbell; @@ -947,6 +969,8 @@ ionic_rx_fill(struct ionic_rx_qcq *rxq, uint32_t len) return -ENOMEM; } + info = IONIC_INFO_PTR(q, q->head_idx); + nsegs = (len + buf_size - 1) / buf_size; desc = &desc_base[q->head_idx]; @@ -989,7 +1013,12 @@ ionic_rx_fill(struct ionic_rx_qcq *rxq, uint32_t len) ring_doorbell = ((q->head_idx + 1) & IONIC_RX_RING_DOORBELL_STRIDE) == 0; - ionic_q_post(q, ring_doorbell, rxm); + info[0] = rxm; + + q->head_idx = Q_NEXT_TO_POST(q, 1); + + if (ring_doorbell) + ionic_q_flush(q); } return 0; -- 2.17.1
This improves performance. Signed-off-by: Andrew Boyer <aboyer@pensando.io> --- drivers/net/ionic/ionic_rxtx.c | 41 +++++++++++----------------------- 1 file changed, 13 insertions(+), 28 deletions(-) diff --git a/drivers/net/ionic/ionic_rxtx.c b/drivers/net/ionic/ionic_rxtx.c index c671b34e94..2522c8283a 100644 --- a/drivers/net/ionic/ionic_rxtx.c +++ b/drivers/net/ionic/ionic_rxtx.c @@ -47,8 +47,6 @@ #include "ionic_lif.h" #include "ionic_rxtx.h" -#define IONIC_RX_RING_DOORBELL_STRIDE (32 - 1) - /********************************************************************* * * TX functions @@ -322,9 +320,6 @@ ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc, } q->head_idx = Q_NEXT_TO_POST(q, 1); - - if (done) - ionic_q_flush(q); } static struct ionic_txq_desc * @@ -341,8 +336,7 @@ ionic_tx_tso_next(struct ionic_tx_qcq *txq, struct ionic_txq_sg_elem **elem) } static int -ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, - bool not_xmit_more) +ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm) { struct ionic_queue *q = &txq->qcq.q; struct ionic_tx_stats *stats = &txq->stats; @@ -410,7 +404,7 @@ ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, hdrlen, mss, encap, vlan_tci, has_vlan, - start, done && not_xmit_more); + start, done); desc = ionic_tx_tso_next(txq, &elem); start = false; seglen = mss; @@ -451,7 +445,7 @@ ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, hdrlen, mss, encap, vlan_tci, has_vlan, - start, done && not_xmit_more); + start, done); desc = ionic_tx_tso_next(txq, &elem); start = false; } @@ -465,8 +459,7 @@ ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, } static __rte_always_inline int -ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, - bool not_xmit_more) +ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm) { struct ionic_queue *q = &txq->qcq.q; struct ionic_txq_desc *desc, *desc_base = q->base; @@ -531,9 +524,6 @@ ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, q->head_idx = Q_NEXT_TO_POST(q, 1); - if (not_xmit_more) - ionic_q_flush(q); - return 0; } @@ -548,7 +538,6 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint32_t bytes_tx = 0; uint16_t nb_tx = 0; int err; - bool last; /* Cleaning old buffers */ ionic_tx_flush(txq); @@ -559,8 +548,6 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, } while (nb_tx < nb_pkts) { - last = (nb_tx == (nb_pkts - 1)); - next_q_head_idx = Q_NEXT_TO_POST(q, 1); if ((next_q_head_idx & 0x3) == 0) { struct ionic_txq_desc *desc_base = q->base; @@ -569,13 +556,11 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, } if (tx_pkts[nb_tx]->ol_flags & PKT_TX_TCP_SEG) - err = ionic_tx_tso(txq, tx_pkts[nb_tx], last); + err = ionic_tx_tso(txq, tx_pkts[nb_tx]); else - err = ionic_tx(txq, tx_pkts[nb_tx], last); + err = ionic_tx(txq, tx_pkts[nb_tx]); if (err) { stats->drop += nb_pkts - nb_tx; - if (nb_tx > 0) - ionic_q_flush(q); break; } @@ -583,6 +568,11 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, nb_tx++; } + if (nb_tx > 0) { + rte_wmb(); + ionic_q_flush(q); + } + stats->packets += nb_tx; stats->bytes += bytes_tx; @@ -954,7 +944,6 @@ ionic_rx_fill(struct ionic_rx_qcq *rxq, uint32_t len) void **info; rte_iova_t dma_addr; uint32_t i, j, nsegs, buf_size, size; - bool ring_doorbell; buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) - RTE_PKTMBUF_HEADROOM); @@ -1010,17 +999,13 @@ ionic_rx_fill(struct ionic_rx_qcq *rxq, uint32_t len) IONIC_PRINT(ERR, "Rx SG size is not sufficient (%d < %d)", size, len); - ring_doorbell = ((q->head_idx + 1) & - IONIC_RX_RING_DOORBELL_STRIDE) == 0; - info[0] = rxm; q->head_idx = Q_NEXT_TO_POST(q, 1); - - if (ring_doorbell) - ionic_q_flush(q); } + ionic_q_flush(q); + return 0; } -- 2.17.1
Rather than dropping the whole burst if some don't fit. This improves performance. Signed-off-by: Andrew Boyer <aboyer@pensando.io> Signed-off-by: Vishwas Danivas <vishwas@pensando.io> --- drivers/net/ionic/ionic_rxtx.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/net/ionic/ionic_rxtx.c b/drivers/net/ionic/ionic_rxtx.c index 2522c8283a..81182b5dc4 100644 --- a/drivers/net/ionic/ionic_rxtx.c +++ b/drivers/net/ionic/ionic_rxtx.c @@ -536,15 +536,16 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, struct ionic_tx_stats *stats = &txq->stats; uint32_t next_q_head_idx; uint32_t bytes_tx = 0; - uint16_t nb_tx = 0; + uint16_t nb_avail, nb_tx = 0; int err; /* Cleaning old buffers */ ionic_tx_flush(txq); - if (unlikely(ionic_q_space_avail(q) < nb_pkts)) { - stats->stop += nb_pkts; - return 0; + nb_avail = ionic_q_space_avail(q); + if (unlikely(nb_avail < nb_pkts)) { + stats->stop += nb_pkts - nb_avail; + nb_pkts = nb_avail; } while (nb_tx < nb_pkts) { -- 2.17.1
This was missed when updating to the v1 Tx queue structures. Store the value in the queue for easy access. Fixes: 786c64763b50 ("net/ionic: clean up Tx queue version support") Signed-off-by: Andrew Boyer <aboyer@pensando.io> --- drivers/net/ionic/ionic_lif.c | 6 +++++- drivers/net/ionic/ionic_lif.h | 1 + drivers/net/ionic/ionic_rxtx.c | 6 +++--- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/drivers/net/ionic/ionic_lif.c b/drivers/net/ionic/ionic_lif.c index dd79068948..b8023e0632 100644 --- a/drivers/net/ionic/ionic_lif.c +++ b/drivers/net/ionic/ionic_lif.c @@ -757,10 +757,13 @@ ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t socket_id, uint32_t index, uint16_t ntxq_descs, struct ionic_tx_qcq **txq_out) { struct ionic_tx_qcq *txq; - uint16_t flags; + uint16_t flags, num_segs_fw; int err; flags = IONIC_QCQ_F_SG; + + num_segs_fw = IONIC_TX_MAX_SG_ELEMS_V1 + 1; + err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, sizeof(struct ionic_tx_qcq), @@ -777,6 +780,7 @@ ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t socket_id, uint32_t index, return err; txq->flags = flags; + txq->num_segs_fw = num_segs_fw; lif->txqcqs[index] = txq; *txq_out = txq; diff --git a/drivers/net/ionic/ionic_lif.h b/drivers/net/ionic/ionic_lif.h index 5885aa1546..9f00ba2973 100644 --- a/drivers/net/ionic/ionic_lif.h +++ b/drivers/net/ionic/ionic_lif.h @@ -92,6 +92,7 @@ struct ionic_tx_qcq { struct ionic_qcq qcq; /* cacheline2 */ + uint16_t num_segs_fw; /* # segs supported by current FW */ uint16_t flags; struct ionic_tx_stats stats; diff --git a/drivers/net/ionic/ionic_rxtx.c b/drivers/net/ionic/ionic_rxtx.c index 81182b5dc4..b83ea1bcaa 100644 --- a/drivers/net/ionic/ionic_rxtx.c +++ b/drivers/net/ionic/ionic_rxtx.c @@ -598,9 +598,9 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, (PKT_TX_OFFLOAD_MASK ^ IONIC_TX_OFFLOAD_MASK) uint16_t -ionic_prep_pkts(void *tx_queue __rte_unused, struct rte_mbuf **tx_pkts, - uint16_t nb_pkts) +ionic_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { + struct ionic_tx_qcq *txq = tx_queue; struct rte_mbuf *txm; uint64_t offloads; int i = 0; @@ -608,7 +608,7 @@ ionic_prep_pkts(void *tx_queue __rte_unused, struct rte_mbuf **tx_pkts, for (i = 0; i < nb_pkts; i++) { txm = tx_pkts[i]; - if (txm->nb_segs > IONIC_TX_MAX_SG_ELEMS) { + if (txm->nb_segs > txq->num_segs_fw) { rte_errno = -EINVAL; break; } -- 2.17.1
The completion type was wrong. Don't check the completion if the wait timed out. Fixes: 669c8de67c88 ("net/ionic: support basic LIF") Cc: cardigliano@ntop.org Cc: stable@dpdk.org Signed-off-by: Andrew Boyer <aboyer@pensando.io> --- drivers/net/ionic/ionic_lif.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/net/ionic/ionic_lif.c b/drivers/net/ionic/ionic_lif.c index b8023e0632..cd220abee2 100644 --- a/drivers/net/ionic/ionic_lif.c +++ b/drivers/net/ionic/ionic_lif.c @@ -1605,17 +1605,18 @@ int ionic_lif_init(struct ionic_lif *lif) { struct ionic_dev *idev = &lif->adapter->idev; - struct ionic_q_init_comp comp; + struct ionic_lif_init_comp comp; int err; memset(&lif->stats_base, 0, sizeof(lif->stats_base)); ionic_dev_cmd_lif_init(idev, lif->info_pa); err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT); - ionic_dev_cmd_comp(idev, &comp); if (err) return err; + ionic_dev_cmd_comp(idev, &comp); + lif->hw_index = rte_cpu_to_le_16(comp.hw_index); err = ionic_lif_adminq_init(lif); -- 2.17.1
> On Feb 4, 2021, at 2:58 PM, Andrew Boyer <aboyer@pensando.io> wrote:
>
> This patch series reorganizes the main datastructure for each
> queue, struct ionic_qcq. Its constituent struct ionic_queue and
> struct ionic_cq are stripped down first. Then the generic struct
> ionic_qcq is stripped down, and a unique struct is created for
> each queue type.
>
> The adminq code is consolidated into ionic_main.c as part of the
> cleanup.
>
> Next comes some minor performance fixups related to queue posting
> and doorbells.
>
> Finally, two minor fixes to Tx packet prep and LIF init.
>
> Signed-off-by: Andrew Boyer <aboyer@pensando.io>
>
> Andrew Boyer (14):
> net/ionic: cut down completion queue structure
> net/ionic: consolidate adminq code
> net/ionic: convert info array to generic pointers
> net/ionic: remove unused field from queue structure
> net/ionic: remove unused interrupt free function
> net/ionic: cut down queue structure
> net/ionic: split up queue-completion queue structure
> net/ionic: use the socket id passed in for Rx and Tx queues
> net/ionic: log queue counters when tearing down
> net/ionic: break up queue post function
> net/ionic: ring doorbell once at the end of each burst
> net/ionic: send as many packets as possible
> net/ionic: fix Tx fragment limit check
> net/ionic: fix code around lif init devcmd
>
> drivers/net/ionic/ionic.h | 13 ++
> drivers/net/ionic/ionic_dev.c | 132 +-------------
> drivers/net/ionic/ionic_dev.h | 81 +++------
> drivers/net/ionic/ionic_lif.c | 242 +++++++++++++------------
> drivers/net/ionic/ionic_lif.h | 77 +++++---
> drivers/net/ionic/ionic_main.c | 93 +++++++++-
> drivers/net/ionic/ionic_rx_filter.c | 1 +
> drivers/net/ionic/ionic_rxtx.c | 265 ++++++++++++++++------------
> 8 files changed, 458 insertions(+), 446 deletions(-)
>
> --
> 2.17.1
>
Looks like I missed the boat on this release. I will re-send 13/14 as a standalone.
-Andrew
This patch series reorganizes the main datastructure for each queue, struct ionic_qcq. Its constituent struct ionic_queue and struct ionic_cq are stripped down first. Then the generic struct ionic_qcq is stripped down, and a unique struct is created for each queue type. The adminq code is consolidated into ionic_main.c as part of the cleanup. Next comes some minor performance fixups related to queue posting and doorbells. Finally, a minor improvement to Tx packet prep and a minor fix for LIF init. Signed-off-by: Andrew Boyer <aboyer@pensando.io> -- v2: * Resend for new DPDK release cycle * Insert a new patch "net/ionic: remove unused filter delete function" so that even more adminq code can be staticized * Update second-to-last patch which was partially applied in 21.02 Andrew Boyer (15): net/ionic: cut down completion queue structure net/ionic: remove unused filter delete function net/ionic: consolidate adminq code net/ionic: convert info array to generic pointers net/ionic: remove unused field from queue structure net/ionic: remove unused interrupt free function net/ionic: cut down queue structure net/ionic: split up queue-completion queue structure net/ionic: use the socket id passed in for Rx and Tx queues net/ionic: log queue counters when tearing down net/ionic: break up queue post function net/ionic: ring doorbell once at the end of each burst net/ionic: send as many packets as possible net/ionic: store Tx fragment limit in queue net/ionic: fix code around lif init devcmd drivers/net/ionic/ionic.h | 13 +- drivers/net/ionic/ionic_dev.c | 132 +------------- drivers/net/ionic/ionic_dev.h | 82 +++------ drivers/net/ionic/ionic_lif.c | 242 +++++++++++++------------ drivers/net/ionic/ionic_lif.h | 77 +++++--- drivers/net/ionic/ionic_main.c | 95 +++++++++- drivers/net/ionic/ionic_rx_filter.c | 15 +- drivers/net/ionic/ionic_rx_filter.h | 1 - drivers/net/ionic/ionic_rxtx.c | 265 ++++++++++++++++------------ 9 files changed, 458 insertions(+), 464 deletions(-) -- 2.17.1
Add Q_NEXT_TO_POST() and Q_NEXT_TO_SRVC() macros. Use a precomputed size mask. This will conserve resources. Signed-off-by: Andrew Boyer <aboyer@pensando.io> --- drivers/net/ionic/ionic_dev.c | 14 +++----------- drivers/net/ionic/ionic_dev.h | 14 +++++++------- drivers/net/ionic/ionic_lif.c | 5 +++-- drivers/net/ionic/ionic_lif.h | 3 ++- drivers/net/ionic/ionic_rxtx.c | 24 ++++++++++++------------ 5 files changed, 27 insertions(+), 33 deletions(-) diff --git a/drivers/net/ionic/ionic_dev.c b/drivers/net/ionic/ionic_dev.c index a9f9e2faf9..f837817a0a 100644 --- a/drivers/net/ionic/ionic_dev.c +++ b/drivers/net/ionic/ionic_dev.c @@ -358,14 +358,8 @@ ionic_dev_cmd_adminq_init(struct ionic_dev *idev, struct ionic_qcq *qcq) } int -ionic_cq_init(struct ionic_lif *lif, struct ionic_cq *cq, - uint32_t num_descs, size_t desc_size) +ionic_cq_init(struct ionic_cq *cq, uint16_t num_descs) { - if (desc_size == 0) { - IONIC_PRINT(ERR, "Descriptor size is %zu", desc_size); - return -EINVAL; - } - if (!rte_is_power_of_2(num_descs) || num_descs < IONIC_MIN_RING_DESC || num_descs > IONIC_MAX_RING_DESC) { @@ -374,9 +368,8 @@ ionic_cq_init(struct ionic_lif *lif, struct ionic_cq *cq, return -EINVAL; } - cq->lif = lif; cq->num_descs = num_descs; - cq->desc_size = desc_size; + cq->size_mask = num_descs - 1; cq->tail_idx = 0; cq->done_color = 1; @@ -393,7 +386,6 @@ ionic_cq_map(struct ionic_cq *cq, void *base, rte_iova_t base_pa) void ionic_cq_bind(struct ionic_cq *cq, struct ionic_queue *q) { - cq->bound_q = q; q->bound_cq = cq; } @@ -407,7 +399,7 @@ ionic_cq_service(struct ionic_cq *cq, uint32_t work_to_do, return 0; while (cb(cq, cq->tail_idx, cb_arg)) { - cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1); + cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1); if (cq->tail_idx == 0) cq->done_color = !cq->done_color; diff --git a/drivers/net/ionic/ionic_dev.h b/drivers/net/ionic/ionic_dev.h index bacbe3f053..b29bfd13be 100644 --- a/drivers/net/ionic/ionic_dev.h +++ b/drivers/net/ionic/ionic_dev.h @@ -142,6 +142,9 @@ struct ionic_desc_info { void *cb_arg; }; +#define Q_NEXT_TO_POST(_q, _n) (((_q)->head_idx + (_n)) & ((_q)->size_mask)) +#define Q_NEXT_TO_SRVC(_q, _n) (((_q)->tail_idx + (_n)) & ((_q)->size_mask)) + struct ionic_queue { struct ionic_dev *idev; struct ionic_lif *lif; @@ -174,11 +177,9 @@ struct ionic_intr_info { }; struct ionic_cq { - struct ionic_lif *lif; - struct ionic_queue *bound_q; - uint32_t tail_idx; - uint32_t num_descs; - uint32_t desc_size; + uint16_t tail_idx; + uint16_t num_descs; + uint16_t size_mask; bool done_color; void *base; rte_iova_t base_pa; @@ -240,8 +241,7 @@ void ionic_dev_cmd_adminq_init(struct ionic_dev *idev, struct ionic_qcq *qcq); struct ionic_doorbell __iomem *ionic_db_map(struct ionic_lif *lif, struct ionic_queue *q); -int ionic_cq_init(struct ionic_lif *lif, struct ionic_cq *cq, - uint32_t num_descs, size_t desc_size); +int ionic_cq_init(struct ionic_cq *cq, uint16_t num_descs); void ionic_cq_map(struct ionic_cq *cq, void *base, rte_iova_t base_pa); void ionic_cq_bind(struct ionic_cq *cq, struct ionic_queue *q); typedef bool (*ionic_cq_cb)(struct ionic_cq *cq, uint32_t cq_desc_index, diff --git a/drivers/net/ionic/ionic_lif.c b/drivers/net/ionic/ionic_lif.c index e083f92417..a9fe34c0f2 100644 --- a/drivers/net/ionic/ionic_lif.c +++ b/drivers/net/ionic/ionic_lif.c @@ -656,7 +656,7 @@ ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type, goto err_out_free_info; } - err = ionic_cq_init(lif, &new->cq, num_descs, cq_desc_size); + err = ionic_cq_init(&new->cq, num_descs); if (err) { IONIC_PRINT(ERR, "Completion queue initialization failed"); goto err_out_free_info; @@ -1169,11 +1169,12 @@ ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index, { struct ionic_admin_comp *cq_desc_base = cq->base; struct ionic_admin_comp *cq_desc = &cq_desc_base[cq_desc_index]; + struct ionic_qcq *qcq = IONIC_CQ_TO_QCQ(cq); if (!color_match(cq_desc->color, cq->done_color)) return false; - ionic_q_service(cq->bound_q, cq_desc_index, cq_desc->comp_index, NULL); + ionic_q_service(&qcq->q, cq_desc_index, cq_desc->comp_index, NULL); return true; } diff --git a/drivers/net/ionic/ionic_lif.h b/drivers/net/ionic/ionic_lif.h index 8f01aefd60..1a898a0ef9 100644 --- a/drivers/net/ionic/ionic_lif.h +++ b/drivers/net/ionic/ionic_lif.h @@ -71,7 +71,8 @@ struct ionic_qcq { struct ionic_intr_info intr; }; -#define IONIC_Q_TO_QCQ(q) container_of(q, struct ionic_qcq, q) +#define IONIC_Q_TO_QCQ(_q) container_of(_q, struct ionic_qcq, q) +#define IONIC_CQ_TO_QCQ(_cq) container_of(_cq, struct ionic_qcq, cq) #define IONIC_Q_TO_TX_STATS(q) (&IONIC_Q_TO_QCQ(q)->stats.tx) #define IONIC_Q_TO_RX_STATS(q) (&IONIC_Q_TO_QCQ(q)->stats.rx) diff --git a/drivers/net/ionic/ionic_rxtx.c b/drivers/net/ionic/ionic_rxtx.c index 99920109eb..76de82882d 100644 --- a/drivers/net/ionic/ionic_rxtx.c +++ b/drivers/net/ionic/ionic_rxtx.c @@ -68,9 +68,10 @@ ionic_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, } static __rte_always_inline void -ionic_tx_flush(struct ionic_cq *cq) +ionic_tx_flush(struct ionic_qcq *txq) { - struct ionic_queue *q = cq->bound_q; + struct ionic_cq *cq = &txq->cq; + struct ionic_queue *q = &txq->q; struct ionic_desc_info *q_desc_info; struct rte_mbuf *txm, *next; struct ionic_txq_comp *cq_desc_base = cq->base; @@ -79,7 +80,7 @@ ionic_tx_flush(struct ionic_cq *cq) cq_desc = &cq_desc_base[cq->tail_idx]; while (color_match(cq_desc->color, cq->done_color)) { - cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1); + cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1); /* Prefetch the next 4 descriptors (not really useful here) */ if ((cq->tail_idx & 0x3) == 0) @@ -149,7 +150,7 @@ ionic_dev_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id) ionic_qcq_disable(txq); - ionic_tx_flush(&txq->cq); + ionic_tx_flush(txq); return 0; } @@ -521,7 +522,6 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, { struct ionic_qcq *txq = (struct ionic_qcq *)tx_queue; struct ionic_queue *q = &txq->q; - struct ionic_cq *cq = &txq->cq; struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q); uint32_t next_q_head_idx; uint32_t bytes_tx = 0; @@ -530,7 +530,7 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, bool last; /* Cleaning old buffers */ - ionic_tx_flush(cq); + ionic_tx_flush(txq); if (unlikely(ionic_q_space_avail(q) < nb_pkts)) { stats->stop += nb_pkts; @@ -1018,10 +1018,11 @@ ionic_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) } static __rte_always_inline void -ionic_rxq_service(struct ionic_cq *cq, uint32_t work_to_do, +ionic_rxq_service(struct ionic_qcq *rxq, uint32_t work_to_do, void *service_cb_arg) { - struct ionic_queue *q = cq->bound_q; + struct ionic_cq *cq = &rxq->cq; + struct ionic_queue *q = &rxq->q; struct ionic_desc_info *q_desc_info; struct ionic_rxq_comp *cq_desc_base = cq->base; struct ionic_rxq_comp *cq_desc; @@ -1035,7 +1036,7 @@ ionic_rxq_service(struct ionic_cq *cq, uint32_t work_to_do, cq_desc = &cq_desc_base[cq->tail_idx]; while (color_match(cq_desc->pkt_type_color, cq->done_color)) { curr_cq_tail_idx = cq->tail_idx; - cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1); + cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1); if (cq->tail_idx == 0) cq->done_color = !cq->done_color; @@ -1087,7 +1088,7 @@ ionic_dev_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) ionic_qcq_disable(rxq); /* Flush */ - ionic_rxq_service(&rxq->cq, -1, NULL); + ionic_rxq_service(rxq, -1, NULL); return 0; } @@ -1099,14 +1100,13 @@ ionic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, struct ionic_qcq *rxq = (struct ionic_qcq *)rx_queue; uint32_t frame_size = rxq->lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; - struct ionic_cq *cq = &rxq->cq; struct ionic_rx_service service_cb_arg; service_cb_arg.rx_pkts = rx_pkts; service_cb_arg.nb_pkts = nb_pkts; service_cb_arg.nb_rx = 0; - ionic_rxq_service(cq, nb_pkts, &service_cb_arg); + ionic_rxq_service(rxq, nb_pkts, &service_cb_arg); ionic_rx_fill(rxq, frame_size); -- 2.17.1
This function is unused. Signed-off-by: Andrew Boyer <aboyer@pensando.io> --- drivers/net/ionic/ionic_dev.h | 1 - drivers/net/ionic/ionic_rx_filter.c | 14 -------------- drivers/net/ionic/ionic_rx_filter.h | 1 - 3 files changed, 16 deletions(-) diff --git a/drivers/net/ionic/ionic_dev.h b/drivers/net/ionic/ionic_dev.h index b29bfd13be..e1b93b1b05 100644 --- a/drivers/net/ionic/ionic_dev.h +++ b/drivers/net/ionic/ionic_dev.h @@ -90,7 +90,6 @@ static inline void ionic_struct_size_checks(void) RTE_BUILD_BUG_ON(sizeof(struct ionic_rx_mode_set_cmd) != 64); RTE_BUILD_BUG_ON(sizeof(struct ionic_rx_filter_add_cmd) != 64); RTE_BUILD_BUG_ON(sizeof(struct ionic_rx_filter_add_comp) != 16); - RTE_BUILD_BUG_ON(sizeof(struct ionic_rx_filter_del_cmd) != 64); /* RDMA commands */ RTE_BUILD_BUG_ON(sizeof(struct ionic_rdma_reset_cmd) != 64); diff --git a/drivers/net/ionic/ionic_rx_filter.c b/drivers/net/ionic/ionic_rx_filter.c index 320b9019b3..4310c91b94 100644 --- a/drivers/net/ionic/ionic_rx_filter.c +++ b/drivers/net/ionic/ionic_rx_filter.c @@ -18,20 +18,6 @@ ionic_rx_filter_free(struct ionic_rx_filter *f) rte_free(f); } -int -ionic_rx_filter_del(struct ionic_lif *lif, struct ionic_rx_filter *f) -{ - struct ionic_admin_ctx ctx = { - .pending_work = true, - .cmd.rx_filter_del = { - .opcode = IONIC_CMD_RX_FILTER_DEL, - .filter_id = rte_cpu_to_le_32(f->filter_id), - }, - }; - - return ionic_adminq_post(lif, &ctx); -} - int ionic_rx_filters_init(struct ionic_lif *lif) { diff --git a/drivers/net/ionic/ionic_rx_filter.h b/drivers/net/ionic/ionic_rx_filter.h index e1dd5f910c..773042fcb2 100644 --- a/drivers/net/ionic/ionic_rx_filter.h +++ b/drivers/net/ionic/ionic_rx_filter.h @@ -35,7 +35,6 @@ struct ionic_admin_ctx; struct ionic_lif; void ionic_rx_filter_free(struct ionic_rx_filter *f); -int ionic_rx_filter_del(struct ionic_lif *lif, struct ionic_rx_filter *f); int ionic_rx_filters_init(struct ionic_lif *lif); void ionic_rx_filters_deinit(struct ionic_lif *lif); int ionic_rx_filter_save(struct ionic_lif *lif, uint32_t flow_id, -- 2.17.1
The adminq is the only caller of ionic_q_service(), so absorb it into ionic_adminq_service(). Move all of the adminq code together into ionic_main.c. Staticize a few things. Signed-off-by: Andrew Boyer <aboyer@pensando.io> --- drivers/net/ionic/ionic.h | 13 ++++- drivers/net/ionic/ionic_dev.c | 85 ---------------------------- drivers/net/ionic/ionic_dev.h | 15 ----- drivers/net/ionic/ionic_lif.c | 16 ------ drivers/net/ionic/ionic_lif.h | 2 - drivers/net/ionic/ionic_main.c | 87 ++++++++++++++++++++++++++--- drivers/net/ionic/ionic_rx_filter.c | 1 + 7 files changed, 93 insertions(+), 126 deletions(-) diff --git a/drivers/net/ionic/ionic.h b/drivers/net/ionic/ionic.h index 49d1fc003f..49b90d1b7c 100644 --- a/drivers/net/ionic/ionic.h +++ b/drivers/net/ionic/ionic.h @@ -65,8 +65,19 @@ struct ionic_adapter { LIST_ENTRY(ionic_adapter) pci_adapters; }; -int ionic_adminq_check_err(struct ionic_admin_ctx *ctx, bool timeout); +/** ionic_admin_ctx - Admin command context. + * @pending_work: Flag that indicates a completion. + * @cmd: Admin command (64B) to be copied to the queue. + * @comp: Admin completion (16B) copied from the queue. + */ +struct ionic_admin_ctx { + bool pending_work; + union ionic_adminq_cmd cmd; + union ionic_adminq_comp comp; +}; + int ionic_adminq_post_wait(struct ionic_lif *lif, struct ionic_admin_ctx *ctx); + int ionic_dev_cmd_wait_check(struct ionic_dev *idev, unsigned long max_wait); int ionic_setup(struct ionic_adapter *adapter); diff --git a/drivers/net/ionic/ionic_dev.c b/drivers/net/ionic/ionic_dev.c index f837817a0a..0eb9f6f21a 100644 --- a/drivers/net/ionic/ionic_dev.c +++ b/drivers/net/ionic/ionic_dev.c @@ -465,88 +465,3 @@ ionic_q_post(struct ionic_queue *q, bool ring_doorbell, desc_cb cb, if (ring_doorbell) ionic_q_flush(q); } - -void -ionic_q_service(struct ionic_queue *q, uint32_t cq_desc_index, - uint32_t stop_index, void *service_cb_arg) -{ - struct ionic_desc_info *desc_info; - uint32_t curr_q_tail_idx; - - do { - desc_info = &q->info[q->tail_idx]; - - if (desc_info->cb) - desc_info->cb(q, q->tail_idx, cq_desc_index, - desc_info->cb_arg, service_cb_arg); - - desc_info->cb = NULL; - desc_info->cb_arg = NULL; - - curr_q_tail_idx = q->tail_idx; - q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); - - } while (curr_q_tail_idx != stop_index); -} - -static void -ionic_adminq_cb(struct ionic_queue *q, - uint32_t q_desc_index, uint32_t cq_desc_index, - void *cb_arg, void *service_cb_arg __rte_unused) -{ - struct ionic_admin_ctx *ctx = cb_arg; - struct ionic_admin_comp *cq_desc_base = q->bound_cq->base; - struct ionic_admin_comp *cq_desc = &cq_desc_base[cq_desc_index]; - uint16_t comp_index; - - if (!ctx) - return; - - comp_index = rte_le_to_cpu_16(cq_desc->comp_index); - if (unlikely(comp_index != q_desc_index)) { - IONIC_WARN_ON(comp_index != q_desc_index); - return; - } - - memcpy(&ctx->comp, cq_desc, sizeof(*cq_desc)); - - ctx->pending_work = false; /* done */ -} - -/** ionic_adminq_post - Post an admin command. - * @lif: Handle to lif. - * @cmd_ctx: Api admin command context. - * - * Post the command to an admin queue in the ethernet driver. If this command - * succeeds, then the command has been posted, but that does not indicate a - * completion. If this command returns success, then the completion callback - * will eventually be called. - * - * Return: zero or negative error status. - */ -int -ionic_adminq_post(struct ionic_lif *lif, struct ionic_admin_ctx *ctx) -{ - struct ionic_queue *adminq = &lif->adminqcq->q; - struct ionic_admin_cmd *q_desc_base = adminq->base; - struct ionic_admin_cmd *q_desc; - int err = 0; - - rte_spinlock_lock(&lif->adminq_lock); - - if (ionic_q_space_avail(adminq) < 1) { - err = -ENOSPC; - goto err_out; - } - - q_desc = &q_desc_base[adminq->head_idx]; - - memcpy(q_desc, &ctx->cmd, sizeof(ctx->cmd)); - - ionic_q_post(adminq, true, ionic_adminq_cb, ctx); - -err_out: - rte_spinlock_unlock(&lif->adminq_lock); - - return err; -} diff --git a/drivers/net/ionic/ionic_dev.h b/drivers/net/ionic/ionic_dev.h index e1b93b1b05..5b258a9e65 100644 --- a/drivers/net/ionic/ionic_dev.h +++ b/drivers/net/ionic/ionic_dev.h @@ -184,17 +184,6 @@ struct ionic_cq { rte_iova_t base_pa; }; -/** ionic_admin_ctx - Admin command context. - * @pending_work: Flag that indicates a completion. - * @cmd: Admin command (64B) to be copied to the queue. - * @comp: Admin completion (16B) copied from the queue. - */ -struct ionic_admin_ctx { - bool pending_work; - union ionic_adminq_cmd cmd; - union ionic_adminq_comp comp; -}; - struct ionic_lif; struct ionic_adapter; struct ionic_qcq; @@ -255,8 +244,6 @@ void ionic_q_map(struct ionic_queue *q, void *base, rte_iova_t base_pa); void ionic_q_sg_map(struct ionic_queue *q, void *base, rte_iova_t base_pa); void ionic_q_post(struct ionic_queue *q, bool ring_doorbell, desc_cb cb, void *cb_arg); -void ionic_q_service(struct ionic_queue *q, uint32_t cq_desc_index, - uint32_t stop_index, void *service_cb_arg); static inline uint32_t ionic_q_space_avail(struct ionic_queue *q) @@ -279,6 +266,4 @@ ionic_q_flush(struct ionic_queue *q) rte_write64(rte_cpu_to_le_64(val), q->db); } -int ionic_adminq_post(struct ionic_lif *lif, struct ionic_admin_ctx *ctx); - #endif /* _IONIC_DEV_H_ */ diff --git a/drivers/net/ionic/ionic_lif.c b/drivers/net/ionic/ionic_lif.c index a9fe34c0f2..52bc0b178d 100644 --- a/drivers/net/ionic/ionic_lif.c +++ b/drivers/net/ionic/ionic_lif.c @@ -1163,22 +1163,6 @@ ionic_lif_notifyq_deinit(struct ionic_lif *lif) nqcq->flags &= ~IONIC_QCQ_F_INITED; } -bool -ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index, - void *cb_arg __rte_unused) -{ - struct ionic_admin_comp *cq_desc_base = cq->base; - struct ionic_admin_comp *cq_desc = &cq_desc_base[cq_desc_index]; - struct ionic_qcq *qcq = IONIC_CQ_TO_QCQ(cq); - - if (!color_match(cq_desc->color, cq->done_color)) - return false; - - ionic_q_service(&qcq->q, cq_desc_index, cq_desc->comp_index, NULL); - - return true; -} - /* This acts like ionic_napi */ int ionic_qcq_service(struct ionic_qcq *qcq, int budget, ionic_cq_cb cb, diff --git a/drivers/net/ionic/ionic_lif.h b/drivers/net/ionic/ionic_lif.h index 1a898a0ef9..c4cb7514fb 100644 --- a/drivers/net/ionic/ionic_lif.h +++ b/drivers/net/ionic/ionic_lif.h @@ -154,8 +154,6 @@ void ionic_lif_reset(struct ionic_lif *lif); int ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr); void ionic_intr_free(struct ionic_lif *lif, struct ionic_intr_info *intr); -bool ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index, - void *cb_arg); int ionic_qcq_service(struct ionic_qcq *qcq, int budget, ionic_cq_cb cb, void *cb_arg); diff --git a/drivers/net/ionic/ionic_main.c b/drivers/net/ionic/ionic_main.c index 3f1a764888..012103921d 100644 --- a/drivers/net/ionic/ionic_main.c +++ b/drivers/net/ionic/ionic_main.c @@ -122,7 +122,7 @@ ionic_opcode_to_str(enum ionic_cmd_opcode opcode) } } -int +static int ionic_adminq_check_err(struct ionic_admin_ctx *ctx, bool timeout) { const char *name; @@ -145,8 +145,81 @@ ionic_adminq_check_err(struct ionic_admin_ctx *ctx, bool timeout) return 0; } +static bool +ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index, + void *cb_arg __rte_unused) +{ + struct ionic_admin_comp *cq_desc_base = cq->base; + struct ionic_admin_comp *cq_desc = &cq_desc_base[cq_desc_index]; + struct ionic_qcq *qcq = IONIC_CQ_TO_QCQ(cq); + struct ionic_queue *q = &qcq->q; + struct ionic_admin_ctx *ctx; + struct ionic_desc_info *desc_info; + uint16_t curr_q_tail_idx; + uint16_t stop_index; + + if (!color_match(cq_desc->color, cq->done_color)) + return false; + + stop_index = rte_le_to_cpu_16(cq_desc->comp_index); + + do { + desc_info = &q->info[q->tail_idx]; + + ctx = desc_info->cb_arg; + if (ctx) { + memcpy(&ctx->comp, cq_desc, sizeof(*cq_desc)); + + ctx->pending_work = false; /* done */ + } + + curr_q_tail_idx = q->tail_idx; + q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); + } while (curr_q_tail_idx != stop_index); + + return true; +} + +/** ionic_adminq_post - Post an admin command. + * @lif: Handle to lif. + * @cmd_ctx: Api admin command context. + * + * Post the command to an admin queue in the ethernet driver. If this command + * succeeds, then the command has been posted, but that does not indicate a + * completion. If this command returns success, then the completion callback + * will eventually be called. + * + * Return: zero or negative error status. + */ +static int +ionic_adminq_post(struct ionic_lif *lif, struct ionic_admin_ctx *ctx) +{ + struct ionic_queue *q = &lif->adminqcq->q; + struct ionic_admin_cmd *q_desc_base = q->base; + struct ionic_admin_cmd *q_desc; + int err = 0; + + rte_spinlock_lock(&lif->adminq_lock); + + if (ionic_q_space_avail(q) < 1) { + err = -ENOSPC; + goto err_out; + } + + q_desc = &q_desc_base[q->head_idx]; + + memcpy(q_desc, &ctx->cmd, sizeof(ctx->cmd)); + + ionic_q_post(q, true, NULL, ctx); + +err_out: + rte_spinlock_unlock(&lif->adminq_lock); + + return err; +} + static int -ionic_wait_ctx_for_completion(struct ionic_lif *lif, struct ionic_qcq *qcq, +ionic_adminq_wait_for_completion(struct ionic_lif *lif, struct ionic_admin_ctx *ctx, unsigned long max_wait) { unsigned long step_usec = IONIC_DEVCMD_CHECK_PERIOD_US; @@ -156,12 +229,13 @@ ionic_wait_ctx_for_completion(struct ionic_lif *lif, struct ionic_qcq *qcq, while (ctx->pending_work && elapsed_usec < max_wait_usec) { /* - * Locking here as adminq is served inline (this could be called - * from multiple places) + * Locking here as adminq is served inline and could be + * called from multiple places */ rte_spinlock_lock(&lif->adminq_service_lock); - ionic_qcq_service(qcq, budget, ionic_adminq_service, NULL); + ionic_qcq_service(lif->adminqcq, budget, + ionic_adminq_service, NULL); rte_spinlock_unlock(&lif->adminq_service_lock); @@ -175,7 +249,6 @@ ionic_wait_ctx_for_completion(struct ionic_lif *lif, struct ionic_qcq *qcq, int ionic_adminq_post_wait(struct ionic_lif *lif, struct ionic_admin_ctx *ctx) { - struct ionic_qcq *qcq = lif->adminqcq; bool done; int err; @@ -189,7 +262,7 @@ ionic_adminq_post_wait(struct ionic_lif *lif, struct ionic_admin_ctx *ctx) return err; } - done = ionic_wait_ctx_for_completion(lif, qcq, ctx, + done = ionic_adminq_wait_for_completion(lif, ctx, IONIC_DEVCMD_TIMEOUT); return ionic_adminq_check_err(ctx, !done /* timed out */); diff --git a/drivers/net/ionic/ionic_rx_filter.c b/drivers/net/ionic/ionic_rx_filter.c index 4310c91b94..bf57a9fa52 100644 --- a/drivers/net/ionic/ionic_rx_filter.c +++ b/drivers/net/ionic/ionic_rx_filter.c @@ -7,6 +7,7 @@ #include <rte_malloc.h> +#include "ionic.h" #include "ionic_lif.h" #include "ionic_rx_filter.h" -- 2.17.1
Drop the callback part of the object and store only the pointers. This saves a bit of space and simplifies the code. Signed-off-by: Andrew Boyer <aboyer@pensando.io> --- drivers/net/ionic/ionic_dev.c | 8 ++------ drivers/net/ionic/ionic_dev.h | 21 +++++-------------- drivers/net/ionic/ionic_main.c | 8 ++++---- drivers/net/ionic/ionic_rxtx.c | 37 ++++++++++++++++++---------------- 4 files changed, 31 insertions(+), 43 deletions(-) diff --git a/drivers/net/ionic/ionic_dev.c b/drivers/net/ionic/ionic_dev.c index 0eb9f6f21a..74ac2e67a5 100644 --- a/drivers/net/ionic/ionic_dev.c +++ b/drivers/net/ionic/ionic_dev.c @@ -452,13 +452,9 @@ ionic_q_sg_map(struct ionic_queue *q, void *base, rte_iova_t base_pa) } void -ionic_q_post(struct ionic_queue *q, bool ring_doorbell, desc_cb cb, - void *cb_arg) +ionic_q_post(struct ionic_queue *q, bool ring_doorbell, void *cb_arg) { - struct ionic_desc_info *head = &q->info[q->head_idx]; - - head->cb = cb; - head->cb_arg = cb_arg; + q->info[q->head_idx] = cb_arg; q->head_idx = (q->head_idx + 1) & (q->num_descs - 1); diff --git a/drivers/net/ionic/ionic_dev.h b/drivers/net/ionic/ionic_dev.h index 5b258a9e65..c51c2a9ae5 100644 --- a/drivers/net/ionic/ionic_dev.h +++ b/drivers/net/ionic/ionic_dev.h @@ -128,22 +128,12 @@ struct ionic_dev { uint32_t port_info_sz; }; -struct ionic_queue; -struct ionic_desc_info; - -typedef void (*desc_cb)(struct ionic_queue *q, - uint32_t q_desc_index, - uint32_t cq_desc_index, - void *cb_arg, void *service_cb_arg); - -struct ionic_desc_info { - desc_cb cb; - void *cb_arg; -}; - #define Q_NEXT_TO_POST(_q, _n) (((_q)->head_idx + (_n)) & ((_q)->size_mask)) #define Q_NEXT_TO_SRVC(_q, _n) (((_q)->tail_idx + (_n)) & ((_q)->size_mask)) +#define IONIC_INFO_IDX(_q, _i) (_i) +#define IONIC_INFO_PTR(_q, _i) (&(_q)->info[IONIC_INFO_IDX((_q), _i)]) + struct ionic_queue { struct ionic_dev *idev; struct ionic_lif *lif; @@ -154,9 +144,9 @@ struct ionic_queue { uint32_t hw_type; void *base; void *sg_base; + void **info; rte_iova_t base_pa; rte_iova_t sg_base_pa; - struct ionic_desc_info *info; uint32_t tail_idx; uint32_t head_idx; uint32_t num_descs; @@ -242,8 +232,7 @@ int ionic_q_init(struct ionic_lif *lif, struct ionic_dev *idev, size_t desc_size, size_t sg_desc_size); void ionic_q_map(struct ionic_queue *q, void *base, rte_iova_t base_pa); void ionic_q_sg_map(struct ionic_queue *q, void *base, rte_iova_t base_pa); -void ionic_q_post(struct ionic_queue *q, bool ring_doorbell, desc_cb cb, - void *cb_arg); +void ionic_q_post(struct ionic_queue *q, bool ring_doorbell, void *cb_arg); static inline uint32_t ionic_q_space_avail(struct ionic_queue *q) diff --git a/drivers/net/ionic/ionic_main.c b/drivers/net/ionic/ionic_main.c index 012103921d..12f2b26570 100644 --- a/drivers/net/ionic/ionic_main.c +++ b/drivers/net/ionic/ionic_main.c @@ -154,9 +154,9 @@ ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index, struct ionic_qcq *qcq = IONIC_CQ_TO_QCQ(cq); struct ionic_queue *q = &qcq->q; struct ionic_admin_ctx *ctx; - struct ionic_desc_info *desc_info; uint16_t curr_q_tail_idx; uint16_t stop_index; + void **info; if (!color_match(cq_desc->color, cq->done_color)) return false; @@ -164,9 +164,9 @@ ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index, stop_index = rte_le_to_cpu_16(cq_desc->comp_index); do { - desc_info = &q->info[q->tail_idx]; + info = IONIC_INFO_PTR(q, q->tail_idx); - ctx = desc_info->cb_arg; + ctx = info[0]; if (ctx) { memcpy(&ctx->comp, cq_desc, sizeof(*cq_desc)); @@ -210,7 +210,7 @@ ionic_adminq_post(struct ionic_lif *lif, struct ionic_admin_ctx *ctx) memcpy(q_desc, &ctx->cmd, sizeof(ctx->cmd)); - ionic_q_post(q, true, NULL, ctx); + ionic_q_post(q, true, ctx); err_out: rte_spinlock_unlock(&lif->adminq_lock); diff --git a/drivers/net/ionic/ionic_rxtx.c b/drivers/net/ionic/ionic_rxtx.c index 76de82882d..d736e24b02 100644 --- a/drivers/net/ionic/ionic_rxtx.c +++ b/drivers/net/ionic/ionic_rxtx.c @@ -72,10 +72,10 @@ ionic_tx_flush(struct ionic_qcq *txq) { struct ionic_cq *cq = &txq->cq; struct ionic_queue *q = &txq->q; - struct ionic_desc_info *q_desc_info; struct rte_mbuf *txm, *next; struct ionic_txq_comp *cq_desc_base = cq->base; struct ionic_txq_comp *cq_desc; + void **info; u_int32_t comp_index = (u_int32_t)-1; cq_desc = &cq_desc_base[cq->tail_idx]; @@ -96,7 +96,7 @@ ionic_tx_flush(struct ionic_qcq *txq) if (comp_index != (u_int32_t)-1) { while (q->tail_idx != comp_index) { - q_desc_info = &q->info[q->tail_idx]; + info = IONIC_INFO_PTR(q, q->tail_idx); q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); @@ -109,7 +109,7 @@ ionic_tx_flush(struct ionic_qcq *txq) * Note: you can just use rte_pktmbuf_free, * but this loop is faster */ - txm = q_desc_info->cb_arg; + txm = info[0]; while (txm != NULL) { next = txm->next; rte_pktmbuf_free_seg(txm); @@ -311,7 +311,7 @@ ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc, desc->hdr_len = hdrlen; desc->mss = mss; - ionic_q_post(q, done, NULL, done ? txm : NULL); + ionic_q_post(q, done, done ? txm : NULL); } static struct ionic_txq_desc * @@ -511,7 +511,7 @@ ionic_tx(struct ionic_qcq *txq, struct rte_mbuf *txm, txm_seg = txm_seg->next; } - ionic_q_post(q, not_xmit_more, NULL, txm); + ionic_q_post(q, not_xmit_more, txm); return 0; } @@ -639,12 +639,12 @@ static void __rte_cold ionic_rx_empty(struct ionic_queue *q) { struct ionic_qcq *rxq = IONIC_Q_TO_QCQ(q); - struct ionic_desc_info *cur; struct rte_mbuf *mbuf; + void **info; while (q->tail_idx != q->head_idx) { - cur = &q->info[q->tail_idx]; - mbuf = cur->cb_arg; + info = IONIC_INFO_PTR(q, q->tail_idx); + mbuf = info[0]; rte_mempool_put(rxq->mb_pool, mbuf); q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); @@ -743,12 +743,11 @@ ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, static __rte_always_inline void ionic_rx_clean(struct ionic_queue *q, uint32_t q_desc_index, uint32_t cq_desc_index, - void *cb_arg, void *service_cb_arg) + void *service_cb_arg) { struct ionic_rxq_comp *cq_desc_base = q->bound_cq->base; struct ionic_rxq_comp *cq_desc = &cq_desc_base[cq_desc_index]; - struct rte_mbuf *rxm = cb_arg; - struct rte_mbuf *rxm_seg; + struct rte_mbuf *rxm, *rxm_seg; struct ionic_qcq *rxq = IONIC_Q_TO_QCQ(q); uint32_t max_frame_size = rxq->lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; @@ -761,6 +760,13 @@ ionic_rx_clean(struct ionic_queue *q, (rte_pktmbuf_data_room_size(rxq->mb_pool) - RTE_PKTMBUF_HEADROOM); uint32_t left; + void **info; + + assert(q_desc_index == cq_desc->comp_index); + + info = IONIC_INFO_PTR(q, cq_desc->comp_index); + + rxm = info[0]; if (!recv_args) { stats->no_cb_arg++; @@ -898,7 +904,7 @@ ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index, new->addr = old->addr; new->len = old->len; - ionic_q_post(q, true, ionic_rx_clean, mbuf); + ionic_q_post(q, true, mbuf); } static __rte_always_inline int @@ -969,7 +975,7 @@ ionic_rx_fill(struct ionic_qcq *rxq, uint32_t len) ring_doorbell = ((q->head_idx + 1) & IONIC_RX_RING_DOORBELL_STRIDE) == 0; - ionic_q_post(q, ring_doorbell, ionic_rx_clean, rxm); + ionic_q_post(q, ring_doorbell, rxm); } return 0; @@ -1023,7 +1029,6 @@ ionic_rxq_service(struct ionic_qcq *rxq, uint32_t work_to_do, { struct ionic_cq *cq = &rxq->cq; struct ionic_queue *q = &rxq->q; - struct ionic_desc_info *q_desc_info; struct ionic_rxq_comp *cq_desc_base = cq->base; struct ionic_rxq_comp *cq_desc; bool more; @@ -1048,8 +1053,6 @@ ionic_rxq_service(struct ionic_qcq *rxq, uint32_t work_to_do, do { more = (q->tail_idx != cq_desc->comp_index); - q_desc_info = &q->info[q->tail_idx]; - curr_q_tail_idx = q->tail_idx; q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); @@ -1059,7 +1062,7 @@ ionic_rxq_service(struct ionic_qcq *rxq, uint32_t work_to_do, rte_prefetch0(&q->info[q->tail_idx]); ionic_rx_clean(q, curr_q_tail_idx, curr_cq_tail_idx, - q_desc_info->cb_arg, service_cb_arg); + service_cb_arg); } while (more); -- 2.17.1
This will conserve resources. Signed-off-by: Andrew Boyer <aboyer@pensando.io> --- drivers/net/ionic/ionic_dev.c | 6 ------ drivers/net/ionic/ionic_dev.h | 2 -- drivers/net/ionic/ionic_lif.c | 1 - drivers/net/ionic/ionic_rxtx.c | 9 +++++---- 4 files changed, 5 insertions(+), 13 deletions(-) diff --git a/drivers/net/ionic/ionic_dev.c b/drivers/net/ionic/ionic_dev.c index 74ac2e67a5..97fb8acf9b 100644 --- a/drivers/net/ionic/ionic_dev.c +++ b/drivers/net/ionic/ionic_dev.c @@ -383,12 +383,6 @@ ionic_cq_map(struct ionic_cq *cq, void *base, rte_iova_t base_pa) cq->base_pa = base_pa; } -void -ionic_cq_bind(struct ionic_cq *cq, struct ionic_queue *q) -{ - q->bound_cq = cq; -} - uint32_t ionic_cq_service(struct ionic_cq *cq, uint32_t work_to_do, ionic_cq_cb cb, void *cb_arg) diff --git a/drivers/net/ionic/ionic_dev.h b/drivers/net/ionic/ionic_dev.h index c51c2a9ae5..a777695d64 100644 --- a/drivers/net/ionic/ionic_dev.h +++ b/drivers/net/ionic/ionic_dev.h @@ -137,7 +137,6 @@ struct ionic_dev { struct ionic_queue { struct ionic_dev *idev; struct ionic_lif *lif; - struct ionic_cq *bound_cq; uint32_t index; uint32_t type; uint32_t hw_index; @@ -221,7 +220,6 @@ struct ionic_doorbell __iomem *ionic_db_map(struct ionic_lif *lif, int ionic_cq_init(struct ionic_cq *cq, uint16_t num_descs); void ionic_cq_map(struct ionic_cq *cq, void *base, rte_iova_t base_pa); -void ionic_cq_bind(struct ionic_cq *cq, struct ionic_queue *q); typedef bool (*ionic_cq_cb)(struct ionic_cq *cq, uint32_t cq_desc_index, void *cb_arg); uint32_t ionic_cq_service(struct ionic_cq *cq, uint32_t work_to_do, diff --git a/drivers/net/ionic/ionic_lif.c b/drivers/net/ionic/ionic_lif.c index 52bc0b178d..1ca6e050ad 100644 --- a/drivers/net/ionic/ionic_lif.c +++ b/drivers/net/ionic/ionic_lif.c @@ -695,7 +695,6 @@ ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type, ionic_q_map(&new->q, q_base, q_base_pa); ionic_cq_map(&new->cq, cq_base, cq_base_pa); - ionic_cq_bind(&new->cq, &new->q); *qcq = new; diff --git a/drivers/net/ionic/ionic_rxtx.c b/drivers/net/ionic/ionic_rxtx.c index d736e24b02..d615e5c4e1 100644 --- a/drivers/net/ionic/ionic_rxtx.c +++ b/drivers/net/ionic/ionic_rxtx.c @@ -741,14 +741,15 @@ ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, } static __rte_always_inline void -ionic_rx_clean(struct ionic_queue *q, +ionic_rx_clean(struct ionic_qcq *rxq, uint32_t q_desc_index, uint32_t cq_desc_index, void *service_cb_arg) { - struct ionic_rxq_comp *cq_desc_base = q->bound_cq->base; + struct ionic_queue *q = &rxq->q; + struct ionic_cq *cq = &rxq->cq; + struct ionic_rxq_comp *cq_desc_base = cq->base; struct ionic_rxq_comp *cq_desc = &cq_desc_base[cq_desc_index]; struct rte_mbuf *rxm, *rxm_seg; - struct ionic_qcq *rxq = IONIC_Q_TO_QCQ(q); uint32_t max_frame_size = rxq->lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; uint64_t pkt_flags = 0; @@ -1061,7 +1062,7 @@ ionic_rxq_service(struct ionic_qcq *rxq, uint32_t work_to_do, /* q desc info */ rte_prefetch0(&q->info[q->tail_idx]); - ionic_rx_clean(q, curr_q_tail_idx, curr_cq_tail_idx, + ionic_rx_clean(rxq, curr_q_tail_idx, curr_cq_tail_idx, service_cb_arg); } while (more); -- 2.17.1
This will conserve resources. Signed-off-by: Andrew Boyer <aboyer@pensando.io> --- drivers/net/ionic/ionic_lif.c | 7 ------- drivers/net/ionic/ionic_lif.h | 1 - 2 files changed, 8 deletions(-) diff --git a/drivers/net/ionic/ionic_lif.c b/drivers/net/ionic/ionic_lif.c index 1ca6e050ad..462a526935 100644 --- a/drivers/net/ionic/ionic_lif.c +++ b/drivers/net/ionic/ionic_lif.c @@ -584,13 +584,6 @@ ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr) return 0; } -void -ionic_intr_free(struct ionic_lif *lif, struct ionic_intr_info *intr) -{ - if (intr->index != IONIC_INTR_NONE) - lif->adapter->intrs[intr->index] = false; -} - static int ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type, uint32_t index, diff --git a/drivers/net/ionic/ionic_lif.h b/drivers/net/ionic/ionic_lif.h index c4cb7514fb..c850a9c08d 100644 --- a/drivers/net/ionic/ionic_lif.h +++ b/drivers/net/ionic/ionic_lif.h @@ -152,7 +152,6 @@ void ionic_lif_configure_vlan_offload(struct ionic_lif *lif, int mask); void ionic_lif_reset(struct ionic_lif *lif); int ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr); -void ionic_intr_free(struct ionic_lif *lif, struct ionic_intr_info *intr); int ionic_qcq_service(struct ionic_qcq *qcq, int budget, ionic_cq_cb cb, void *cb_arg); -- 2.17.1
This will conserve resources. Rename ionic_qcq_alloc() arg from 'base' to 'type_name' for clarity. Signed-off-by: Andrew Boyer <aboyer@pensando.io> --- drivers/net/ionic/ionic_dev.c | 14 +++------- drivers/net/ionic/ionic_dev.h | 34 ++++++++++------------- drivers/net/ionic/ionic_lif.c | 49 ++++++++++++++++++++++------------ drivers/net/ionic/ionic_main.c | 4 +-- drivers/net/ionic/ionic_rxtx.c | 8 +++--- 5 files changed, 56 insertions(+), 53 deletions(-) diff --git a/drivers/net/ionic/ionic_dev.c b/drivers/net/ionic/ionic_dev.c index 97fb8acf9b..cfaf4abc23 100644 --- a/drivers/net/ionic/ionic_dev.c +++ b/drivers/net/ionic/ionic_dev.c @@ -405,26 +405,20 @@ ionic_cq_service(struct ionic_cq *cq, uint32_t work_to_do, } int -ionic_q_init(struct ionic_lif *lif, struct ionic_dev *idev, - struct ionic_queue *q, uint32_t index, uint32_t num_descs, - size_t desc_size, size_t sg_desc_size) +ionic_q_init(struct ionic_queue *q, uint32_t index, uint16_t num_descs) { uint32_t ring_size; - if (desc_size == 0 || !rte_is_power_of_2(num_descs)) + if (!rte_is_power_of_2(num_descs)) return -EINVAL; ring_size = rte_log2_u32(num_descs); - if (ring_size < 2 || ring_size > 16) return -EINVAL; - q->lif = lif; - q->idev = idev; q->index = index; q->num_descs = num_descs; - q->desc_size = desc_size; - q->sg_desc_size = sg_desc_size; + q->size_mask = num_descs - 1; q->head_idx = 0; q->tail_idx = 0; @@ -450,7 +444,7 @@ ionic_q_post(struct ionic_queue *q, bool ring_doorbell, void *cb_arg) { q->info[q->head_idx] = cb_arg; - q->head_idx = (q->head_idx + 1) & (q->num_descs - 1); + q->head_idx = Q_NEXT_TO_POST(q, 1); if (ring_doorbell) ionic_q_flush(q); diff --git a/drivers/net/ionic/ionic_dev.h b/drivers/net/ionic/ionic_dev.h index a777695d64..2f27e63646 100644 --- a/drivers/net/ionic/ionic_dev.h +++ b/drivers/net/ionic/ionic_dev.h @@ -135,25 +135,21 @@ struct ionic_dev { #define IONIC_INFO_PTR(_q, _i) (&(_q)->info[IONIC_INFO_IDX((_q), _i)]) struct ionic_queue { - struct ionic_dev *idev; - struct ionic_lif *lif; - uint32_t index; - uint32_t type; - uint32_t hw_index; - uint32_t hw_type; + uint16_t num_descs; + uint16_t head_idx; + uint16_t tail_idx; + uint16_t size_mask; + uint8_t type; + uint8_t hw_type; void *base; void *sg_base; + struct ionic_doorbell __iomem *db; void **info; + + uint32_t index; + uint32_t hw_index; rte_iova_t base_pa; rte_iova_t sg_base_pa; - uint32_t tail_idx; - uint32_t head_idx; - uint32_t num_descs; - uint32_t desc_size; - uint32_t sg_desc_size; - uint32_t qid; - uint32_t qtype; - struct ionic_doorbell __iomem *db; }; #define IONIC_INTR_NONE (-1) @@ -220,22 +216,20 @@ struct ionic_doorbell __iomem *ionic_db_map(struct ionic_lif *lif, int ionic_cq_init(struct ionic_cq *cq, uint16_t num_descs); void ionic_cq_map(struct ionic_cq *cq, void *base, rte_iova_t base_pa); -typedef bool (*ionic_cq_cb)(struct ionic_cq *cq, uint32_t cq_desc_index, +typedef bool (*ionic_cq_cb)(struct ionic_cq *cq, uint16_t cq_desc_index, void *cb_arg); uint32_t ionic_cq_service(struct ionic_cq *cq, uint32_t work_to_do, ionic_cq_cb cb, void *cb_arg); -int ionic_q_init(struct ionic_lif *lif, struct ionic_dev *idev, - struct ionic_queue *q, uint32_t index, uint32_t num_descs, - size_t desc_size, size_t sg_desc_size); +int ionic_q_init(struct ionic_queue *q, uint32_t index, uint16_t num_descs); void ionic_q_map(struct ionic_queue *q, void *base, rte_iova_t base_pa); void ionic_q_sg_map(struct ionic_queue *q, void *base, rte_iova_t base_pa); void ionic_q_post(struct ionic_queue *q, bool ring_doorbell, void *cb_arg); -static inline uint32_t +static inline uint16_t ionic_q_space_avail(struct ionic_queue *q) { - uint32_t avail = q->tail_idx; + uint16_t avail = q->tail_idx; if (q->head_idx >= avail) avail += q->num_descs - q->head_idx - 1; diff --git a/drivers/net/ionic/ionic_lif.c b/drivers/net/ionic/ionic_lif.c index 462a526935..fcd6fca9a3 100644 --- a/drivers/net/ionic/ionic_lif.c +++ b/drivers/net/ionic/ionic_lif.c @@ -34,7 +34,7 @@ int ionic_qcq_enable(struct ionic_qcq *qcq) { struct ionic_queue *q = &qcq->q; - struct ionic_lif *lif = q->lif; + struct ionic_lif *lif = qcq->lif; struct ionic_admin_ctx ctx = { .pending_work = true, .cmd.q_control = { @@ -52,7 +52,7 @@ int ionic_qcq_disable(struct ionic_qcq *qcq) { struct ionic_queue *q = &qcq->q; - struct ionic_lif *lif = q->lif; + struct ionic_lif *lif = qcq->lif; struct ionic_admin_ctx ctx = { .pending_work = true, .cmd.q_control = { @@ -585,16 +585,17 @@ ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr) } static int -ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type, +ionic_qcq_alloc(struct ionic_lif *lif, + uint8_t type, uint32_t index, - const char *base, uint32_t flags, - uint32_t num_descs, - uint32_t desc_size, - uint32_t cq_desc_size, - uint32_t sg_desc_size, + const char *type_name, + uint16_t flags, + uint16_t num_descs, + uint16_t desc_size, + uint16_t cq_desc_size, + uint16_t sg_desc_size, struct ionic_qcq **qcq) { - struct ionic_dev *idev = &lif->adapter->idev; struct ionic_qcq *new; uint32_t q_size, cq_size, sg_size, total_size; void *q_base, *cq_base, *sg_base; @@ -642,8 +643,7 @@ ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type, new->q.type = type; - err = ionic_q_init(lif, idev, &new->q, index, num_descs, - desc_size, sg_desc_size); + err = ionic_q_init(&new->q, index, num_descs); if (err) { IONIC_PRINT(ERR, "Queue initialization failed"); goto err_out_free_info; @@ -656,7 +656,7 @@ ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type, } new->base_z = rte_eth_dma_zone_reserve(lif->eth_dev, - base /* name */, index /* queue_idx */, + type_name, index /* queue_idx */, total_size, IONIC_ALIGN, socket_id); if (!new->base_z) { @@ -727,7 +727,11 @@ ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t nrxq_descs, int err = -ENOMEM; flags = IONIC_QCQ_F_SG; - err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, index, "rx", flags, + err = ionic_qcq_alloc(lif, + IONIC_QTYPE_RXQ, + index, + "rx", + flags, nrxq_descs, sizeof(struct ionic_rxq_desc), sizeof(struct ionic_rxq_comp), @@ -749,7 +753,11 @@ ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t ntxq_descs, int err = -ENOMEM; flags = IONIC_QCQ_F_SG; - err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, index, "tx", flags, + err = ionic_qcq_alloc(lif, + IONIC_QTYPE_TXQ, + index, + "tx", + flags, ntxq_descs, sizeof(struct ionic_txq_desc), sizeof(struct ionic_txq_comp), @@ -770,7 +778,11 @@ ionic_admin_qcq_alloc(struct ionic_lif *lif) int err = -ENOMEM; flags = 0; - err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags, + err = ionic_qcq_alloc(lif, + IONIC_QTYPE_ADMINQ, + 0, + "admin", + flags, IONIC_ADMINQ_LENGTH, sizeof(struct ionic_admin_cmd), sizeof(struct ionic_admin_comp), @@ -790,7 +802,10 @@ ionic_notify_qcq_alloc(struct ionic_lif *lif) uint32_t flags = 0; int err = -ENOMEM; - err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notify", + err = ionic_qcq_alloc(lif, + IONIC_QTYPE_NOTIFYQ, + 0, + "notify", flags, IONIC_NOTIFYQ_LENGTH, sizeof(struct ionic_notifyq_cmd), @@ -1216,7 +1231,7 @@ ionic_lif_handle_fw_down(struct ionic_lif *lif) } static bool -ionic_notifyq_cb(struct ionic_cq *cq, uint32_t cq_desc_index, void *cb_arg) +ionic_notifyq_cb(struct ionic_cq *cq, uint16_t cq_desc_index, void *cb_arg) { union ionic_notifyq_comp *cq_desc_base = cq->base; union ionic_notifyq_comp *cq_desc = &cq_desc_base[cq_desc_index]; diff --git a/drivers/net/ionic/ionic_main.c b/drivers/net/ionic/ionic_main.c index 12f2b26570..167a97dcf4 100644 --- a/drivers/net/ionic/ionic_main.c +++ b/drivers/net/ionic/ionic_main.c @@ -146,7 +146,7 @@ ionic_adminq_check_err(struct ionic_admin_ctx *ctx, bool timeout) } static bool -ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index, +ionic_adminq_service(struct ionic_cq *cq, uint16_t cq_desc_index, void *cb_arg __rte_unused) { struct ionic_admin_comp *cq_desc_base = cq->base; @@ -174,7 +174,7 @@ ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index, } curr_q_tail_idx = q->tail_idx; - q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); + q->tail_idx = Q_NEXT_TO_SRVC(q, 1); } while (curr_q_tail_idx != stop_index); return true; diff --git a/drivers/net/ionic/ionic_rxtx.c b/drivers/net/ionic/ionic_rxtx.c index d615e5c4e1..6ecb500b9e 100644 --- a/drivers/net/ionic/ionic_rxtx.c +++ b/drivers/net/ionic/ionic_rxtx.c @@ -98,7 +98,7 @@ ionic_tx_flush(struct ionic_qcq *txq) while (q->tail_idx != comp_index) { info = IONIC_INFO_PTR(q, q->tail_idx); - q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); + q->tail_idx = Q_NEXT_TO_SRVC(q, 1); /* Prefetch the next 4 descriptors */ if ((q->tail_idx & 0x3) == 0) @@ -540,7 +540,7 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, while (nb_tx < nb_pkts) { last = (nb_tx == (nb_pkts - 1)); - next_q_head_idx = (q->head_idx + 1) & (q->num_descs - 1); + next_q_head_idx = Q_NEXT_TO_POST(q, 1); if ((next_q_head_idx & 0x3) == 0) { struct ionic_txq_desc *desc_base = q->base; rte_prefetch0(&desc_base[next_q_head_idx]); @@ -647,7 +647,7 @@ ionic_rx_empty(struct ionic_queue *q) mbuf = info[0]; rte_mempool_put(rxq->mb_pool, mbuf); - q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); + q->tail_idx = Q_NEXT_TO_SRVC(q, 1); } } @@ -1055,7 +1055,7 @@ ionic_rxq_service(struct ionic_qcq *rxq, uint32_t work_to_do, more = (q->tail_idx != cq_desc->comp_index); curr_q_tail_idx = q->tail_idx; - q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); + q->tail_idx = Q_NEXT_TO_SRVC(q, 1); /* Prefetch the next 4 descriptors */ if ((q->tail_idx & 0x3) == 0) -- 2.17.1
Create a unique Q-CQ struct for adminq, notifyq, rxq, and txq to reduce the size of each object. Minimize the size of each field to squeeze into as few cachelines as possible. Signed-off-by: Andrew Boyer <aboyer@pensando.io> --- drivers/net/ionic/ionic_lif.c | 153 ++++++++++++++++++--------------- drivers/net/ionic/ionic_lif.h | 62 +++++++++---- drivers/net/ionic/ionic_main.c | 4 +- drivers/net/ionic/ionic_rxtx.c | 129 +++++++++++++-------------- 4 files changed, 195 insertions(+), 153 deletions(-) diff --git a/drivers/net/ionic/ionic_lif.c b/drivers/net/ionic/ionic_lif.c index fcd6fca9a3..87579a09e5 100644 --- a/drivers/net/ionic/ionic_lif.c +++ b/drivers/net/ionic/ionic_lif.c @@ -76,13 +76,13 @@ ionic_lif_stop(struct ionic_lif *lif) lif->state &= ~IONIC_LIF_F_UP; for (i = 0; i < lif->nrxqcqs; i++) { - struct ionic_qcq *rxq = lif->rxqcqs[i]; + struct ionic_rx_qcq *rxq = lif->rxqcqs[i]; if (rxq->flags & IONIC_QCQ_F_INITED) (void)ionic_dev_rx_queue_stop(lif->eth_dev, i); } for (i = 0; i < lif->ntxqcqs; i++) { - struct ionic_qcq *txq = lif->txqcqs[i]; + struct ionic_tx_qcq *txq = lif->txqcqs[i]; if (txq->flags & IONIC_QCQ_F_INITED) (void)ionic_dev_tx_queue_stop(lif->eth_dev, i); } @@ -131,7 +131,7 @@ ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats ls->rx_bcast_bytes; for (i = 0; i < lif->nrxqcqs; i++) { - struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx; + struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats; stats->imissed += rx_stats->no_cb_arg + rx_stats->bad_cq_status + @@ -152,7 +152,7 @@ ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats ls->rx_desc_data_error; for (i = 0; i < num_rx_q_counters; i++) { - struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx; + struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats; stats->q_ipackets[i] = rx_stats->packets; stats->q_ibytes[i] = rx_stats->bytes; stats->q_errors[i] = @@ -173,7 +173,7 @@ ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats ls->tx_bcast_bytes; for (i = 0; i < lif->ntxqcqs; i++) { - struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx; + struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats; stats->oerrors += tx_stats->drop; } @@ -189,7 +189,7 @@ ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats ls->tx_desc_data_error; for (i = 0; i < num_tx_q_counters; i++) { - struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx; + struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats; stats->q_opackets[i] = tx_stats->packets; stats->q_obytes[i] = tx_stats->bytes; } @@ -217,9 +217,9 @@ ionic_lif_reset_stats(struct ionic_lif *lif) uint32_t i; for (i = 0; i < lif->nrxqcqs; i++) { - memset(&lif->rxqcqs[i]->stats.rx, 0, + memset(&lif->rxqcqs[i]->stats, 0, sizeof(struct ionic_rx_stats)); - memset(&lif->txqcqs[i]->stats.tx, 0, + memset(&lif->txqcqs[i]->stats, 0, sizeof(struct ionic_tx_stats)); } @@ -587,6 +587,7 @@ ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr) static int ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type, + size_t struct_size, uint32_t index, const char *type_name, uint16_t flags, @@ -625,16 +626,17 @@ ionic_qcq_alloc(struct ionic_lif *lif, total_size += PAGE_SIZE; } - new = rte_zmalloc("ionic", sizeof(*new), 0); + new = rte_zmalloc("ionic", struct_size, 0); if (!new) { IONIC_PRINT(ERR, "Cannot allocate queue structure"); return -ENOMEM; } new->lif = lif; - new->flags = flags; - new->q.info = rte_zmalloc("ionic", sizeof(*new->q.info) * num_descs, 0); + new->q.info = rte_calloc_socket("ionic", + num_descs, sizeof(void *), + PAGE_SIZE, socket_id); if (!new->q.info) { IONIC_PRINT(ERR, "Cannot allocate queue info"); err = -ENOMEM; @@ -667,7 +669,6 @@ ionic_qcq_alloc(struct ionic_lif *lif, new->base = new->base_z->addr; new->base_pa = new->base_z->iova; - new->total_size = total_size; q_base = new->base; q_base_pa = new->base_pa; @@ -720,15 +721,17 @@ ionic_qcq_free(struct ionic_qcq *qcq) } int -ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t nrxq_descs, - struct ionic_qcq **qcq) +ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, + uint16_t nrxq_descs, struct ionic_rx_qcq **rxq_out) { - uint32_t flags; - int err = -ENOMEM; + struct ionic_rx_qcq *rxq; + uint16_t flags; + int err; flags = IONIC_QCQ_F_SG; err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, + sizeof(struct ionic_rx_qcq), index, "rx", flags, @@ -736,25 +739,30 @@ ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t nrxq_descs, sizeof(struct ionic_rxq_desc), sizeof(struct ionic_rxq_comp), sizeof(struct ionic_rxq_sg_desc), - &lif->rxqcqs[index]); + (struct ionic_qcq **)&rxq); if (err) return err; - *qcq = lif->rxqcqs[index]; + rxq->flags = flags; + + lif->rxqcqs[index] = rxq; + *rxq_out = rxq; return 0; } int -ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t ntxq_descs, - struct ionic_qcq **qcq) +ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, + uint16_t ntxq_descs, struct ionic_tx_qcq **txq_out) { - uint32_t flags; - int err = -ENOMEM; + struct ionic_tx_qcq *txq; + uint16_t flags; + int err; flags = IONIC_QCQ_F_SG; err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, + sizeof(struct ionic_tx_qcq), index, "tx", flags, @@ -762,11 +770,14 @@ ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t ntxq_descs, sizeof(struct ionic_txq_desc), sizeof(struct ionic_txq_comp), sizeof(struct ionic_txq_sg_desc_v1), - &lif->txqcqs[index]); + (struct ionic_qcq **)&txq); if (err) return err; - *qcq = lif->txqcqs[index]; + txq->flags = flags; + + lif->txqcqs[index] = txq; + *txq_out = txq; return 0; } @@ -774,12 +785,12 @@ ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t ntxq_descs, static int ionic_admin_qcq_alloc(struct ionic_lif *lif) { - uint32_t flags; - int err = -ENOMEM; + uint16_t flags = 0; + int err; - flags = 0; err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, + sizeof(struct ionic_admin_qcq), 0, "admin", flags, @@ -787,7 +798,7 @@ ionic_admin_qcq_alloc(struct ionic_lif *lif) sizeof(struct ionic_admin_cmd), sizeof(struct ionic_admin_comp), 0, - &lif->adminqcq); + (struct ionic_qcq **)&lif->adminqcq); if (err) return err; @@ -797,13 +808,14 @@ ionic_admin_qcq_alloc(struct ionic_lif *lif) static int ionic_notify_qcq_alloc(struct ionic_lif *lif) { - struct ionic_qcq *nqcq; + struct ionic_notify_qcq *nqcq; struct ionic_dev *idev = &lif->adapter->idev; - uint32_t flags = 0; - int err = -ENOMEM; + uint16_t flags = 0; + int err; err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, + sizeof(struct ionic_notify_qcq), 0, "notify", flags, @@ -811,13 +823,13 @@ ionic_notify_qcq_alloc(struct ionic_lif *lif) sizeof(struct ionic_notifyq_cmd), sizeof(union ionic_notifyq_comp), 0, - &nqcq); + (struct ionic_qcq **)&nqcq); if (err) return err; err = ionic_intr_alloc(lif, &nqcq->intr); if (err) { - ionic_qcq_free(nqcq); + ionic_qcq_free(&nqcq->qcq); return err; } @@ -1002,12 +1014,12 @@ void ionic_lif_free(struct ionic_lif *lif) { if (lif->notifyqcq) { - ionic_qcq_free(lif->notifyqcq); + ionic_qcq_free(&lif->notifyqcq->qcq); lif->notifyqcq = NULL; } if (lif->adminqcq) { - ionic_qcq_free(lif->adminqcq); + ionic_qcq_free(&lif->adminqcq->qcq); lif->adminqcq = NULL; } @@ -1137,28 +1149,28 @@ ionic_lif_rss_teardown(struct ionic_lif *lif) } } -static void -ionic_lif_qcq_deinit(struct ionic_qcq *qcq) +void +ionic_lif_txq_deinit(struct ionic_tx_qcq *txq) { - qcq->flags &= ~IONIC_QCQ_F_INITED; + txq->flags &= ~IONIC_QCQ_F_INITED; } void -ionic_lif_txq_deinit(struct ionic_qcq *qcq) +ionic_lif_rxq_deinit(struct ionic_rx_qcq *rxq) { - ionic_lif_qcq_deinit(qcq); + rxq->flags &= ~IONIC_QCQ_F_INITED; } -void -ionic_lif_rxq_deinit(struct ionic_qcq *qcq) +static void +ionic_lif_adminq_deinit(struct ionic_lif *lif) { - ionic_lif_qcq_deinit(qcq); + lif->adminqcq->flags &= ~IONIC_QCQ_F_INITED; } static void ionic_lif_notifyq_deinit(struct ionic_lif *lif) { - struct ionic_qcq *nqcq = lif->notifyqcq; + struct ionic_notify_qcq *nqcq = lif->notifyqcq; struct ionic_dev *idev = &lif->adapter->idev; if (!(nqcq->flags & IONIC_QCQ_F_INITED)) @@ -1283,26 +1295,27 @@ int ionic_notifyq_handler(struct ionic_lif *lif, int budget) { struct ionic_dev *idev = &lif->adapter->idev; - struct ionic_qcq *qcq = lif->notifyqcq; + struct ionic_notify_qcq *nqcq = lif->notifyqcq; uint32_t work_done; - if (!(qcq->flags & IONIC_QCQ_F_INITED)) { + if (!(nqcq->flags & IONIC_QCQ_F_INITED)) { IONIC_PRINT(DEBUG, "Notifyq not yet initialized"); return -1; } - ionic_intr_mask(idev->intr_ctrl, qcq->intr.index, + ionic_intr_mask(idev->intr_ctrl, nqcq->intr.index, IONIC_INTR_MASK_SET); - work_done = ionic_qcq_service(qcq, budget, ionic_notifyq_cb, lif); + work_done = ionic_qcq_service(&nqcq->qcq, budget, + ionic_notifyq_cb, lif); if (lif->state & IONIC_LIF_F_LINK_CHECK_NEEDED) ionic_link_status_check(lif); - ionic_intr_credits(idev->intr_ctrl, qcq->intr.index, + ionic_intr_credits(idev->intr_ctrl, nqcq->intr.index, work_done, IONIC_INTR_CRED_RESET_COALESCE); - ionic_intr_mask(idev->intr_ctrl, qcq->intr.index, + ionic_intr_mask(idev->intr_ctrl, nqcq->intr.index, IONIC_INTR_MASK_CLEAR); return 0; @@ -1312,12 +1325,12 @@ static int ionic_lif_adminq_init(struct ionic_lif *lif) { struct ionic_dev *idev = &lif->adapter->idev; - struct ionic_qcq *qcq = lif->adminqcq; - struct ionic_queue *q = &qcq->q; + struct ionic_admin_qcq *aqcq = lif->adminqcq; + struct ionic_queue *q = &aqcq->qcq.q; struct ionic_q_init_comp comp; int err; - ionic_dev_cmd_adminq_init(idev, qcq); + ionic_dev_cmd_adminq_init(idev, &aqcq->qcq); err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT); if (err) return err; @@ -1332,7 +1345,7 @@ ionic_lif_adminq_init(struct ionic_lif *lif) IONIC_PRINT(DEBUG, "adminq->hw_index %d", q->hw_index); IONIC_PRINT(DEBUG, "adminq->db %p", q->db); - qcq->flags |= IONIC_QCQ_F_INITED; + aqcq->flags |= IONIC_QCQ_F_INITED; return 0; } @@ -1341,8 +1354,8 @@ static int ionic_lif_notifyq_init(struct ionic_lif *lif) { struct ionic_dev *idev = &lif->adapter->idev; - struct ionic_qcq *qcq = lif->notifyqcq; - struct ionic_queue *q = &qcq->q; + struct ionic_notify_qcq *nqcq = lif->notifyqcq; + struct ionic_queue *q = &nqcq->qcq.q; int err; struct ionic_admin_ctx ctx = { @@ -1352,7 +1365,7 @@ ionic_lif_notifyq_init(struct ionic_lif *lif) .type = q->type, .ver = lif->qtype_info[q->type].version, .index = rte_cpu_to_le_32(q->index), - .intr_index = rte_cpu_to_le_16(qcq->intr.index), + .intr_index = rte_cpu_to_le_16(nqcq->intr.index), .flags = rte_cpu_to_le_16(IONIC_QINIT_F_IRQ | IONIC_QINIT_F_ENA), .ring_size = rte_log2_u32(q->num_descs), @@ -1378,10 +1391,10 @@ ionic_lif_notifyq_init(struct ionic_lif *lif) IONIC_PRINT(DEBUG, "notifyq->hw_index %d", q->hw_index); IONIC_PRINT(DEBUG, "notifyq->db %p", q->db); - ionic_intr_mask(idev->intr_ctrl, qcq->intr.index, + ionic_intr_mask(idev->intr_ctrl, nqcq->intr.index, IONIC_INTR_MASK_CLEAR); - qcq->flags |= IONIC_QCQ_F_INITED; + nqcq->flags |= IONIC_QCQ_F_INITED; return 0; } @@ -1445,8 +1458,9 @@ ionic_lif_set_features(struct ionic_lif *lif) } int -ionic_lif_txq_init(struct ionic_qcq *qcq) +ionic_lif_txq_init(struct ionic_tx_qcq *txq) { + struct ionic_qcq *qcq = &txq->qcq; struct ionic_queue *q = &qcq->q; struct ionic_lif *lif = qcq->lif; struct ionic_cq *cq = &qcq->cq; @@ -1474,7 +1488,7 @@ ionic_lif_txq_init(struct ionic_qcq *qcq) ctx.cmd.q_init.ring_size); IONIC_PRINT(DEBUG, "txq_init.ver %u", ctx.cmd.q_init.ver); - err = ionic_adminq_post_wait(qcq->lif, &ctx); + err = ionic_adminq_post_wait(lif, &ctx); if (err) return err; @@ -1486,14 +1500,15 @@ ionic_lif_txq_init(struct ionic_qcq *qcq) IONIC_PRINT(DEBUG, "txq->hw_index %d", q->hw_index); IONIC_PRINT(DEBUG, "txq->db %p", q->db); - qcq->flags |= IONIC_QCQ_F_INITED; + txq->flags |= IONIC_QCQ_F_INITED; return 0; } int -ionic_lif_rxq_init(struct ionic_qcq *qcq) +ionic_lif_rxq_init(struct ionic_rx_qcq *rxq) { + struct ionic_qcq *qcq = &rxq->qcq; struct ionic_queue *q = &qcq->q; struct ionic_lif *lif = qcq->lif; struct ionic_cq *cq = &qcq->cq; @@ -1521,7 +1536,7 @@ ionic_lif_rxq_init(struct ionic_qcq *qcq) ctx.cmd.q_init.ring_size); IONIC_PRINT(DEBUG, "rxq_init.ver %u", ctx.cmd.q_init.ver); - err = ionic_adminq_post_wait(qcq->lif, &ctx); + err = ionic_adminq_post_wait(lif, &ctx); if (err) return err; @@ -1529,7 +1544,7 @@ ionic_lif_rxq_init(struct ionic_qcq *qcq) q->hw_index = rte_le_to_cpu_32(ctx.comp.q_init.hw_index); q->db = ionic_db_map(lif, q); - qcq->flags |= IONIC_QCQ_F_INITED; + rxq->flags |= IONIC_QCQ_F_INITED; IONIC_PRINT(DEBUG, "rxq->hw_type %d", q->hw_type); IONIC_PRINT(DEBUG, "rxq->hw_index %d", q->hw_index); @@ -1634,7 +1649,7 @@ ionic_lif_init(struct ionic_lif *lif) ionic_lif_notifyq_deinit(lif); err_out_adminq_deinit: - ionic_lif_qcq_deinit(lif->adminqcq); + ionic_lif_adminq_deinit(lif); return err; } @@ -1648,7 +1663,7 @@ ionic_lif_deinit(struct ionic_lif *lif) ionic_rx_filters_deinit(lif); ionic_lif_rss_teardown(lif); ionic_lif_notifyq_deinit(lif); - ionic_lif_qcq_deinit(lif->adminqcq); + ionic_lif_adminq_deinit(lif); lif->state &= ~IONIC_LIF_F_INITED; } @@ -1788,7 +1803,7 @@ ionic_lif_start(struct ionic_lif *lif) lif->nrxqcqs, lif->ntxqcqs, lif->port_id); for (i = 0; i < lif->nrxqcqs; i++) { - struct ionic_qcq *rxq = lif->rxqcqs[i]; + struct ionic_rx_qcq *rxq = lif->rxqcqs[i]; if (!(rxq->flags & IONIC_QCQ_F_DEFERRED)) { err = ionic_dev_rx_queue_start(lif->eth_dev, i); @@ -1798,7 +1813,7 @@ ionic_lif_start(struct ionic_lif *lif) } for (i = 0; i < lif->ntxqcqs; i++) { - struct ionic_qcq *txq = lif->txqcqs[i]; + struct ionic_tx_qcq *txq = lif->txqcqs[i]; if (!(txq->flags & IONIC_QCQ_F_DEFERRED)) { err = ionic_dev_tx_queue_start(lif->eth_dev, i); diff --git a/drivers/net/ionic/ionic_lif.h b/drivers/net/ionic/ionic_lif.h index c850a9c08d..a8243ebf21 100644 --- a/drivers/net/ionic/ionic_lif.h +++ b/drivers/net/ionic/ionic_lif.h @@ -58,23 +58,47 @@ struct ionic_qcq { struct ionic_queue q; /**< Queue */ struct ionic_cq cq; /**< Completion Queue */ struct ionic_lif *lif; /**< LIF */ - struct rte_mempool *mb_pool; /**< mbuf pool to populate the RX ring */ - union { - struct ionic_tx_stats tx; - struct ionic_rx_stats rx; - } stats; const struct rte_memzone *base_z; void *base; rte_iova_t base_pa; - uint32_t total_size; - uint32_t flags; +}; + +struct ionic_admin_qcq { + struct ionic_qcq qcq; + uint16_t flags; +}; + +struct ionic_notify_qcq { + struct ionic_qcq qcq; + uint16_t flags; + struct ionic_intr_info intr; }; +struct ionic_rx_qcq { + /* cacheline0, cacheline1 */ + struct ionic_qcq qcq; + + /* cacheline2 */ + struct rte_mempool *mb_pool; + uint16_t flags; + + /* cacheline3 (inside stats) */ + struct ionic_rx_stats stats; +}; + +struct ionic_tx_qcq { + /* cacheline0, cacheline1 */ + struct ionic_qcq qcq; + + /* cacheline2 */ + uint16_t flags; + + struct ionic_tx_stats stats; +}; + #define IONIC_Q_TO_QCQ(_q) container_of(_q, struct ionic_qcq, q) #define IONIC_CQ_TO_QCQ(_cq) container_of(_cq, struct ionic_qcq, cq) -#define IONIC_Q_TO_TX_STATS(q) (&IONIC_Q_TO_QCQ(q)->stats.tx) -#define IONIC_Q_TO_RX_STATS(q) (&IONIC_Q_TO_QCQ(q)->stats.rx) struct ionic_qtype_info { uint8_t version; @@ -104,10 +128,10 @@ struct ionic_lif { uint32_t nrxqcqs; rte_spinlock_t adminq_lock; rte_spinlock_t adminq_service_lock; - struct ionic_qcq *adminqcq; - struct ionic_qcq *notifyqcq; - struct ionic_qcq **txqcqs; - struct ionic_qcq **rxqcqs; + struct ionic_admin_qcq *adminqcq; + struct ionic_notify_qcq *notifyqcq; + struct ionic_tx_qcq **txqcqs; + struct ionic_rx_qcq **rxqcqs; struct ionic_rx_filters rx_filters; struct ionic_doorbell __iomem *kern_dbpage; uint64_t last_eid; @@ -173,19 +197,19 @@ int ionic_dev_allmulticast_enable(struct rte_eth_dev *dev); int ionic_dev_allmulticast_disable(struct rte_eth_dev *dev); int ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, - uint16_t nrxq_descs, struct ionic_qcq **qcq); + uint16_t nrxq_descs, struct ionic_rx_qcq **qcq_out); int ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, - uint16_t ntxq_descs, struct ionic_qcq **qcq); + uint16_t ntxq_descs, struct ionic_tx_qcq **qcq_out); void ionic_qcq_free(struct ionic_qcq *qcq); int ionic_qcq_enable(struct ionic_qcq *qcq); int ionic_qcq_disable(struct ionic_qcq *qcq); -int ionic_lif_rxq_init(struct ionic_qcq *qcq); -void ionic_lif_rxq_deinit(struct ionic_qcq *qcq); +int ionic_lif_rxq_init(struct ionic_rx_qcq *rxq); +void ionic_lif_rxq_deinit(struct ionic_rx_qcq *rxq); -int ionic_lif_txq_init(struct ionic_qcq *qcq); -void ionic_lif_txq_deinit(struct ionic_qcq *qcq); +int ionic_lif_txq_init(struct ionic_tx_qcq *txq); +void ionic_lif_txq_deinit(struct ionic_tx_qcq *txq); int ionic_lif_rss_config(struct ionic_lif *lif, const uint16_t types, const uint8_t *key, const uint32_t *indir); diff --git a/drivers/net/ionic/ionic_main.c b/drivers/net/ionic/ionic_main.c index 167a97dcf4..0d2e02fdd0 100644 --- a/drivers/net/ionic/ionic_main.c +++ b/drivers/net/ionic/ionic_main.c @@ -194,7 +194,7 @@ ionic_adminq_service(struct ionic_cq *cq, uint16_t cq_desc_index, static int ionic_adminq_post(struct ionic_lif *lif, struct ionic_admin_ctx *ctx) { - struct ionic_queue *q = &lif->adminqcq->q; + struct ionic_queue *q = &lif->adminqcq->qcq.q; struct ionic_admin_cmd *q_desc_base = q->base; struct ionic_admin_cmd *q_desc; int err = 0; @@ -234,7 +234,7 @@ ionic_adminq_wait_for_completion(struct ionic_lif *lif, */ rte_spinlock_lock(&lif->adminq_service_lock); - ionic_qcq_service(lif->adminqcq, budget, + ionic_qcq_service(&lif->adminqcq->qcq, budget, ionic_adminq_service, NULL); rte_spinlock_unlock(&lif->adminq_service_lock); diff --git a/drivers/net/ionic/ionic_rxtx.c b/drivers/net/ionic/ionic_rxtx.c index 6ecb500b9e..5f836f0134 100644 --- a/drivers/net/ionic/ionic_rxtx.c +++ b/drivers/net/ionic/ionic_rxtx.c @@ -59,8 +59,8 @@ void ionic_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, struct rte_eth_txq_info *qinfo) { - struct ionic_qcq *txq = dev->data->tx_queues[queue_id]; - struct ionic_queue *q = &txq->q; + struct ionic_tx_qcq *txq = dev->data->tx_queues[queue_id]; + struct ionic_queue *q = &txq->qcq.q; qinfo->nb_desc = q->num_descs; qinfo->conf.offloads = dev->data->dev_conf.txmode.offloads; @@ -68,10 +68,10 @@ ionic_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, } static __rte_always_inline void -ionic_tx_flush(struct ionic_qcq *txq) +ionic_tx_flush(struct ionic_tx_qcq *txq) { - struct ionic_cq *cq = &txq->cq; - struct ionic_queue *q = &txq->q; + struct ionic_cq *cq = &txq->qcq.cq; + struct ionic_queue *q = &txq->qcq.q; struct rte_mbuf *txm, *next; struct ionic_txq_comp *cq_desc_base = cq->base; struct ionic_txq_comp *cq_desc; @@ -122,19 +122,19 @@ ionic_tx_flush(struct ionic_qcq *txq) void __rte_cold ionic_dev_tx_queue_release(void *tx_queue) { - struct ionic_qcq *txq = (struct ionic_qcq *)tx_queue; + struct ionic_tx_qcq *txq = tx_queue; IONIC_PRINT_CALL(); ionic_lif_txq_deinit(txq); - ionic_qcq_free(txq); + ionic_qcq_free(&txq->qcq); } int __rte_cold ionic_dev_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id) { - struct ionic_qcq *txq; + struct ionic_tx_qcq *txq; IONIC_PRINT(DEBUG, "Stopping TX queue %u", tx_queue_id); @@ -148,7 +148,7 @@ ionic_dev_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id) * before disabling Tx queue */ - ionic_qcq_disable(txq); + ionic_qcq_disable(&txq->qcq); ionic_tx_flush(txq); @@ -161,7 +161,7 @@ ionic_dev_tx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id, const struct rte_eth_txconf *tx_conf) { struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); - struct ionic_qcq *txq; + struct ionic_tx_qcq *txq; uint64_t offloads; int err; @@ -221,7 +221,7 @@ int __rte_cold ionic_dev_tx_queue_start(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id) { uint8_t *tx_queue_state = eth_dev->data->tx_queue_state; - struct ionic_qcq *txq; + struct ionic_tx_qcq *txq; int err; if (tx_queue_state[tx_queue_id] == RTE_ETH_QUEUE_STATE_STARTED) { @@ -233,14 +233,14 @@ ionic_dev_tx_queue_start(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id) txq = eth_dev->data->tx_queues[tx_queue_id]; IONIC_PRINT(DEBUG, "Starting TX queue %u, %u descs", - tx_queue_id, txq->q.num_descs); + tx_queue_id, txq->qcq.q.num_descs); if (!(txq->flags & IONIC_QCQ_F_INITED)) { err = ionic_lif_txq_init(txq); if (err) return err; } else { - ionic_qcq_enable(txq); + ionic_qcq_enable(&txq->qcq); } tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; @@ -315,8 +315,9 @@ ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc, } static struct ionic_txq_desc * -ionic_tx_tso_next(struct ionic_queue *q, struct ionic_txq_sg_elem **elem) +ionic_tx_tso_next(struct ionic_tx_qcq *txq, struct ionic_txq_sg_elem **elem) { + struct ionic_queue *q = &txq->qcq.q; struct ionic_txq_desc *desc_base = q->base; struct ionic_txq_sg_desc_v1 *sg_desc_base = q->sg_base; struct ionic_txq_desc *desc = &desc_base[q->head_idx]; @@ -327,11 +328,11 @@ ionic_tx_tso_next(struct ionic_queue *q, struct ionic_txq_sg_elem **elem) } static int -ionic_tx_tso(struct ionic_qcq *txq, struct rte_mbuf *txm, +ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, bool not_xmit_more) { - struct ionic_queue *q = &txq->q; - struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q); + struct ionic_queue *q = &txq->qcq.q; + struct ionic_tx_stats *stats = &txq->stats; struct ionic_txq_desc *desc; struct ionic_txq_sg_elem *elem; struct rte_mbuf *txm_seg; @@ -375,7 +376,7 @@ ionic_tx_tso(struct ionic_qcq *txq, struct rte_mbuf *txm, left = txm->data_len; data_iova = rte_mbuf_data_iova(txm); - desc = ionic_tx_tso_next(q, &elem); + desc = ionic_tx_tso_next(txq, &elem); start = true; /* Chop data up into desc segments */ @@ -397,7 +398,7 @@ ionic_tx_tso(struct ionic_qcq *txq, struct rte_mbuf *txm, encap, vlan_tci, has_vlan, start, done && not_xmit_more); - desc = ionic_tx_tso_next(q, &elem); + desc = ionic_tx_tso_next(txq, &elem); start = false; seglen = mss; } @@ -439,7 +440,7 @@ ionic_tx_tso(struct ionic_qcq *txq, struct rte_mbuf *txm, encap, vlan_tci, has_vlan, start, done && not_xmit_more); - desc = ionic_tx_tso_next(q, &elem); + desc = ionic_tx_tso_next(txq, &elem); start = false; } @@ -452,16 +453,14 @@ ionic_tx_tso(struct ionic_qcq *txq, struct rte_mbuf *txm, } static __rte_always_inline int -ionic_tx(struct ionic_qcq *txq, struct rte_mbuf *txm, +ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, bool not_xmit_more) { - struct ionic_queue *q = &txq->q; - struct ionic_txq_desc *desc_base = q->base; + struct ionic_queue *q = &txq->qcq.q; + struct ionic_txq_desc *desc, *desc_base = q->base; struct ionic_txq_sg_desc_v1 *sg_desc_base = q->sg_base; - struct ionic_txq_desc *desc = &desc_base[q->head_idx]; - struct ionic_txq_sg_desc_v1 *sg_desc = &sg_desc_base[q->head_idx]; - struct ionic_txq_sg_elem *elem = sg_desc->elems; - struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q); + struct ionic_txq_sg_elem *elem; + struct ionic_tx_stats *stats = &txq->stats; struct rte_mbuf *txm_seg; bool encap; bool has_vlan; @@ -470,6 +469,8 @@ ionic_tx(struct ionic_qcq *txq, struct rte_mbuf *txm, uint8_t opcode = IONIC_TXQ_DESC_OPCODE_CSUM_NONE; uint8_t flags = 0; + desc = &desc_base[q->head_idx]; + if ((ol_flags & PKT_TX_IP_CKSUM) && (txq->flags & IONIC_QCQ_F_CSUM_L3)) { opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW; @@ -502,6 +503,7 @@ ionic_tx(struct ionic_qcq *txq, struct rte_mbuf *txm, desc->len = txm->data_len; desc->vlan_tci = txm->vlan_tci; + elem = sg_desc_base[q->head_idx].elems; txm_seg = txm->next; while (txm_seg != NULL) { elem->len = txm_seg->data_len; @@ -520,9 +522,9 @@ uint16_t ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { - struct ionic_qcq *txq = (struct ionic_qcq *)tx_queue; - struct ionic_queue *q = &txq->q; - struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q); + struct ionic_tx_qcq *txq = tx_queue; + struct ionic_queue *q = &txq->qcq.q; + struct ionic_tx_stats *stats = &txq->stats; uint32_t next_q_head_idx; uint32_t bytes_tx = 0; uint16_t nb_tx = 0; @@ -625,8 +627,8 @@ void ionic_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, struct rte_eth_rxq_info *qinfo) { - struct ionic_qcq *rxq = dev->data->rx_queues[queue_id]; - struct ionic_queue *q = &rxq->q; + struct ionic_rx_qcq *rxq = dev->data->rx_queues[queue_id]; + struct ionic_queue *q = &rxq->qcq.q; qinfo->mp = rxq->mb_pool; qinfo->scattered_rx = dev->data->scattered_rx; @@ -636,9 +638,9 @@ ionic_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, } static void __rte_cold -ionic_rx_empty(struct ionic_queue *q) +ionic_rx_empty(struct ionic_rx_qcq *rxq) { - struct ionic_qcq *rxq = IONIC_Q_TO_QCQ(q); + struct ionic_queue *q = &rxq->qcq.q; struct rte_mbuf *mbuf; void **info; @@ -654,15 +656,18 @@ ionic_rx_empty(struct ionic_queue *q) void __rte_cold ionic_dev_rx_queue_release(void *rx_queue) { - struct ionic_qcq *rxq = (struct ionic_qcq *)rx_queue; + struct ionic_rx_qcq *rxq = rx_queue; + + if (!rxq) + return; IONIC_PRINT_CALL(); - ionic_rx_empty(&rxq->q); + ionic_rx_empty(rxq); ionic_lif_rxq_deinit(rxq); - ionic_qcq_free(rxq); + ionic_qcq_free(&rxq->qcq); } int __rte_cold @@ -674,7 +679,7 @@ ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, struct rte_mempool *mp) { struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); - struct ionic_qcq *rxq; + struct ionic_rx_qcq *rxq; uint64_t offloads; int err; @@ -713,7 +718,8 @@ ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, eth_dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; - err = ionic_rx_qcq_alloc(lif, rx_queue_id, nb_desc, &rxq); + err = ionic_rx_qcq_alloc(lif, rx_queue_id, nb_desc, + &rxq); if (err) { IONIC_PRINT(ERR, "Queue %d allocation failure", rx_queue_id); return -EINVAL; @@ -741,20 +747,20 @@ ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, } static __rte_always_inline void -ionic_rx_clean(struct ionic_qcq *rxq, +ionic_rx_clean(struct ionic_rx_qcq *rxq, uint32_t q_desc_index, uint32_t cq_desc_index, void *service_cb_arg) { - struct ionic_queue *q = &rxq->q; - struct ionic_cq *cq = &rxq->cq; + struct ionic_queue *q = &rxq->qcq.q; + struct ionic_cq *cq = &rxq->qcq.cq; struct ionic_rxq_comp *cq_desc_base = cq->base; struct ionic_rxq_comp *cq_desc = &cq_desc_base[cq_desc_index]; struct rte_mbuf *rxm, *rxm_seg; uint32_t max_frame_size = - rxq->lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; + rxq->qcq.lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; uint64_t pkt_flags = 0; uint32_t pkt_type; - struct ionic_rx_stats *stats = IONIC_Q_TO_RX_STATS(q); + struct ionic_rx_stats *stats = &rxq->stats; struct ionic_rx_service *recv_args = (struct ionic_rx_service *) service_cb_arg; uint32_t buf_size = (uint16_t) @@ -803,7 +809,7 @@ ionic_rx_clean(struct ionic_qcq *rxq, rte_prefetch1((char *)rxm->buf_addr + rxm->data_off); rxm->nb_segs = 1; /* cq_desc->num_sg_elems */ rxm->pkt_len = cq_desc->len; - rxm->port = rxq->lif->port_id; + rxm->port = rxq->qcq.lif->port_id; left = cq_desc->len; @@ -909,13 +915,11 @@ ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index, } static __rte_always_inline int -ionic_rx_fill(struct ionic_qcq *rxq, uint32_t len) +ionic_rx_fill(struct ionic_rx_qcq *rxq, uint32_t len) { - struct ionic_queue *q = &rxq->q; - struct ionic_rxq_desc *desc_base = q->base; - struct ionic_rxq_sg_desc *sg_desc_base = q->sg_base; - struct ionic_rxq_desc *desc; - struct ionic_rxq_sg_desc *sg_desc; + struct ionic_queue *q = &rxq->qcq.q; + struct ionic_rxq_desc *desc, *desc_base = q->base; + struct ionic_rxq_sg_desc *sg_desc, *sg_desc_base = q->sg_base; struct ionic_rxq_sg_elem *elem; rte_iova_t dma_addr; uint32_t i, j, nsegs, buf_size, size; @@ -990,7 +994,7 @@ ionic_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) { uint32_t frame_size = eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; uint8_t *rx_queue_state = eth_dev->data->rx_queue_state; - struct ionic_qcq *rxq; + struct ionic_rx_qcq *rxq; int err; if (rx_queue_state[rx_queue_id] == RTE_ETH_QUEUE_STATE_STARTED) { @@ -1002,14 +1006,14 @@ ionic_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) rxq = eth_dev->data->rx_queues[rx_queue_id]; IONIC_PRINT(DEBUG, "Starting RX queue %u, %u descs (size: %u)", - rx_queue_id, rxq->q.num_descs, frame_size); + rx_queue_id, rxq->qcq.q.num_descs, frame_size); if (!(rxq->flags & IONIC_QCQ_F_INITED)) { err = ionic_lif_rxq_init(rxq); if (err) return err; } else { - ionic_qcq_enable(rxq); + ionic_qcq_enable(&rxq->qcq); } /* Allocate buffers for descriptor rings */ @@ -1025,13 +1029,12 @@ ionic_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) } static __rte_always_inline void -ionic_rxq_service(struct ionic_qcq *rxq, uint32_t work_to_do, +ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do, void *service_cb_arg) { - struct ionic_cq *cq = &rxq->cq; - struct ionic_queue *q = &rxq->q; - struct ionic_rxq_comp *cq_desc_base = cq->base; - struct ionic_rxq_comp *cq_desc; + struct ionic_cq *cq = &rxq->qcq.cq; + struct ionic_queue *q = &rxq->qcq.q; + struct ionic_rxq_comp *cq_desc, *cq_desc_base = cq->base; bool more; uint32_t curr_q_tail_idx, curr_cq_tail_idx; uint32_t work_done = 0; @@ -1080,7 +1083,7 @@ ionic_rxq_service(struct ionic_qcq *rxq, uint32_t work_to_do, int __rte_cold ionic_dev_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) { - struct ionic_qcq *rxq; + struct ionic_rx_qcq *rxq; IONIC_PRINT(DEBUG, "Stopping RX queue %u", rx_queue_id); @@ -1089,7 +1092,7 @@ ionic_dev_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) eth_dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; - ionic_qcq_disable(rxq); + ionic_qcq_disable(&rxq->qcq); /* Flush */ ionic_rxq_service(rxq, -1, NULL); @@ -1101,9 +1104,9 @@ uint16_t ionic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { - struct ionic_qcq *rxq = (struct ionic_qcq *)rx_queue; + struct ionic_rx_qcq *rxq = rx_queue; uint32_t frame_size = - rxq->lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; + rxq->qcq.lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; struct ionic_rx_service service_cb_arg; service_cb_arg.rx_pkts = rx_pkts; -- 2.17.1
Pipe the value from the queue setup routines through to ionic_qcq_alloc(). Signed-off-by: Andrew Boyer <aboyer@pensando.io> --- drivers/net/ionic/ionic_lif.c | 10 +++++++--- drivers/net/ionic/ionic_lif.h | 10 ++++++---- drivers/net/ionic/ionic_rxtx.c | 4 ++-- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/drivers/net/ionic/ionic_lif.c b/drivers/net/ionic/ionic_lif.c index 87579a09e5..dd79068948 100644 --- a/drivers/net/ionic/ionic_lif.c +++ b/drivers/net/ionic/ionic_lif.c @@ -588,6 +588,7 @@ static int ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type, size_t struct_size, + uint32_t socket_id, uint32_t index, const char *type_name, uint16_t flags, @@ -603,7 +604,6 @@ ionic_qcq_alloc(struct ionic_lif *lif, rte_iova_t q_base_pa = 0; rte_iova_t cq_base_pa = 0; rte_iova_t sg_base_pa = 0; - uint32_t socket_id = rte_socket_id(); int err; *qcq = NULL; @@ -721,7 +721,7 @@ ionic_qcq_free(struct ionic_qcq *qcq) } int -ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, +ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t socket_id, uint32_t index, uint16_t nrxq_descs, struct ionic_rx_qcq **rxq_out) { struct ionic_rx_qcq *rxq; @@ -732,6 +732,7 @@ ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, sizeof(struct ionic_rx_qcq), + socket_id, index, "rx", flags, @@ -752,7 +753,7 @@ ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, } int -ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, +ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t socket_id, uint32_t index, uint16_t ntxq_descs, struct ionic_tx_qcq **txq_out) { struct ionic_tx_qcq *txq; @@ -763,6 +764,7 @@ ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, sizeof(struct ionic_tx_qcq), + socket_id, index, "tx", flags, @@ -791,6 +793,7 @@ ionic_admin_qcq_alloc(struct ionic_lif *lif) err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, sizeof(struct ionic_admin_qcq), + rte_socket_id(), 0, "admin", flags, @@ -816,6 +819,7 @@ ionic_notify_qcq_alloc(struct ionic_lif *lif) err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, sizeof(struct ionic_notify_qcq), + rte_socket_id(), 0, "notify", flags, diff --git a/drivers/net/ionic/ionic_lif.h b/drivers/net/ionic/ionic_lif.h index a8243ebf21..ba1471b6e9 100644 --- a/drivers/net/ionic/ionic_lif.h +++ b/drivers/net/ionic/ionic_lif.h @@ -196,10 +196,12 @@ int ionic_dev_promiscuous_disable(struct rte_eth_dev *dev); int ionic_dev_allmulticast_enable(struct rte_eth_dev *dev); int ionic_dev_allmulticast_disable(struct rte_eth_dev *dev); -int ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, - uint16_t nrxq_descs, struct ionic_rx_qcq **qcq_out); -int ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, - uint16_t ntxq_descs, struct ionic_tx_qcq **qcq_out); +int ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t socket_id, + uint32_t index, uint16_t nrxq_descs, + struct ionic_rx_qcq **qcq_out); +int ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t socket_id, + uint32_t index, uint16_t ntxq_descs, + struct ionic_tx_qcq **qcq_out); void ionic_qcq_free(struct ionic_qcq *qcq); int ionic_qcq_enable(struct ionic_qcq *qcq); diff --git a/drivers/net/ionic/ionic_rxtx.c b/drivers/net/ionic/ionic_rxtx.c index 5f836f0134..89b37733b6 100644 --- a/drivers/net/ionic/ionic_rxtx.c +++ b/drivers/net/ionic/ionic_rxtx.c @@ -191,7 +191,7 @@ ionic_dev_tx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id, eth_dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; - err = ionic_tx_qcq_alloc(lif, tx_queue_id, nb_desc, &txq); + err = ionic_tx_qcq_alloc(lif, socket_id, tx_queue_id, nb_desc, &txq); if (err) { IONIC_PRINT(DEBUG, "Queue allocation failure"); return -EINVAL; @@ -718,7 +718,7 @@ ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, eth_dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; - err = ionic_rx_qcq_alloc(lif, rx_queue_id, nb_desc, + err = ionic_rx_qcq_alloc(lif, socket_id, rx_queue_id, nb_desc, &rxq); if (err) { IONIC_PRINT(ERR, "Queue %d allocation failure", rx_queue_id); -- 2.17.1
This improves debuggability. To see the logs, use EAL arg: --log-level=pmd.net.ionic,debug While here, stop counting fragments, but start counting mtods. Signed-off-by: Andrew Boyer <aboyer@pensando.io> --- drivers/net/ionic/ionic_lif.h | 2 +- drivers/net/ionic/ionic_rxtx.c | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/drivers/net/ionic/ionic_lif.h b/drivers/net/ionic/ionic_lif.h index ba1471b6e9..5885aa1546 100644 --- a/drivers/net/ionic/ionic_lif.h +++ b/drivers/net/ionic/ionic_lif.h @@ -34,7 +34,6 @@ struct ionic_tx_stats { uint64_t stop; uint64_t no_csum; uint64_t tso; - uint64_t frags; }; struct ionic_rx_stats { @@ -44,6 +43,7 @@ struct ionic_rx_stats { uint64_t bad_cq_status; uint64_t no_room; uint64_t bad_len; + uint64_t mtods; }; #define IONIC_QCQ_F_INITED BIT(0) diff --git a/drivers/net/ionic/ionic_rxtx.c b/drivers/net/ionic/ionic_rxtx.c index 89b37733b6..fa92fca2f5 100644 --- a/drivers/net/ionic/ionic_rxtx.c +++ b/drivers/net/ionic/ionic_rxtx.c @@ -123,9 +123,13 @@ void __rte_cold ionic_dev_tx_queue_release(void *tx_queue) { struct ionic_tx_qcq *txq = tx_queue; + struct ionic_tx_stats *stats = &txq->stats; IONIC_PRINT_CALL(); + IONIC_PRINT(DEBUG, "TX queue %u pkts %ju tso %ju", + txq->qcq.q.index, stats->packets, stats->tso); + ionic_lif_txq_deinit(txq); ionic_qcq_free(&txq->qcq); @@ -410,7 +414,6 @@ ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, offset = 0; data_iova = rte_mbuf_data_iova(txm_seg); left = txm_seg->data_len; - stats->frags++; while (left > 0) { next_addr = rte_cpu_to_le_64(data_iova + offset); @@ -508,7 +511,6 @@ ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, while (txm_seg != NULL) { elem->len = txm_seg->data_len; elem->addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm_seg)); - stats->frags++; elem++; txm_seg = txm_seg->next; } @@ -657,12 +659,18 @@ void __rte_cold ionic_dev_rx_queue_release(void *rx_queue) { struct ionic_rx_qcq *rxq = rx_queue; + struct ionic_rx_stats *stats; if (!rxq) return; IONIC_PRINT_CALL(); + stats = &rxq->stats; + + IONIC_PRINT(DEBUG, "RX queue %u pkts %ju mtod %ju", + rxq->qcq.q.index, stats->packets, stats->mtods); + ionic_rx_empty(rxq); ionic_lif_rxq_deinit(rxq); @@ -887,6 +895,7 @@ ionic_rx_clean(struct ionic_rx_qcq *rxq, pkt_type = RTE_PTYPE_L2_ETHER_ARP; else pkt_type = RTE_PTYPE_UNKNOWN; + stats->mtods++; break; } } -- 2.17.1
Break it up rather than inlining it, so that we can remove branches from the hot path. Signed-off-by: Andrew Boyer <aboyer@pensando.io> --- drivers/net/ionic/ionic_dev.c | 11 ---------- drivers/net/ionic/ionic_dev.h | 1 - drivers/net/ionic/ionic_main.c | 10 ++++++++- drivers/net/ionic/ionic_rxtx.c | 37 ++++++++++++++++++++++++++++++---- 4 files changed, 42 insertions(+), 17 deletions(-) diff --git a/drivers/net/ionic/ionic_dev.c b/drivers/net/ionic/ionic_dev.c index cfaf4abc23..43e9ca3de3 100644 --- a/drivers/net/ionic/ionic_dev.c +++ b/drivers/net/ionic/ionic_dev.c @@ -438,14 +438,3 @@ ionic_q_sg_map(struct ionic_queue *q, void *base, rte_iova_t base_pa) q->sg_base = base; q->sg_base_pa = base_pa; } - -void -ionic_q_post(struct ionic_queue *q, bool ring_doorbell, void *cb_arg) -{ - q->info[q->head_idx] = cb_arg; - - q->head_idx = Q_NEXT_TO_POST(q, 1); - - if (ring_doorbell) - ionic_q_flush(q); -} diff --git a/drivers/net/ionic/ionic_dev.h b/drivers/net/ionic/ionic_dev.h index 2f27e63646..38c078efdf 100644 --- a/drivers/net/ionic/ionic_dev.h +++ b/drivers/net/ionic/ionic_dev.h @@ -224,7 +224,6 @@ uint32_t ionic_cq_service(struct ionic_cq *cq, uint32_t work_to_do, int ionic_q_init(struct ionic_queue *q, uint32_t index, uint16_t num_descs); void ionic_q_map(struct ionic_queue *q, void *base, rte_iova_t base_pa); void ionic_q_sg_map(struct ionic_queue *q, void *base, rte_iova_t base_pa); -void ionic_q_post(struct ionic_queue *q, bool ring_doorbell, void *cb_arg); static inline uint16_t ionic_q_space_avail(struct ionic_queue *q) diff --git a/drivers/net/ionic/ionic_main.c b/drivers/net/ionic/ionic_main.c index 0d2e02fdd0..9aa7b2e96c 100644 --- a/drivers/net/ionic/ionic_main.c +++ b/drivers/net/ionic/ionic_main.c @@ -197,6 +197,7 @@ ionic_adminq_post(struct ionic_lif *lif, struct ionic_admin_ctx *ctx) struct ionic_queue *q = &lif->adminqcq->qcq.q; struct ionic_admin_cmd *q_desc_base = q->base; struct ionic_admin_cmd *q_desc; + void **info; int err = 0; rte_spinlock_lock(&lif->adminq_lock); @@ -210,7 +211,14 @@ ionic_adminq_post(struct ionic_lif *lif, struct ionic_admin_ctx *ctx) memcpy(q_desc, &ctx->cmd, sizeof(ctx->cmd)); - ionic_q_post(q, true, ctx); + info = IONIC_INFO_PTR(q, q->head_idx); + info[0] = ctx; + + q->head_idx = Q_NEXT_TO_POST(q, 1); + + /* Ring doorbell */ + rte_wmb(); + ionic_q_flush(q); err_out: rte_spinlock_unlock(&lif->adminq_lock); diff --git a/drivers/net/ionic/ionic_rxtx.c b/drivers/net/ionic/ionic_rxtx.c index fa92fca2f5..5236cae211 100644 --- a/drivers/net/ionic/ionic_rxtx.c +++ b/drivers/net/ionic/ionic_rxtx.c @@ -302,6 +302,7 @@ ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc, uint16_t vlan_tci, bool has_vlan, bool start, bool done) { + void **info; uint8_t flags = 0; flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; @@ -315,7 +316,15 @@ ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc, desc->hdr_len = hdrlen; desc->mss = mss; - ionic_q_post(q, done, done ? txm : NULL); + if (done) { + info = IONIC_INFO_PTR(q, q->head_idx); + info[0] = txm; + } + + q->head_idx = Q_NEXT_TO_POST(q, 1); + + if (done) + ionic_q_flush(q); } static struct ionic_txq_desc * @@ -465,6 +474,7 @@ ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, struct ionic_txq_sg_elem *elem; struct ionic_tx_stats *stats = &txq->stats; struct rte_mbuf *txm_seg; + void **info; bool encap; bool has_vlan; uint64_t ol_flags = txm->ol_flags; @@ -473,6 +483,7 @@ ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, uint8_t flags = 0; desc = &desc_base[q->head_idx]; + info = IONIC_INFO_PTR(q, q->head_idx); if ((ol_flags & PKT_TX_IP_CKSUM) && (txq->flags & IONIC_QCQ_F_CSUM_L3)) { @@ -506,7 +517,10 @@ ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, desc->len = txm->data_len; desc->vlan_tci = txm->vlan_tci; + info[0] = txm; + elem = sg_desc_base[q->head_idx].elems; + txm_seg = txm->next; while (txm_seg != NULL) { elem->len = txm_seg->data_len; @@ -515,7 +529,10 @@ ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, txm_seg = txm_seg->next; } - ionic_q_post(q, not_xmit_more, txm); + q->head_idx = Q_NEXT_TO_POST(q, 1); + + if (not_xmit_more) + ionic_q_flush(q); return 0; } @@ -920,7 +937,11 @@ ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index, new->addr = old->addr; new->len = old->len; - ionic_q_post(q, true, mbuf); + q->info[q->head_idx] = mbuf; + + q->head_idx = Q_NEXT_TO_POST(q, 1); + + ionic_q_flush(q); } static __rte_always_inline int @@ -930,6 +951,7 @@ ionic_rx_fill(struct ionic_rx_qcq *rxq, uint32_t len) struct ionic_rxq_desc *desc, *desc_base = q->base; struct ionic_rxq_sg_desc *sg_desc, *sg_desc_base = q->sg_base; struct ionic_rxq_sg_elem *elem; + void **info; rte_iova_t dma_addr; uint32_t i, j, nsegs, buf_size, size; bool ring_doorbell; @@ -947,6 +969,8 @@ ionic_rx_fill(struct ionic_rx_qcq *rxq, uint32_t len) return -ENOMEM; } + info = IONIC_INFO_PTR(q, q->head_idx); + nsegs = (len + buf_size - 1) / buf_size; desc = &desc_base[q->head_idx]; @@ -989,7 +1013,12 @@ ionic_rx_fill(struct ionic_rx_qcq *rxq, uint32_t len) ring_doorbell = ((q->head_idx + 1) & IONIC_RX_RING_DOORBELL_STRIDE) == 0; - ionic_q_post(q, ring_doorbell, rxm); + info[0] = rxm; + + q->head_idx = Q_NEXT_TO_POST(q, 1); + + if (ring_doorbell) + ionic_q_flush(q); } return 0; -- 2.17.1
This improves performance. Signed-off-by: Andrew Boyer <aboyer@pensando.io> --- drivers/net/ionic/ionic_rxtx.c | 41 +++++++++++----------------------- 1 file changed, 13 insertions(+), 28 deletions(-) diff --git a/drivers/net/ionic/ionic_rxtx.c b/drivers/net/ionic/ionic_rxtx.c index 5236cae211..bb67c497a1 100644 --- a/drivers/net/ionic/ionic_rxtx.c +++ b/drivers/net/ionic/ionic_rxtx.c @@ -47,8 +47,6 @@ #include "ionic_lif.h" #include "ionic_rxtx.h" -#define IONIC_RX_RING_DOORBELL_STRIDE (32 - 1) - /********************************************************************* * * TX functions @@ -322,9 +320,6 @@ ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc, } q->head_idx = Q_NEXT_TO_POST(q, 1); - - if (done) - ionic_q_flush(q); } static struct ionic_txq_desc * @@ -341,8 +336,7 @@ ionic_tx_tso_next(struct ionic_tx_qcq *txq, struct ionic_txq_sg_elem **elem) } static int -ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, - bool not_xmit_more) +ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm) { struct ionic_queue *q = &txq->qcq.q; struct ionic_tx_stats *stats = &txq->stats; @@ -410,7 +404,7 @@ ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, hdrlen, mss, encap, vlan_tci, has_vlan, - start, done && not_xmit_more); + start, done); desc = ionic_tx_tso_next(txq, &elem); start = false; seglen = mss; @@ -451,7 +445,7 @@ ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, hdrlen, mss, encap, vlan_tci, has_vlan, - start, done && not_xmit_more); + start, done); desc = ionic_tx_tso_next(txq, &elem); start = false; } @@ -465,8 +459,7 @@ ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, } static __rte_always_inline int -ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, - bool not_xmit_more) +ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm) { struct ionic_queue *q = &txq->qcq.q; struct ionic_txq_desc *desc, *desc_base = q->base; @@ -531,9 +524,6 @@ ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, q->head_idx = Q_NEXT_TO_POST(q, 1); - if (not_xmit_more) - ionic_q_flush(q); - return 0; } @@ -548,7 +538,6 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint32_t bytes_tx = 0; uint16_t nb_tx = 0; int err; - bool last; /* Cleaning old buffers */ ionic_tx_flush(txq); @@ -559,8 +548,6 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, } while (nb_tx < nb_pkts) { - last = (nb_tx == (nb_pkts - 1)); - next_q_head_idx = Q_NEXT_TO_POST(q, 1); if ((next_q_head_idx & 0x3) == 0) { struct ionic_txq_desc *desc_base = q->base; @@ -569,13 +556,11 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, } if (tx_pkts[nb_tx]->ol_flags & PKT_TX_TCP_SEG) - err = ionic_tx_tso(txq, tx_pkts[nb_tx], last); + err = ionic_tx_tso(txq, tx_pkts[nb_tx]); else - err = ionic_tx(txq, tx_pkts[nb_tx], last); + err = ionic_tx(txq, tx_pkts[nb_tx]); if (err) { stats->drop += nb_pkts - nb_tx; - if (nb_tx > 0) - ionic_q_flush(q); break; } @@ -583,6 +568,11 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, nb_tx++; } + if (nb_tx > 0) { + rte_wmb(); + ionic_q_flush(q); + } + stats->packets += nb_tx; stats->bytes += bytes_tx; @@ -954,7 +944,6 @@ ionic_rx_fill(struct ionic_rx_qcq *rxq, uint32_t len) void **info; rte_iova_t dma_addr; uint32_t i, j, nsegs, buf_size, size; - bool ring_doorbell; buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) - RTE_PKTMBUF_HEADROOM); @@ -1010,17 +999,13 @@ ionic_rx_fill(struct ionic_rx_qcq *rxq, uint32_t len) IONIC_PRINT(ERR, "Rx SG size is not sufficient (%d < %d)", size, len); - ring_doorbell = ((q->head_idx + 1) & - IONIC_RX_RING_DOORBELL_STRIDE) == 0; - info[0] = rxm; q->head_idx = Q_NEXT_TO_POST(q, 1); - - if (ring_doorbell) - ionic_q_flush(q); } + ionic_q_flush(q); + return 0; } -- 2.17.1
Rather than dropping the whole burst if some don't fit. This improves performance. Signed-off-by: Andrew Boyer <aboyer@pensando.io> Signed-off-by: Vishwas Danivas <vishwas@pensando.io> --- drivers/net/ionic/ionic_rxtx.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/net/ionic/ionic_rxtx.c b/drivers/net/ionic/ionic_rxtx.c index bb67c497a1..b4bdeabad1 100644 --- a/drivers/net/ionic/ionic_rxtx.c +++ b/drivers/net/ionic/ionic_rxtx.c @@ -536,15 +536,16 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, struct ionic_tx_stats *stats = &txq->stats; uint32_t next_q_head_idx; uint32_t bytes_tx = 0; - uint16_t nb_tx = 0; + uint16_t nb_avail, nb_tx = 0; int err; /* Cleaning old buffers */ ionic_tx_flush(txq); - if (unlikely(ionic_q_space_avail(q) < nb_pkts)) { - stats->stop += nb_pkts; - return 0; + nb_avail = ionic_q_space_avail(q); + if (unlikely(nb_avail < nb_pkts)) { + stats->stop += nb_pkts - nb_avail; + nb_pkts = nb_avail; } while (nb_tx < nb_pkts) { -- 2.17.1
A future patch will allow Tx scatter/gather to be disabled. Store the value in the queue so it can be changed at runtime based on the configuration. Signed-off-by: Andrew Boyer <aboyer@pensando.io> --- drivers/net/ionic/ionic_lif.c | 6 +++++- drivers/net/ionic/ionic_lif.h | 1 + drivers/net/ionic/ionic_rxtx.c | 6 +++--- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/drivers/net/ionic/ionic_lif.c b/drivers/net/ionic/ionic_lif.c index dd79068948..b8023e0632 100644 --- a/drivers/net/ionic/ionic_lif.c +++ b/drivers/net/ionic/ionic_lif.c @@ -757,10 +757,13 @@ ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t socket_id, uint32_t index, uint16_t ntxq_descs, struct ionic_tx_qcq **txq_out) { struct ionic_tx_qcq *txq; - uint16_t flags; + uint16_t flags, num_segs_fw; int err; flags = IONIC_QCQ_F_SG; + + num_segs_fw = IONIC_TX_MAX_SG_ELEMS_V1 + 1; + err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, sizeof(struct ionic_tx_qcq), @@ -777,6 +780,7 @@ ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t socket_id, uint32_t index, return err; txq->flags = flags; + txq->num_segs_fw = num_segs_fw; lif->txqcqs[index] = txq; *txq_out = txq; diff --git a/drivers/net/ionic/ionic_lif.h b/drivers/net/ionic/ionic_lif.h index 5885aa1546..9f00ba2973 100644 --- a/drivers/net/ionic/ionic_lif.h +++ b/drivers/net/ionic/ionic_lif.h @@ -92,6 +92,7 @@ struct ionic_tx_qcq { struct ionic_qcq qcq; /* cacheline2 */ + uint16_t num_segs_fw; /* # segs supported by current FW */ uint16_t flags; struct ionic_tx_stats stats; diff --git a/drivers/net/ionic/ionic_rxtx.c b/drivers/net/ionic/ionic_rxtx.c index b4bdeabad1..b83ea1bcaa 100644 --- a/drivers/net/ionic/ionic_rxtx.c +++ b/drivers/net/ionic/ionic_rxtx.c @@ -598,9 +598,9 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, (PKT_TX_OFFLOAD_MASK ^ IONIC_TX_OFFLOAD_MASK) uint16_t -ionic_prep_pkts(void *tx_queue __rte_unused, struct rte_mbuf **tx_pkts, - uint16_t nb_pkts) +ionic_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { + struct ionic_tx_qcq *txq = tx_queue; struct rte_mbuf *txm; uint64_t offloads; int i = 0; @@ -608,7 +608,7 @@ ionic_prep_pkts(void *tx_queue __rte_unused, struct rte_mbuf **tx_pkts, for (i = 0; i < nb_pkts; i++) { txm = tx_pkts[i]; - if (txm->nb_segs > IONIC_TX_MAX_SG_ELEMS_V1 + 1) { + if (txm->nb_segs > txq->num_segs_fw) { rte_errno = -EINVAL; break; } -- 2.17.1
The completion type was wrong. Don't check the completion if the wait timed out. Fixes: 669c8de67c88 ("net/ionic: support basic LIF") Cc: cardigliano@ntop.org Cc: stable@dpdk.org Signed-off-by: Andrew Boyer <aboyer@pensando.io> --- drivers/net/ionic/ionic_lif.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/net/ionic/ionic_lif.c b/drivers/net/ionic/ionic_lif.c index b8023e0632..cd220abee2 100644 --- a/drivers/net/ionic/ionic_lif.c +++ b/drivers/net/ionic/ionic_lif.c @@ -1605,17 +1605,18 @@ int ionic_lif_init(struct ionic_lif *lif) { struct ionic_dev *idev = &lif->adapter->idev; - struct ionic_q_init_comp comp; + struct ionic_lif_init_comp comp; int err; memset(&lif->stats_base, 0, sizeof(lif->stats_base)); ionic_dev_cmd_lif_init(idev, lif->info_pa); err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT); - ionic_dev_cmd_comp(idev, &comp); if (err) return err; + ionic_dev_cmd_comp(idev, &comp); + lif->hw_index = rte_cpu_to_le_16(comp.hw_index); err = ionic_lif_adminq_init(lif); -- 2.17.1
On 2/16/2021 8:35 PM, Andrew Boyer wrote:
> This patch series reorganizes the main datastructure for each
> queue, struct ionic_qcq. Its constituent struct ionic_queue and
> struct ionic_cq are stripped down first. Then the generic struct
> ionic_qcq is stripped down, and a unique struct is created for
> each queue type.
>
> The adminq code is consolidated into ionic_main.c as part of the
> cleanup.
>
> Next comes some minor performance fixups related to queue posting
> and doorbells.
>
> Finally, a minor improvement to Tx packet prep and a minor fix
> for LIF init.
>
> Signed-off-by: Andrew Boyer <aboyer@pensando.io>
>
> --
> v2:
> * Resend for new DPDK release cycle
> * Insert a new patch "net/ionic: remove unused filter delete function" so
> that even more adminq code can be staticized
> * Update second-to-last patch which was partially applied in 21.02
>
> Andrew Boyer (15):
> net/ionic: cut down completion queue structure
> net/ionic: remove unused filter delete function
> net/ionic: consolidate adminq code
> net/ionic: convert info array to generic pointers
> net/ionic: remove unused field from queue structure
> net/ionic: remove unused interrupt free function
> net/ionic: cut down queue structure
> net/ionic: split up queue-completion queue structure
> net/ionic: use the socket id passed in for Rx and Tx queues
> net/ionic: log queue counters when tearing down
> net/ionic: break up queue post function
> net/ionic: ring doorbell once at the end of each burst
> net/ionic: send as many packets as possible
> net/ionic: store Tx fragment limit in queue
> net/ionic: fix code around lif init devcmd
>
Series applied to dpdk-next-net/main, thanks.