From: Spike Du <spiked@nvidia.com>
To: <matan@nvidia.com>, <viacheslavo@nvidia.com>, <orika@nvidia.com>,
<thomas@monjalon.net>
Cc: <dev@dpdk.org>, <rasland@nvidia.com>
Subject: [RFC 4/6] net/mlx5: add private API to configure Rxq LWM
Date: Fri, 1 Apr 2022 06:22:30 +0300 [thread overview]
Message-ID: <20220401032232.1267376-5-spiked@nvidia.com> (raw)
In-Reply-To: <20220401032232.1267376-1-spiked@nvidia.com>
The new API allows setting/unsetting/modifying an LWM(limit watermark)
event per Rxq.
While the Rx queue fullness reaches the LWM limit, the driver catches
an HW event and invokes the user callback.
Signed-off-by: Spike Du <spiked@nvidia.com>
---
doc/guides/nics/mlx5.rst | 4 ++
doc/guides/rel_notes/release_22_03.rst | 6 +++
drivers/common/mlx5/mlx5_prm.h | 1 +
drivers/net/mlx5/mlx5_rx.c | 88 +++++++++++++++++++++++++++++++++-
drivers/net/mlx5/mlx5_rx.h | 1 +
drivers/net/mlx5/rte_pmd_mlx5.h | 32 +++++++++++++
drivers/net/mlx5/version.map | 1 +
7 files changed, 132 insertions(+), 1 deletion(-)
diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst
index a734d10..0e983a6 100644
--- a/doc/guides/nics/mlx5.rst
+++ b/doc/guides/nics/mlx5.rst
@@ -92,6 +92,7 @@ Features
- Connection tracking.
- Sub-Function representors.
- Sub-Function.
+- Rx queue LWM (Limit WaterMark) configuration.
Limitations
@@ -507,6 +508,9 @@ Limitations
- The NIC egress flow rules on representor port are not supported.
+- LWM:
+ - Doesn't support shared Rx queue and Hairpin Rx queue.
+
Statistics
----------
diff --git a/doc/guides/rel_notes/release_22_03.rst b/doc/guides/rel_notes/release_22_03.rst
index 60e5b4f..0c9d3b6 100644
--- a/doc/guides/rel_notes/release_22_03.rst
+++ b/doc/guides/rel_notes/release_22_03.rst
@@ -187,6 +187,12 @@ New Features
An API was added to get/set an asymmetric crypto session's user data.
+* **Updated Mellanox mlx5 driver.**
+
+ Updated the Mellanox mlx5 driver with new features and improvements, including:
+
+ * Added Rx queue LWM(Limit WaterMark) support.
+
* **Updated Marvell cnxk crypto PMD.**
* Added SHA256-HMAC support in lookaside protocol (IPsec) for CN10K.
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index 44b1822..23b13e3 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -3290,6 +3290,7 @@ struct mlx5_aso_wqe {
enum {
MLX5_EVENT_TYPE_OBJECT_CHANGE = 0x27,
+ MLX5_EVENT_TYPE_SRQ_LIMIT_REACHED = 0x14,
};
enum {
diff --git a/drivers/net/mlx5/mlx5_rx.c b/drivers/net/mlx5/mlx5_rx.c
index f72364e..0390412 100644
--- a/drivers/net/mlx5/mlx5_rx.c
+++ b/drivers/net/mlx5/mlx5_rx.c
@@ -19,15 +19,16 @@
#include <mlx5_prm.h>
#include <mlx5_common.h>
#include <mlx5_common_mr.h>
+#include <rte_pmd_mlx5.h>
#include "mlx5_autoconf.h"
#include "mlx5_defs.h"
#include "mlx5.h"
#include "mlx5_utils.h"
#include "mlx5_rxtx.h"
+#include "mlx5_devx.h"
#include "mlx5_rx.h"
-
static __rte_always_inline uint32_t
rxq_cq_to_pkt_type(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe,
volatile struct mlx5_mini_cqe8 *mcqe);
@@ -1216,3 +1217,88 @@ int mlx5_get_monitor_addr(void *rx_queue, struct rte_power_monitor_cond *pmc)
if (rxq && rxq->lwm_event_rxq_limit_reached)
rxq->lwm_event_rxq_limit_reached(port_id, rxq_idx);
}
+
+int
+rte_pmd_mlx5_config_rxq_lwm(uint16_t port_id, uint16_t rx_queue_id,
+ uint8_t lwm,
+ lwm_event_rxq_limit_reached_t cb)
+{
+ struct rte_eth_dev *dev = &rte_eth_devices[port_id];
+ struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, rx_queue_id);
+ uint16_t event_nums[1] = {MLX5_EVENT_TYPE_SRQ_LIMIT_REACHED};
+ struct mlx5_rxq_data *rxq_data;
+ struct mlx5_priv *priv;
+ uint32_t wqe_cnt;
+ uint64_t cookie;
+ int ret = 0;
+
+ if (!rxq) {
+ rte_errno = EINVAL;
+ return -rte_errno;
+ }
+ rxq_data = &rxq->ctrl->rxq;
+ priv = rxq->priv;
+ /* Ensure the Rq is created by devx. */
+ if (priv->obj_ops.rxq_obj_new != devx_obj_ops.rxq_obj_new) {
+ rte_errno = EINVAL;
+ return -rte_errno;
+ }
+ if (lwm > 99) {
+ DRV_LOG(WARNING, "Too big LWM configuration.");
+ rte_errno = E2BIG;
+ return -rte_errno;
+ }
+ /* Start config LWM. */
+ pthread_mutex_lock(&priv->sh->lwm_config_lock);
+ if (rxq->lwm == 0 && lwm == 0) {
+ /* Both old/new values are 0, do nothing. */
+ ret = 0;
+ goto end;
+ }
+ wqe_cnt = mlx5_rxq_mprq_enabled(rxq_data)
+ ? RTE_BIT32(rxq_data->cqe_n - rxq_data->log_strd_num) :
+ RTE_BIT32(rxq_data->cqe_n);
+ if (lwm) {
+ if (!priv->sh->devx_channel_lwm) {
+ ret = mlx5_lwm_setup(priv);
+ if (ret) {
+ DRV_LOG(WARNING,
+ "Failed to create shared_lwm.");
+ rte_errno = ENOMEM;
+ ret = -rte_errno;
+ goto end;
+ }
+ }
+ if (!rxq->lwm_devx_subscribed) {
+ cookie = ((uint32_t)
+ (port_id << LWM_COOKIE_PORTID_OFFSET)) |
+ (rx_queue_id << LWM_COOKIE_RXQID_OFFSET);
+ ret = mlx5_os_devx_subscribe_devx_event
+ (priv->sh->devx_channel_lwm,
+ rxq->devx_rq.rq->obj,
+ sizeof(event_nums),
+ event_nums,
+ cookie);
+ if (ret) {
+ rte_errno = rte_errno ? rte_errno : EINVAL;
+ ret = -rte_errno;
+ goto end;
+ }
+ rxq->lwm_devx_subscribed = 1;
+ }
+ }
+ /* Save LWM to rxq and send modfiy_rq devx command. */
+ rxq->lwm = lwm * wqe_cnt / 100;
+ if (lwm && !rxq->lwm) {
+ /* With mprq, wqe_cnt may be < 100. */
+ DRV_LOG(WARNING, "Too small LWM configuration.");
+ rte_errno = EINVAL;
+ ret = -rte_errno;
+ goto end;
+ }
+ rxq->lwm_event_rxq_limit_reached = lwm ? cb : NULL;
+ ret = mlx5_devx_modify_rq(rxq, MLX5_RXQ_MOD_RDY2RDY);
+end:
+ pthread_mutex_unlock(&priv->sh->lwm_config_lock);
+ return ret;
+}
diff --git a/drivers/net/mlx5/mlx5_rx.h b/drivers/net/mlx5/mlx5_rx.h
index bf3c5e1..5e56258 100644
--- a/drivers/net/mlx5/mlx5_rx.h
+++ b/drivers/net/mlx5/mlx5_rx.h
@@ -175,6 +175,7 @@ struct mlx5_rxq_priv {
struct rte_eth_hairpin_conf hairpin_conf; /* Hairpin configuration. */
uint32_t hairpin_status; /* Hairpin binding status. */
uint32_t lwm:16;
+ uint32_t lwm_devx_subscribed:1;
void (*lwm_event_rxq_limit_reached)(uint16_t port_id, uint16_t rxq_id);
};
diff --git a/drivers/net/mlx5/rte_pmd_mlx5.h b/drivers/net/mlx5/rte_pmd_mlx5.h
index 6e7907e..4d2fb42 100644
--- a/drivers/net/mlx5/rte_pmd_mlx5.h
+++ b/drivers/net/mlx5/rte_pmd_mlx5.h
@@ -109,6 +109,38 @@ int rte_pmd_mlx5_external_rx_queue_id_map(uint16_t port_id, uint16_t dpdk_idx,
int rte_pmd_mlx5_external_rx_queue_id_unmap(uint16_t port_id,
uint16_t dpdk_idx);
+typedef void (*lwm_event_rxq_limit_reached_t)(uint16_t port_id,
+ uint16_t rxq_id);
+/**
+ * Arm an Rx queue LWM(limit watermark) event.
+ * While the Rx queue fullness reaches the LWM limit, the driver catches
+ * an HW event and invokes the user event callback.
+ * After the last event handling, the user needs to call this API again
+ * to arm an additional event.
+ *
+ * @param[in] port_id
+ * The port identifier of the Ethernet device.
+ * @param[in] rxq_id
+ * The rxq id.
+ * @param[in] lwm
+ * The LWM value, is defined by a percentage of the Rx queue size.
+ * [1-99] to set a new LWM (update the old value).
+ * 0 to unarm the event.
+ * @param[in] cb
+ * The LWM event callback.
+ *
+ * @return
+ * 0 : operation success.
+ * Otherwise:
+ * - ENOMEM - not enough memory to create LWM event channel.
+ * - EINVAL - the input Rxq is not created by devx.
+ * - E2BIG - lwm is bigger than 99.
+ */
+__rte_experimental
+int rte_pmd_mlx5_config_rxq_lwm(uint16_t port_id, uint16_t rxq_id,
+ uint8_t lwm,
+ lwm_event_rxq_limit_reached_t cb);
+
#ifdef __cplusplus
}
#endif
diff --git a/drivers/net/mlx5/version.map b/drivers/net/mlx5/version.map
index 79cb79a..8c965dd 100644
--- a/drivers/net/mlx5/version.map
+++ b/drivers/net/mlx5/version.map
@@ -12,4 +12,5 @@ EXPERIMENTAL {
# added in 22.03
rte_pmd_mlx5_external_rx_queue_id_map;
rte_pmd_mlx5_external_rx_queue_id_unmap;
+ rte_pmd_mlx5_config_rxq_lwm;
};
--
1.8.3.1
next prev parent reply other threads:[~2022-04-01 3:23 UTC|newest]
Thread overview: 131+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-04-01 3:22 [RFC 0/6] net/mlx5: introduce limit watermark and host shaper Spike Du
2022-04-01 3:22 ` [RFC 1/6] net/mlx5: add LWM support for Rxq Spike Du
2022-05-06 3:56 ` [RFC v1 0/7] net/mlx5: introduce limit watermark and host shaper Spike Du
2022-05-06 3:56 ` [RFC v1 1/7] net/mlx5: add LWM support for Rxq Spike Du
2022-05-06 3:56 ` [RFC v1 2/7] common/mlx5: share interrupt management Spike Du
2022-05-06 3:56 ` [RFC v1 3/7] ethdev: introduce Rx queue based limit watermark Spike Du
2022-05-19 9:37 ` Andrew Rybchenko
2022-05-06 3:56 ` [RFC v1 4/7] net/mlx5: add LWM event handling support Spike Du
2022-05-06 3:56 ` [RFC v1 5/7] net/mlx5: support Rx queue based limit watermark Spike Du
2022-05-06 3:56 ` [RFC v1 6/7] net/mlx5: add private API to config host port shaper Spike Du
2022-05-06 3:56 ` [RFC v1 7/7] app/testpmd: add LWM and Host Shaper command Spike Du
2022-05-22 5:58 ` [RFC v2 0/7] introduce per-queue limit watermark and host shaper Spike Du
2022-05-22 5:58 ` [RFC v2 1/7] net/mlx5: add LWM support for Rxq Spike Du
2022-05-22 5:58 ` [RFC v2 2/7] common/mlx5: share interrupt management Spike Du
2022-05-22 5:58 ` [RFC v2 3/7] ethdev: introduce Rx queue based limit watermark Spike Du
2022-05-22 15:23 ` Stephen Hemminger
2022-05-23 3:01 ` Spike Du
2022-05-23 21:45 ` Thomas Monjalon
2022-05-24 2:50 ` Spike Du
2022-05-24 8:18 ` Thomas Monjalon
2022-05-25 12:59 ` Andrew Rybchenko
2022-05-25 13:58 ` Thomas Monjalon
2022-05-25 14:23 ` Andrew Rybchenko
2022-05-23 22:54 ` Stephen Hemminger
2022-05-24 3:46 ` Spike Du
2022-05-22 15:24 ` Stephen Hemminger
2022-05-23 2:18 ` Spike Du
2022-05-23 6:07 ` Morten Brørup
2022-05-23 10:58 ` Thomas Monjalon
2022-05-23 14:10 ` Spike Du
2022-05-23 14:39 ` Thomas Monjalon
2022-05-24 6:35 ` Andrew Rybchenko
2022-05-24 9:40 ` Morten Brørup
2022-05-22 5:58 ` [RFC v2 4/7] net/mlx5: add LWM event handling support Spike Du
2022-05-22 5:58 ` [RFC v2 5/7] net/mlx5: support Rx queue based limit watermark Spike Du
2022-05-22 5:58 ` [RFC v2 6/7] net/mlx5: add private API to config host port shaper Spike Du
2022-05-22 5:59 ` [RFC v2 7/7] app/testpmd: add LWM and Host Shaper command Spike Du
2022-05-24 15:20 ` [PATCH v3 0/7] introduce per-queue limit watermark and host shaper Spike Du
2022-05-24 15:20 ` [PATCH v3 1/7] net/mlx5: add LWM support for Rxq Spike Du
2022-05-24 15:20 ` [PATCH v3 2/7] common/mlx5: share interrupt management Spike Du
2022-05-24 15:20 ` [PATCH v3 3/7] ethdev: introduce Rx queue based limit watermark Spike Du
2022-05-24 15:20 ` [PATCH v3 4/7] net/mlx5: add LWM event handling support Spike Du
2022-05-24 15:20 ` [PATCH v3 5/7] net/mlx5: support Rx queue based limit watermark Spike Du
2022-05-24 15:20 ` [PATCH v3 6/7] net/mlx5: add private API to config host port shaper Spike Du
2022-05-24 15:20 ` [PATCH v3 7/7] app/testpmd: add LWM and Host Shaper command Spike Du
2022-05-24 15:59 ` [PATCH v3 0/7] introduce per-queue limit watermark and host shaper Thomas Monjalon
2022-05-24 19:00 ` Morten Brørup
2022-05-24 19:22 ` Thomas Monjalon
2022-05-25 14:11 ` Andrew Rybchenko
2022-05-25 13:14 ` Spike Du
2022-05-25 13:40 ` Morten Brørup
2022-05-25 13:59 ` Spike Du
2022-05-25 14:16 ` Morten Brørup
2022-05-25 14:30 ` Andrew Rybchenko
2022-06-03 12:48 ` [PATCH v4 0/7] introduce per-queue fill threshold " Spike Du
2022-06-03 12:48 ` [PATCH v4 1/7] net/mlx5: add LWM support for Rxq Spike Du
2022-06-03 12:48 ` [PATCH v4 2/7] common/mlx5: share interrupt management Spike Du
2022-06-03 14:30 ` Ray Kinsella
2022-06-03 12:48 ` [PATCH v4 3/7] ethdev: introduce Rx queue based fill threshold Spike Du
2022-06-03 14:30 ` Ray Kinsella
2022-06-04 12:46 ` Andrew Rybchenko
2022-06-06 13:16 ` Spike Du
2022-06-06 17:15 ` Andrew Rybchenko
2022-06-06 21:30 ` Thomas Monjalon
2022-06-07 8:02 ` Andrew Rybchenko
2022-06-07 6:00 ` Spike Du
2022-06-06 15:49 ` Stephen Hemminger
2022-06-03 12:48 ` [PATCH v4 4/7] net/mlx5: add LWM event handling support Spike Du
2022-06-03 12:48 ` [PATCH v4 5/7] net/mlx5: support Rx queue based fill threshold Spike Du
2022-06-03 12:48 ` [PATCH v4 6/7] net/mlx5: add private API to config host port shaper Spike Du
2022-06-03 14:55 ` Ray Kinsella
2022-06-03 12:48 ` [PATCH v4 7/7] app/testpmd: add Host Shaper command Spike Du
2022-06-07 12:59 ` [PATCH v5 0/7] introduce per-queue available descriptor threshold and host shaper Spike Du
2022-06-07 12:59 ` [PATCH v5 1/7] net/mlx5: add LWM support for Rxq Spike Du
2022-06-08 20:10 ` Matan Azrad
2022-06-07 12:59 ` [PATCH v5 2/7] common/mlx5: share interrupt management Spike Du
2022-06-07 12:59 ` [PATCH v5 3/7] ethdev: introduce Rx queue based available descriptor threshold Spike Du
2022-06-07 12:59 ` [PATCH v5 4/7] net/mlx5: add LWM event handling support Spike Du
2022-06-07 12:59 ` [PATCH v5 5/7] net/mlx5: support Rx queue based available descriptor threshold Spike Du
2022-06-07 12:59 ` [PATCH v5 6/7] net/mlx5: add private API to config host port shaper Spike Du
2022-06-07 12:59 ` [PATCH v5 7/7] app/testpmd: add Host Shaper command Spike Du
2022-06-09 7:55 ` Andrew Rybchenko
2022-06-10 2:22 ` Spike Du
2022-06-13 2:50 ` [PATCH v6] " Spike Du
2022-06-13 2:50 ` Spike Du
2022-06-14 9:43 ` Singh, Aman Deep
2022-06-14 9:54 ` Spike Du
2022-06-14 12:01 ` [PATCH v7] " Spike Du
2022-06-14 12:01 ` Spike Du
2022-06-15 7:51 ` Matan Azrad
2022-06-15 11:08 ` Thomas Monjalon
2022-06-15 12:58 ` [PATCH v8 0/6] introduce per-queue available descriptor threshold and host shaper Spike Du
2022-06-15 12:58 ` [PATCH v8 1/6] net/mlx5: add LWM support for Rxq Spike Du
2022-06-15 14:43 ` [PATCH v9 0/6] introduce per-queue available descriptor threshold and host shaper Spike Du
2022-06-15 14:43 ` [PATCH v9 1/6] net/mlx5: add LWM support for Rxq Spike Du
2022-06-16 8:41 ` [PATCH v10 0/6] introduce per-queue available descriptor threshold and host shaper Spike Du
2022-06-16 8:41 ` [PATCH v10 1/6] net/mlx5: add LWM support for Rxq Spike Du
2022-06-16 8:41 ` [PATCH v10 2/6] common/mlx5: share interrupt management Spike Du
2022-06-23 16:05 ` Ray Kinsella
2022-06-16 8:41 ` [PATCH v10 3/6] net/mlx5: add LWM event handling support Spike Du
2022-06-16 8:41 ` [PATCH v10 4/6] net/mlx5: support Rx queue based available descriptor threshold Spike Du
2022-06-16 8:41 ` [PATCH v10 5/6] net/mlx5: add private API to config host port shaper Spike Du
2022-06-16 8:41 ` [PATCH v10 6/6] app/testpmd: add Host Shaper command Spike Du
2022-06-19 8:14 ` [PATCH v10 0/6] introduce per-queue available descriptor threshold and host shaper Raslan Darawsheh
2022-06-15 14:43 ` [PATCH v9 2/6] common/mlx5: share interrupt management Spike Du
2022-06-15 14:43 ` [PATCH v9 3/6] net/mlx5: add LWM event handling support Spike Du
2022-06-15 14:43 ` [PATCH v9 4/6] net/mlx5: support Rx queue based available descriptor threshold Spike Du
2022-06-15 14:43 ` [PATCH v9 5/6] net/mlx5: add private API to config host port shaper Spike Du
2022-06-15 14:43 ` [PATCH v9 6/6] app/testpmd: add Host Shaper command Spike Du
2022-06-15 12:58 ` [PATCH v8 2/6] common/mlx5: share interrupt management Spike Du
2022-06-15 12:58 ` [PATCH v8 3/6] net/mlx5: add LWM event handling support Spike Du
2022-06-15 12:58 ` [PATCH v8 4/6] net/mlx5: support Rx queue based available descriptor threshold Spike Du
2022-06-15 12:58 ` [PATCH v8 5/6] net/mlx5: add private API to config host port shaper Spike Du
2022-06-15 12:58 ` [PATCH v8 6/6] app/testpmd: add Host Shaper command Spike Du
2022-06-08 9:43 ` [PATCH v5 0/7] introduce per-queue available descriptor threshold and host shaper Andrew Rybchenko
2022-06-08 16:35 ` [PATCH v6] ethdev: introduce available Rx descriptors threshold Andrew Rybchenko
2022-06-08 17:22 ` Thomas Monjalon
2022-06-08 17:46 ` Thomas Monjalon
2022-06-09 0:17 ` fengchengwen
2022-06-09 7:05 ` Thomas Monjalon
2022-06-10 0:01 ` fengchengwen
2022-04-01 3:22 ` [RFC 2/6] common/mlx5: share interrupt management Spike Du
2022-04-01 3:22 ` [RFC 3/6] net/mlx5: add LWM event handling support Spike Du
2022-04-01 3:22 ` Spike Du [this message]
2022-04-01 3:22 ` [RFC 5/6] net/mlx5: add private API to config host port shaper Spike Du
2022-04-01 3:22 ` [RFC 6/6] app/testpmd: add LWM and Host Shaper command Spike Du
2022-04-05 8:58 ` [RFC 0/6] net/mlx5: introduce limit watermark and host shaper Jerin Jacob
2022-04-26 2:42 ` Spike Du
2022-05-01 12:50 ` Jerin Jacob
2022-05-02 3:58 ` Spike Du
2022-04-29 5:48 ` Spike Du
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220401032232.1267376-5-spiked@nvidia.com \
--to=spiked@nvidia.com \
--cc=dev@dpdk.org \
--cc=matan@nvidia.com \
--cc=orika@nvidia.com \
--cc=rasland@nvidia.com \
--cc=thomas@monjalon.net \
--cc=viacheslavo@nvidia.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).