From: <pbhagavatula@marvell.com>
To: <jerinj@marvell.com>, Pavan Nikhilesh <pbhagavatula@marvell.com>,
"Shijith Thotton" <sthotton@marvell.com>
Cc: <dev@dpdk.org>
Subject: [PATCH v2 07/21] event/cnxk: add CN20K SSO dequeue fast path
Date: Tue, 22 Oct 2024 02:27:30 +0530 [thread overview]
Message-ID: <20241021205745.7310-7-pbhagavatula@marvell.com> (raw)
In-Reply-To: <20241021205745.7310-1-pbhagavatula@marvell.com>
From: Pavan Nikhilesh <pbhagavatula@marvell.com>
Add CN20K SSO GWS event dequeue fastpath functions.
Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com>
---
drivers/event/cnxk/cn20k_eventdev.c | 5 +
drivers/event/cnxk/cn20k_worker.c | 54 +++++++++++
drivers/event/cnxk/cn20k_worker.h | 137 +++++++++++++++++++++++++++-
3 files changed, 195 insertions(+), 1 deletion(-)
diff --git a/drivers/event/cnxk/cn20k_eventdev.c b/drivers/event/cnxk/cn20k_eventdev.c
index d2c0fcceee..6c810ed77d 100644
--- a/drivers/event/cnxk/cn20k_eventdev.c
+++ b/drivers/event/cnxk/cn20k_eventdev.c
@@ -112,11 +112,16 @@ static void
cn20k_sso_fp_fns_set(struct rte_eventdev *event_dev)
{
#if defined(RTE_ARCH_ARM64)
+ struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
event_dev->enqueue_burst = cn20k_sso_hws_enq_burst;
event_dev->enqueue_new_burst = cn20k_sso_hws_enq_new_burst;
event_dev->enqueue_forward_burst = cn20k_sso_hws_enq_fwd_burst;
+ event_dev->dequeue_burst = cn20k_sso_hws_deq_burst;
+ if (dev->deq_tmo_ns)
+ event_dev->dequeue_burst = cn20k_sso_hws_tmo_deq_burst;
+
#else
RTE_SET_USED(event_dev);
#endif
diff --git a/drivers/event/cnxk/cn20k_worker.c b/drivers/event/cnxk/cn20k_worker.c
index 19db29e7a9..075461c3ff 100644
--- a/drivers/event/cnxk/cn20k_worker.c
+++ b/drivers/event/cnxk/cn20k_worker.c
@@ -378,3 +378,57 @@ cn20k_sso_hws_enq_fwd_burst(void *port, const struct rte_event ev[], uint16_t nb
return 1;
}
+
+uint16_t __rte_hot
+cn20k_sso_hws_deq(void *port, struct rte_event *ev, uint64_t timeout_ticks)
+{
+ struct cn20k_sso_hws *ws = port;
+
+ RTE_SET_USED(timeout_ticks);
+
+ if (ws->swtag_req) {
+ ws->swtag_req = 0;
+ cnxk_sso_hws_swtag_wait(ws->base + SSOW_LF_GWS_WQE0);
+ return 1;
+ }
+
+ return cn20k_sso_hws_get_work(ws, ev, 0);
+}
+
+uint16_t __rte_hot
+cn20k_sso_hws_deq_burst(void *port, struct rte_event ev[], uint16_t nb_events,
+ uint64_t timeout_ticks)
+{
+ RTE_SET_USED(nb_events);
+
+ return cn20k_sso_hws_deq(port, ev, timeout_ticks);
+}
+
+uint16_t __rte_hot
+cn20k_sso_hws_tmo_deq(void *port, struct rte_event *ev, uint64_t timeout_ticks)
+{
+ struct cn20k_sso_hws *ws = port;
+ uint16_t ret = 1;
+ uint64_t iter;
+
+ if (ws->swtag_req) {
+ ws->swtag_req = 0;
+ cnxk_sso_hws_swtag_wait(ws->base + SSOW_LF_GWS_WQE0);
+ return ret;
+ }
+
+ ret = cn20k_sso_hws_get_work(ws, ev, 0);
+ for (iter = 1; iter < timeout_ticks && (ret == 0); iter++)
+ ret = cn20k_sso_hws_get_work(ws, ev, 0);
+
+ return ret;
+}
+
+uint16_t __rte_hot
+cn20k_sso_hws_tmo_deq_burst(void *port, struct rte_event ev[], uint16_t nb_events,
+ uint64_t timeout_ticks)
+{
+ RTE_SET_USED(nb_events);
+
+ return cn20k_sso_hws_tmo_deq(port, ev, timeout_ticks);
+}
diff --git a/drivers/event/cnxk/cn20k_worker.h b/drivers/event/cnxk/cn20k_worker.h
index 5ff8f11b38..8dc60a06ec 100644
--- a/drivers/event/cnxk/cn20k_worker.h
+++ b/drivers/event/cnxk/cn20k_worker.h
@@ -7,8 +7,136 @@
#include <rte_eventdev.h>
-#include "cnxk_worker.h"
#include "cn20k_eventdev.h"
+#include "cnxk_worker.h"
+
+static __rte_always_inline void
+cn20k_sso_hws_post_process(struct cn20k_sso_hws *ws, uint64_t *u64, const uint32_t flags)
+{
+ RTE_SET_USED(ws);
+ RTE_SET_USED(flags);
+
+ u64[0] = (u64[0] & (0x3ull << 32)) << 6 | (u64[0] & (0x3FFull << 36)) << 4 |
+ (u64[0] & 0xffffffff);
+}
+
+static __rte_always_inline uint16_t
+cn20k_sso_hws_get_work(struct cn20k_sso_hws *ws, struct rte_event *ev, const uint32_t flags)
+{
+ union {
+ __uint128_t get_work;
+ uint64_t u64[2];
+ } gw;
+
+ gw.get_work = ws->gw_wdata;
+#if defined(RTE_ARCH_ARM64)
+#if defined(__clang__)
+ register uint64_t x0 __asm("x0") = (uint64_t)gw.u64[0];
+ register uint64_t x1 __asm("x1") = (uint64_t)gw.u64[1];
+#if defined(RTE_ARM_USE_WFE)
+ plt_write64(gw.u64[0], ws->base + SSOW_LF_GWS_OP_GET_WORK0);
+ asm volatile(PLT_CPU_FEATURE_PREAMBLE
+ " ldp %[x0], %[x1], [%[tag_loc]] \n"
+ " tbz %[x0], %[pend_gw], done%= \n"
+ " sevl \n"
+ "rty%=: wfe \n"
+ " ldp %[x0], %[x1], [%[tag_loc]] \n"
+ " tbnz %[x0], %[pend_gw], rty%= \n"
+ "done%=: \n"
+ " dmb ld \n"
+ : [x0] "+r" (x0), [x1] "+r" (x1)
+ : [tag_loc] "r"(ws->base + SSOW_LF_GWS_WQE0),
+ [pend_gw] "i"(SSOW_LF_GWS_TAG_PEND_GET_WORK_BIT)
+ : "memory");
+#else
+ asm volatile(".arch armv8-a+lse\n"
+ "caspal %[x0], %[x1], %[x0], %[x1], [%[dst]]\n"
+ : [x0] "+r" (x0), [x1] "+r" (x1)
+ : [dst] "r"(ws->base + SSOW_LF_GWS_OP_GET_WORK0)
+ : "memory");
+#endif
+ gw.u64[0] = x0;
+ gw.u64[1] = x1;
+#else
+#if defined(RTE_ARM_USE_WFE)
+ plt_write64(gw.u64[0], ws->base + SSOW_LF_GWS_OP_GET_WORK0);
+ asm volatile(PLT_CPU_FEATURE_PREAMBLE
+ " ldp %[wdata], %H[wdata], [%[tag_loc]] \n"
+ " tbz %[wdata], %[pend_gw], done%= \n"
+ " sevl \n"
+ "rty%=: wfe \n"
+ " ldp %[wdata], %H[wdata], [%[tag_loc]] \n"
+ " tbnz %[wdata], %[pend_gw], rty%= \n"
+ "done%=: \n"
+ " dmb ld \n"
+ : [wdata] "=&r"(gw.get_work)
+ : [tag_loc] "r"(ws->base + SSOW_LF_GWS_WQE0),
+ [pend_gw] "i"(SSOW_LF_GWS_TAG_PEND_GET_WORK_BIT)
+ : "memory");
+#else
+ asm volatile(PLT_CPU_FEATURE_PREAMBLE
+ "caspal %[wdata], %H[wdata], %[wdata], %H[wdata], [%[gw_loc]]\n"
+ : [wdata] "+r"(gw.get_work)
+ : [gw_loc] "r"(ws->base + SSOW_LF_GWS_OP_GET_WORK0)
+ : "memory");
+#endif
+#endif
+#else
+ plt_write64(gw.u64[0], ws->base + SSOW_LF_GWS_OP_GET_WORK0);
+ do {
+ roc_load_pair(gw.u64[0], gw.u64[1], ws->base + SSOW_LF_GWS_WQE0);
+ } while (gw.u64[0] & BIT_ULL(63));
+ rte_atomic_thread_fence(rte_memory_order_seq_cst);
+#endif
+ ws->gw_rdata = gw.u64[0];
+ if (gw.u64[1])
+ cn20k_sso_hws_post_process(ws, gw.u64, flags);
+
+ ev->event = gw.u64[0];
+ ev->u64 = gw.u64[1];
+
+ return !!gw.u64[1];
+}
+
+/* Used in cleaning up workslot. */
+static __rte_always_inline uint16_t
+cn20k_sso_hws_get_work_empty(struct cn20k_sso_hws *ws, struct rte_event *ev, const uint32_t flags)
+{
+ union {
+ __uint128_t get_work;
+ uint64_t u64[2];
+ } gw;
+
+#ifdef RTE_ARCH_ARM64
+ asm volatile(PLT_CPU_FEATURE_PREAMBLE
+ " ldp %[tag], %[wqp], [%[tag_loc]] \n"
+ " tbz %[tag], 63, .Ldone%= \n"
+ " sevl \n"
+ ".Lrty%=: wfe \n"
+ " ldp %[tag], %[wqp], [%[tag_loc]] \n"
+ " tbnz %[tag], 63, .Lrty%= \n"
+ ".Ldone%=: dmb ld \n"
+ : [tag] "=&r"(gw.u64[0]), [wqp] "=&r"(gw.u64[1])
+ : [tag_loc] "r"(ws->base + SSOW_LF_GWS_WQE0)
+ : "memory");
+#else
+ do {
+ roc_load_pair(gw.u64[0], gw.u64[1], ws->base + SSOW_LF_GWS_WQE0);
+ } while (gw.u64[0] & BIT_ULL(63));
+#endif
+
+ ws->gw_rdata = gw.u64[0];
+ if (gw.u64[1])
+ cn20k_sso_hws_post_process(ws, gw.u64, flags);
+ else
+ gw.u64[0] = (gw.u64[0] & (0x3ull << 32)) << 6 |
+ (gw.u64[0] & (0x3FFull << 36)) << 4 | (gw.u64[0] & 0xffffffff);
+
+ ev->event = gw.u64[0];
+ ev->u64 = gw.u64[1];
+
+ return !!gw.u64[1];
+}
/* CN20K Fastpath functions. */
uint16_t __rte_hot cn20k_sso_hws_enq_burst(void *port, const struct rte_event ev[],
@@ -18,4 +146,11 @@ uint16_t __rte_hot cn20k_sso_hws_enq_new_burst(void *port, const struct rte_even
uint16_t __rte_hot cn20k_sso_hws_enq_fwd_burst(void *port, const struct rte_event ev[],
uint16_t nb_events);
+uint16_t __rte_hot cn20k_sso_hws_deq(void *port, struct rte_event *ev, uint64_t timeout_ticks);
+uint16_t __rte_hot cn20k_sso_hws_deq_burst(void *port, struct rte_event ev[], uint16_t nb_events,
+ uint64_t timeout_ticks);
+uint16_t __rte_hot cn20k_sso_hws_tmo_deq(void *port, struct rte_event *ev, uint64_t timeout_ticks);
+uint16_t __rte_hot cn20k_sso_hws_tmo_deq_burst(void *port, struct rte_event ev[],
+ uint16_t nb_events, uint64_t timeout_ticks);
+
#endif
--
2.25.1
next prev parent reply other threads:[~2024-10-21 20:58 UTC|newest]
Thread overview: 91+ messages / expand[flat|nested] mbox.gz Atom feed top
2024-10-03 13:22 [PATCH 01/20] common/cnxk: implement SSO HW info pbhagavatula
2024-10-03 13:22 ` [PATCH 02/20] event/cnxk: add CN20K specific device probe pbhagavatula
2024-10-15 16:17 ` Stephen Hemminger
2024-10-03 13:22 ` [PATCH 03/20] event/cnxk: add CN20K device config pbhagavatula
2024-10-03 13:22 ` [PATCH 04/20] event/cnxk: add CN20k event queue config pbhagavatula
2024-10-03 13:22 ` [PATCH 05/20] event/cnxk: add CN20K event port configuration pbhagavatula
2024-10-03 13:22 ` [PATCH 06/20] event/cnxk: add CN20K SSO enqueue fast path pbhagavatula
2024-10-22 1:46 ` Stephen Hemminger
2024-10-03 13:22 ` [PATCH 07/20] event/cnxk: add CN20K SSO dequeue " pbhagavatula
2024-10-22 1:49 ` Stephen Hemminger
2024-10-22 8:54 ` [EXTERNAL] " Pavan Nikhilesh Bhagavatula
2024-10-03 13:22 ` [PATCH 08/20] event/cnxk: add CN20K event port quiesce pbhagavatula
2024-10-03 13:22 ` [PATCH 09/20] event/cnxk: add CN20K event port profile switch pbhagavatula
2024-10-03 13:22 ` [PATCH 10/20] event/cnxk: add CN20K device start pbhagavatula
2024-10-03 13:22 ` [PATCH 11/20] event/cnxk: add CN20K device stop and close pbhagavatula
2024-10-03 13:22 ` [PATCH 12/20] event/cnxk: add CN20K xstats, selftest and dump pbhagavatula
2024-10-03 13:22 ` [PATCH 13/20] event/cnxk: support CN20K Rx adapter pbhagavatula
2024-10-03 13:22 ` [PATCH 14/20] event/cnxk: support CN20K Rx adapter fast path pbhagavatula
2024-10-03 13:22 ` [PATCH 15/20] event/cnxk: support CN20K Tx adapter pbhagavatula
2024-10-03 13:22 ` [PATCH 16/20] event/cnxk: support CN20K Tx adapter fast path pbhagavatula
2024-10-03 13:22 ` [PATCH 17/20] common/cnxk: add SSO event aggregator pbhagavatula
2024-10-03 13:22 ` [PATCH 18/20] event/cnxk: add Rx/Tx event vector support pbhagavatula
2024-10-03 13:22 ` [PATCH 19/20] common/cnxk: update timer base code pbhagavatula
2024-10-03 13:22 ` [PATCH 20/20] event/cnxk: add CN20K timer adapter pbhagavatula
2024-10-21 20:57 ` [PATCH v2 01/21] common/cnxk: implement SSO HW info pbhagavatula
2024-10-21 20:57 ` [PATCH v2 02/21] event/cnxk: add CN20K specific device probe pbhagavatula
2024-10-21 20:57 ` [PATCH v2 03/21] event/cnxk: add CN20K device config pbhagavatula
2024-10-21 20:57 ` [PATCH v2 04/21] event/cnxk: add CN20k event queue configuration pbhagavatula
2024-10-21 20:57 ` [PATCH v2 05/21] event/cnxk: add CN20K event port configuration pbhagavatula
2024-10-21 20:57 ` [PATCH v2 06/21] event/cnxk: add CN20K SSO enqueue fast path pbhagavatula
2024-10-21 20:57 ` pbhagavatula [this message]
2024-10-21 20:57 ` [PATCH v2 08/21] event/cnxk: add CN20K event port quiesce pbhagavatula
2024-10-21 20:57 ` [PATCH v2 09/21] event/cnxk: add CN20K event port profile switch pbhagavatula
2024-10-21 20:57 ` [PATCH v2 10/21] event/cnxk: add CN20K event port preschedule pbhagavatula
2024-10-21 20:57 ` [PATCH v2 11/21] event/cnxk: add CN20K device start pbhagavatula
2024-10-21 20:57 ` [PATCH v2 12/21] event/cnxk: add CN20K device stop and close pbhagavatula
2024-10-21 20:57 ` [PATCH v2 13/21] event/cnxk: add CN20K xstats, selftest and dump pbhagavatula
2024-10-21 20:57 ` [PATCH v2 14/21] event/cnxk: support CN20K Rx adapter pbhagavatula
2024-10-21 20:57 ` [PATCH v2 15/21] event/cnxk: support CN20K Rx adapter fast path pbhagavatula
2024-10-21 20:57 ` [PATCH v2 16/21] event/cnxk: support CN20K Tx adapter pbhagavatula
2024-10-21 20:57 ` [PATCH v2 17/21] event/cnxk: support CN20K Tx adapter fast path pbhagavatula
2024-10-21 20:57 ` [PATCH v2 18/21] common/cnxk: add SSO event aggregator pbhagavatula
2024-10-21 20:57 ` [PATCH v2 19/21] event/cnxk: add Rx/Tx event vector support pbhagavatula
2024-10-21 20:57 ` [PATCH v2 20/21] common/cnxk: update timer base code pbhagavatula
2024-10-21 20:57 ` [PATCH v2 21/21] event/cnxk: add CN20K timer adapter pbhagavatula
2024-10-22 8:46 ` [PATCH v3 01/22] event/cnxk: use stdatomic API pbhagavatula
2024-10-22 8:46 ` [PATCH v3 02/22] common/cnxk: implement SSO HW info pbhagavatula
2024-10-22 8:46 ` [PATCH v3 03/22] event/cnxk: add CN20K specific device probe pbhagavatula
2024-10-22 8:46 ` [PATCH v3 04/22] event/cnxk: add CN20K device config pbhagavatula
2024-10-22 8:46 ` [PATCH v3 05/22] event/cnxk: add CN20k event queue configuration pbhagavatula
2024-10-22 8:46 ` [PATCH v3 06/22] event/cnxk: add CN20K event port configuration pbhagavatula
2024-10-22 8:46 ` [PATCH v3 07/22] event/cnxk: add CN20K SSO enqueue fast path pbhagavatula
2024-10-22 8:46 ` [PATCH v3 08/22] event/cnxk: add CN20K SSO dequeue " pbhagavatula
2024-10-22 8:46 ` [PATCH v3 09/22] event/cnxk: add CN20K event port quiesce pbhagavatula
2024-10-22 8:46 ` [PATCH v3 10/22] event/cnxk: add CN20K event port profile switch pbhagavatula
2024-10-22 8:46 ` [PATCH v3 11/22] event/cnxk: add CN20K event port preschedule pbhagavatula
2024-10-22 8:46 ` [PATCH v3 12/22] event/cnxk: add CN20K device start pbhagavatula
2024-10-22 8:46 ` [PATCH v3 13/22] event/cnxk: add CN20K device stop and close pbhagavatula
2024-10-22 8:46 ` [PATCH v3 14/22] event/cnxk: add CN20K xstats, selftest and dump pbhagavatula
2024-10-22 8:46 ` [PATCH v3 15/22] event/cnxk: support CN20K Rx adapter pbhagavatula
2024-10-22 8:46 ` [PATCH v3 16/22] event/cnxk: support CN20K Rx adapter fast path pbhagavatula
2024-10-22 8:46 ` [PATCH v3 17/22] event/cnxk: support CN20K Tx adapter pbhagavatula
2024-10-22 8:46 ` [PATCH v3 18/22] event/cnxk: support CN20K Tx adapter fast path pbhagavatula
2024-10-22 8:46 ` [PATCH v3 19/22] common/cnxk: add SSO event aggregator pbhagavatula
2024-10-22 8:46 ` [PATCH v3 20/22] event/cnxk: add Rx/Tx event vector support pbhagavatula
2024-10-22 8:46 ` [PATCH v3 21/22] common/cnxk: update timer base code pbhagavatula
2024-10-22 8:46 ` [PATCH v3 22/22] event/cnxk: add CN20K timer adapter pbhagavatula
2024-10-22 19:34 ` [PATCH v4 01/22] event/cnxk: use stdatomic API pbhagavatula
2024-10-22 19:34 ` [PATCH v4 02/22] common/cnxk: implement SSO HW info pbhagavatula
2024-10-22 19:34 ` [PATCH v4 03/22] event/cnxk: add CN20K specific device probe pbhagavatula
2024-10-22 19:34 ` [PATCH v4 04/22] event/cnxk: add CN20K device config pbhagavatula
2024-10-22 19:34 ` [PATCH v4 05/22] event/cnxk: add CN20k event queue configuration pbhagavatula
2024-10-22 19:34 ` [PATCH v4 06/22] event/cnxk: add CN20K event port configuration pbhagavatula
2024-10-22 19:34 ` [PATCH v4 07/22] event/cnxk: add CN20K SSO enqueue fast path pbhagavatula
2024-10-22 19:34 ` [PATCH v4 08/22] event/cnxk: add CN20K SSO dequeue " pbhagavatula
2024-10-22 19:34 ` [PATCH v4 09/22] event/cnxk: add CN20K event port quiesce pbhagavatula
2024-10-22 19:34 ` [PATCH v4 10/22] event/cnxk: add CN20K event port profile switch pbhagavatula
2024-10-22 19:34 ` [PATCH v4 11/22] event/cnxk: add CN20K event port preschedule pbhagavatula
2024-10-22 19:34 ` [PATCH v4 12/22] event/cnxk: add CN20K device start pbhagavatula
2024-10-22 19:34 ` [PATCH v4 13/22] event/cnxk: add CN20K device stop and close pbhagavatula
2024-10-22 19:34 ` [PATCH v4 14/22] event/cnxk: add CN20K xstats, selftest and dump pbhagavatula
2024-10-22 19:34 ` [PATCH v4 15/22] event/cnxk: support CN20K Rx adapter pbhagavatula
2024-10-22 19:34 ` [PATCH v4 16/22] event/cnxk: support CN20K Rx adapter fast path pbhagavatula
2024-10-22 19:35 ` [PATCH v4 17/22] event/cnxk: support CN20K Tx adapter pbhagavatula
2024-10-22 19:35 ` [PATCH v4 18/22] event/cnxk: support CN20K Tx adapter fast path pbhagavatula
2024-10-22 19:35 ` [PATCH v4 19/22] common/cnxk: add SSO event aggregator pbhagavatula
2024-10-22 19:35 ` [PATCH v4 20/22] event/cnxk: add Rx/Tx event vector support pbhagavatula
2024-10-22 19:35 ` [PATCH v4 21/22] common/cnxk: update timer base code pbhagavatula
2024-10-22 19:35 ` [PATCH v4 22/22] event/cnxk: add CN20K timer adapter pbhagavatula
2024-10-22 1:52 ` [PATCH 01/20] common/cnxk: implement SSO HW info Stephen Hemminger
2024-10-22 8:53 ` [EXTERNAL] " Pavan Nikhilesh Bhagavatula
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20241021205745.7310-7-pbhagavatula@marvell.com \
--to=pbhagavatula@marvell.com \
--cc=dev@dpdk.org \
--cc=jerinj@marvell.com \
--cc=sthotton@marvell.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).