[dpdk-dev] [PATCH 05/10] event/octeontx: add support for arm and cancel

[Pavan Nikhilesh <pbhagavatula@caviumnetworks.com>]

Signed-off-by: Pavan Nikhilesh <pbhagavatula@caviumnetworks.com>
---
 drivers/event/octeontx/Makefile       |   5 +
 drivers/event/octeontx/meson.build    |   1 +
 drivers/event/octeontx/timvf_evdev.c  |  11 +-
 drivers/event/octeontx/timvf_evdev.h  |  31 ++++
 drivers/event/octeontx/timvf_worker.c |  53 +++++++
 drivers/event/octeontx/timvf_worker.h | 278 ++++++++++++++++++++++++++++++++++
 6 files changed, 378 insertions(+), 1 deletion(-)
 create mode 100644 drivers/event/octeontx/timvf_worker.c
 create mode 100644 drivers/event/octeontx/timvf_worker.h

diff --git a/drivers/event/octeontx/Makefile b/drivers/event/octeontx/Makefile
index f1d10a99e..ba1ccfb04 100644
--- a/drivers/event/octeontx/Makefile
+++ b/drivers/event/octeontx/Makefile
@@ -27,19 +27,24 @@ LIBABIVER := 1
 SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_SSOVF) += ssovf_worker.c
 SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_SSOVF) += ssovf_evdev.c
 SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_SSOVF) += ssovf_evdev_selftest.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_SSOVF) += timvf_worker.c
 SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_SSOVF) += timvf_evdev.c
 
 ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y)
 CFLAGS_ssovf_worker.o += -fno-prefetch-loop-arrays
+CFLAGS_timvf_worker.o += -fno-prefetch-loop-arrays
 
 ifeq ($(shell test $(GCC_VERSION) -ge 46 && echo 1), 1)
 CFLAGS_ssovf_worker.o += -Ofast
+CFLAGS_timvf_worker.o += -Ofast
 else
 CFLAGS_ssovf_worker.o += -O3 -ffast-math
+CFLAGS_timvf_worker.o += -O3 -ffast-math
 endif
 
 else
 CFLAGS_ssovf_worker.o += -Ofast
+CFLAGS_timvf_worker.o += -Ofast
 endif
 
 include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/event/octeontx/meson.build b/drivers/event/octeontx/meson.build
index 8941f8a56..efac94cba 100644
--- a/drivers/event/octeontx/meson.build
+++ b/drivers/event/octeontx/meson.build
@@ -4,6 +4,7 @@
 sources = files('ssovf_worker.c',
 		'ssovf_evdev.c',
 		'ssovf_evdev_selftest.c',
+		'timvf_worker.c',
 		'timvf_evdev.c',
 )
 
diff --git a/drivers/event/octeontx/timvf_evdev.c b/drivers/event/octeontx/timvf_evdev.c
index dccdf1ddb..d0ba42263 100644
--- a/drivers/event/octeontx/timvf_evdev.c
+++ b/drivers/event/octeontx/timvf_evdev.c
@@ -261,6 +261,10 @@ timvf_ring_create(struct rte_event_timer_adapter *adptr)
 	timvf_write64(0x7, (uint8_t *)timr->vbar0 + TIM_VF_NRSPERR_ENA_W1C);
 	timvf_write64(0x7, (uint8_t *)timr->vbar0 + TIM_VF_NRSPERR_ENA_W1S);
 
+	adptr->arm_burst = timvf_timer_reg_burst_mp;
+	adptr->arm_tmo_tick_burst = NULL;
+	adptr->cancel_burst = timvf_timer_unreg_burst;
+
 	return 0;
 mem_err:
 	rte_free(timr);
@@ -294,7 +298,12 @@ timvf_timer_adapter_caps_get(const struct rte_eventdev *dev, uint64_t flags,
 {
 	RTE_SET_USED(dev);
 	RTE_SET_USED(flags);
+
+	timvf_ops.arm_burst = timvf_timer_reg_burst_mp;
+	timvf_ops.arm_tmo_tick_burst = NULL;
+
+	timvf_ops.cancel_burst = timvf_timer_unreg_burst;
 	*caps = RTE_EVENT_TIMER_ADAPTER_CAP_INTERNAL_PORT;
 	*ops = &timvf_ops;
-	return -EINVAL;
+	return 0;
 }
diff --git a/drivers/event/octeontx/timvf_evdev.h b/drivers/event/octeontx/timvf_evdev.h
index a6a64587f..c80e147e8 100644
--- a/drivers/event/octeontx/timvf_evdev.h
+++ b/drivers/event/octeontx/timvf_evdev.h
@@ -74,6 +74,33 @@
 #define TIM_VRING_AURA				(0x108)
 #define TIM_VRING_REL				(0x110)
 
+#define TIM_CTL1_W0_S_BUCKET			20
+#define TIM_CTL1_W0_M_BUCKET			((1ull << (40 - 20)) - 1)
+
+#define TIM_BUCKET_W1_S_NUM_ENTRIES		(0) /*Shift*/
+#define TIM_BUCKET_W1_M_NUM_ENTRIES		((1ull << (32 - 0)) - 1)
+#define TIM_BUCKET_W1_S_SBT			(32)
+#define TIM_BUCKET_W1_M_SBT			((1ull << (33 - 32)) - 1)
+#define TIM_BUCKET_W1_S_HBT			(33)
+#define TIM_BUCKET_W1_M_HBT			((1ull << (34 - 33)) - 1)
+#define TIM_BUCKET_W1_S_BSK			(34)
+#define TIM_BUCKET_W1_M_BSK			((1ull << (35 - 34)) - 1)
+#define TIM_BUCKET_W1_S_LOCK			(40)
+#define TIM_BUCKET_W1_M_LOCK			((1ull << (48 - 40)) - 1)
+#define TIM_BUCKET_W1_S_CHUNK_REMAINDER		(48)
+#define TIM_BUCKET_W1_M_CHUNK_REMAINDER		((1ull << (64 - 48)) - 1)
+
+#define TIM_BUCKET_SEMA	\
+	(TIM_BUCKET_CHUNK_REMAIN)
+
+#define TIM_BUCKET_CHUNK_REMAIN \
+	(TIM_BUCKET_W1_M_CHUNK_REMAINDER << TIM_BUCKET_W1_S_CHUNK_REMAINDER)
+
+#define TIM_BUCKET_LOCK \
+	(TIM_BUCKET_W1_M_LOCK << TIM_BUCKET_W1_S_LOCK)
+
+#define TIM_BUCKET_SEMA_WLOCK \
+	(TIM_BUCKET_CHUNK_REMAIN | (1ull << TIM_BUCKET_W1_S_LOCK))
 
 #define NSEC_PER_SEC 1E9
 #define NSEC2CLK(__ns, __freq) (((__ns) * (__freq)) / NSEC_PER_SEC)
@@ -159,5 +186,9 @@ bkt_mod(uint32_t rel_bkt, uint32_t nb_bkts)
 
 int timvf_timer_adapter_caps_get(const struct rte_eventdev *dev, uint64_t flags,
 		uint32_t *caps, const struct rte_event_timer_adapter_ops **ops);
+int timvf_timer_unreg_burst(const struct rte_event_timer_adapter *adptr,
+		struct rte_event_timer **tim, const uint16_t nb_timers);
+int timvf_timer_reg_burst_mp(const struct rte_event_timer_adapter *adptr,
+		struct rte_event_timer **tim, const uint16_t nb_timers);
 
 #endif /* __TIMVF_EVDEV_H__ */
diff --git a/drivers/event/octeontx/timvf_worker.c b/drivers/event/octeontx/timvf_worker.c
new file mode 100644
index 000000000..7a924fd11
--- /dev/null
+++ b/drivers/event/octeontx/timvf_worker.c
@@ -0,0 +1,53 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Cavium, Inc
+ */
+
+#include "timvf_worker.h"
+
+int
+timvf_timer_unreg_burst(const struct rte_event_timer_adapter *adptr,
+		struct rte_event_timer **tim, const uint16_t nb_timers)
+{
+	RTE_SET_USED(adptr);
+	int ret;
+	uint16_t index;
+
+	for (index = 0; index < nb_timers; index++) {
+		ret = timvf_rem_entry(tim[index]);
+		if (ret) {
+			rte_errno = -ret;
+			break;
+		}
+	}
+	return index;
+}
+
+int
+timvf_timer_reg_burst_mp(const struct rte_event_timer_adapter *adptr,
+		struct rte_event_timer **tim, const uint16_t nb_timers)
+{
+	int ret;
+	uint16_t index;
+	struct tim_mem_entry entry;
+	struct timvf_ring *timr = adptr->data->adapter_priv;
+	for (index = 0; index < nb_timers; index++) {
+		if (unlikely(tim[index]->timeout_ticks > timr->meta.nb_bkts)) {
+			tim[index]->state = RTE_EVENT_TIMER_ERROR_TOOLATE;
+			rte_errno = -EINVAL;
+			break;
+		}
+
+		entry.w0 = (tim[index]->ev.event & 0xFFC000000000) >> 6 |
+			(tim[index]->ev.event & 0xFFFFFFFFF);
+		entry.wqe = tim[index]->ev.u64;
+		ret = timvf_add_entry_mp(timr, tim[index]->timeout_ticks,
+				tim[index], &entry);
+		if (unlikely(ret)) {
+			rte_errno = -ret;
+			break;
+		}
+	}
+
+	return index;
+}
diff --git a/drivers/event/octeontx/timvf_worker.h b/drivers/event/octeontx/timvf_worker.h
new file mode 100644
index 000000000..b63dd763c
--- /dev/null
+++ b/drivers/event/octeontx/timvf_worker.h
@@ -0,0 +1,278 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Cavium, Inc
+ */
+
+#include <rte_common.h>
+#include <rte_branch_prediction.h>
+
+#include "timvf_evdev.h"
+
+static inline int16_t
+timr_bkt_fetch_rem(uint64_t w1)
+{
+	return (w1 >> TIM_BUCKET_W1_S_CHUNK_REMAINDER) &
+		TIM_BUCKET_W1_M_CHUNK_REMAINDER;
+}
+
+static inline int16_t
+timr_bkt_get_rem(struct tim_mem_bucket *bktp)
+{
+	return __atomic_load_n((int16_t *)&bktp->chunk_remainder,
+			__ATOMIC_ACQUIRE);
+}
+
+static inline void
+timr_bkt_set_rem(struct tim_mem_bucket *bktp, uint16_t v)
+{
+	__atomic_store_n((int16_t *)&bktp->chunk_remainder, v,
+			__ATOMIC_RELEASE);
+}
+
+static inline void
+timr_bkt_sub_rem(struct tim_mem_bucket *bktp, uint16_t v)
+{
+	__atomic_fetch_sub((int16_t *)&bktp->chunk_remainder, v,
+			__ATOMIC_RELEASE);
+}
+
+static inline uint8_t
+timr_bkt_get_sbt(uint64_t w1)
+{
+	return (w1 >> TIM_BUCKET_W1_S_SBT) & TIM_BUCKET_W1_M_SBT;
+}
+
+static inline uint64_t
+timr_bkt_set_sbt(struct tim_mem_bucket *bktp)
+{
+	const uint64_t v = TIM_BUCKET_W1_M_SBT << TIM_BUCKET_W1_S_SBT;
+	return __atomic_fetch_or((uint64_t *)&bktp->w1, v, __ATOMIC_ACQ_REL);
+}
+
+static inline uint64_t
+timr_bkt_clr_sbt(struct tim_mem_bucket *bktp)
+{
+	const uint64_t v = ~(TIM_BUCKET_W1_M_SBT << TIM_BUCKET_W1_S_SBT);
+	return __atomic_fetch_and((uint64_t *)&bktp->w1, v, __ATOMIC_ACQ_REL);
+}
+
+static inline uint8_t
+timr_bkt_get_shbt(uint64_t w1)
+{
+	return ((w1 >> TIM_BUCKET_W1_S_HBT) & TIM_BUCKET_W1_M_HBT) |
+		((w1 >> TIM_BUCKET_W1_S_SBT) & TIM_BUCKET_W1_M_SBT);
+}
+
+static inline uint8_t
+timr_bkt_hbt_chk(struct tim_mem_bucket *bktp)
+{
+	return (__atomic_load_n((uint64_t *)&bktp->w1, __ATOMIC_ACQ_REL)
+			>> TIM_BUCKET_W1_S_HBT) & TIM_BUCKET_W1_M_HBT;
+}
+
+static inline uint8_t
+timr_bkt_sbt_chk(struct tim_mem_bucket *bktp)
+{
+	return (__atomic_load_n((uint64_t *)&bktp->w1, __ATOMIC_ACQ_REL)
+			>> TIM_BUCKET_W1_S_SBT) & TIM_BUCKET_W1_M_SBT;
+}
+
+static inline uint8_t
+timr_bkt_get_bsk(uint64_t w1)
+{
+	return (w1 >> TIM_BUCKET_W1_S_BSK) & TIM_BUCKET_W1_M_BSK;
+}
+
+static inline uint64_t
+timr_bkt_clr_bsk(struct tim_mem_bucket *bktp)
+{
+	/*Clear everything except lock. */
+	const uint64_t v = TIM_BUCKET_W1_M_LOCK << TIM_BUCKET_W1_S_LOCK;
+	return __atomic_fetch_and((uint64_t *)&bktp->w1, v, __ATOMIC_ACQ_REL);
+}
+
+static inline uint64_t
+timr_bkt_fetch_sema_lock(struct tim_mem_bucket *bktp)
+{
+	return __atomic_fetch_add((uint64_t *)&bktp->w1, TIM_BUCKET_SEMA_WLOCK,
+			__ATOMIC_ACQ_REL);
+}
+
+static inline uint64_t
+timr_bkt_fetch_sema(struct tim_mem_bucket *bktp)
+{
+	return __atomic_fetch_add((uint64_t *)&bktp->w1, TIM_BUCKET_SEMA,
+			__ATOMIC_ACQ_REL);
+}
+
+static inline uint64_t
+timr_bkt_inc_lock(struct tim_mem_bucket *bktp)
+{
+	const uint64_t v = 1ull << TIM_BUCKET_W1_S_LOCK;
+	return __atomic_fetch_add((uint64_t *)&bktp->w1, v, __ATOMIC_ACQ_REL);
+}
+
+static inline void
+timr_bkt_dec_lock(struct tim_mem_bucket *bktp)
+{
+	__atomic_add_fetch((uint8_t *)&bktp->lock, 0xff, __ATOMIC_ACQ_REL);
+}
+
+static inline uint32_t
+timr_bkt_get_nent(uint64_t w1)
+{
+	return (w1 >> TIM_BUCKET_W1_S_NUM_ENTRIES) &
+		TIM_BUCKET_W1_M_NUM_ENTRIES;
+}
+
+static inline void
+timr_bkt_inc_nent(struct tim_mem_bucket *bktp)
+{
+	__atomic_add_fetch((uint32_t *)&bktp->nb_entry, 1, __ATOMIC_ACQ_REL);
+}
+
+static inline void
+timr_bkt_add_nent(struct tim_mem_bucket *bktp, uint32_t v)
+{
+	__atomic_add_fetch((uint32_t *)&bktp->nb_entry, v, __ATOMIC_ACQ_REL);
+}
+
+static inline uint64_t
+timr_bkt_clr_nent(struct tim_mem_bucket *bktp)
+{
+	const uint64_t v = ~(TIM_BUCKET_W1_M_NUM_ENTRIES <<
+			TIM_BUCKET_W1_S_NUM_ENTRIES);
+	return __atomic_and_fetch((uint64_t *)&bktp->w1, v, __ATOMIC_ACQ_REL);
+}
+
+static inline __hot struct tim_mem_entry*
+timr_clr_bkt(struct timvf_ring *timr, struct tim_mem_bucket *bkt)
+{
+	struct tim_mem_entry *chunk;
+	struct tim_mem_entry *pnext;
+	chunk = ((struct tim_mem_entry *)bkt->first_chunk);
+	chunk = (struct tim_mem_entry *)(chunk + nb_chunk_slots)->w0;
+
+	while (chunk) {
+		pnext = (struct tim_mem_entry *)((chunk + nb_chunk_slots)->w0);
+		rte_mempool_put(timr->meta.chunk_pool, chunk);
+		chunk = pnext;
+	}
+	return (struct tim_mem_entry *)bkt->first_chunk;
+}
+
+static inline int __hot
+timvf_rem_entry(struct rte_event_timer *tim)
+{
+	uint64_t lock_sema;
+	struct tim_mem_entry *entry;
+	struct tim_mem_bucket *bkt;
+	if (tim->impl_opaque[1] == 0 ||
+			tim->impl_opaque[0] == 0)
+		return -ENOENT;
+
+	entry = (struct tim_mem_entry *)tim->impl_opaque[0];
+	if (entry->wqe != tim->ev.u64) {
+		tim->impl_opaque[1] = tim->impl_opaque[0] = 0;
+		return -ENOENT;
+	}
+	bkt = (struct tim_mem_bucket *)tim->impl_opaque[1];
+	lock_sema = timr_bkt_inc_lock(bkt);
+	if (timr_bkt_get_shbt(lock_sema)
+			|| !timr_bkt_get_nent(lock_sema)) {
+		timr_bkt_dec_lock(bkt);
+		tim->impl_opaque[1] = tim->impl_opaque[0] = 0;
+		return -ENOENT;
+	}
+
+	entry->w0 = entry->wqe = 0;
+	timr_bkt_dec_lock(bkt);
+
+	tim->state = RTE_EVENT_TIMER_CANCELED;
+	tim->impl_opaque[1] = tim->impl_opaque[0] = 0;
+	return 0;
+}
+
+/* Multi producer functions. */
+static inline int __hot
+timvf_add_entry_mp(struct timvf_ring *timr, const uint32_t rel_bkt,
+		struct rte_event_timer *tim, const struct tim_mem_entry *pent)
+{
+	uint8_t lock_cnt;
+	int16_t rem;
+	uint32_t bucket;
+	uint32_t tbkt_id;
+	const uint32_t nb_bkts = timr->meta.nb_bkts;
+	uint64_t lock_sema;
+	uint64_t pos_reg;
+	const uint64_t start = timr->meta.ring_start_cyc;
+	struct tim_mem_bucket *bkt;
+	struct tim_mem_entry *chunk;
+
+__retry:
+	pos_reg = (rte_rdtsc() - start);
+	bucket = rte_reciprocal_divide_u64(pos_reg,
+			&timr->meta.fast_div) + rel_bkt;
+	tbkt_id = timr->meta.get_target_bkt(bucket, nb_bkts);
+	bkt = &timr->meta.bkt[tbkt_id];
+	while (1) {
+		/* Bucket related checks. */
+
+		/*Get Bucket sema*/
+		lock_sema = timr_bkt_fetch_sema_lock(bkt);
+		if (unlikely(timr_bkt_get_shbt(lock_sema))) {
+			timr_bkt_dec_lock(bkt);
+			goto __retry;
+		}
+
+		RTE_SET_USED(lock_cnt);
+		/* Insert the work. */
+		rem = timr_bkt_fetch_rem(lock_sema);
+
+		if (rem < 0) {
+			/* goto diff bucket. */
+			timr_bkt_dec_lock(bkt);
+			goto __retry;
+		} else if (!rem) {
+			/*Only one thread can be here*/
+			if (bkt->nb_entry || !bkt->first_chunk) {
+				if (unlikely(rte_mempool_get(
+							timr->meta.chunk_pool,
+							(void **)&chunk))) {
+					timr_bkt_set_rem(bkt, 0);
+					timr_bkt_dec_lock(bkt);
+					tim->impl_opaque[0] =
+						tim->impl_opaque[1] = 0;
+					tim->state = RTE_EVENT_TIMER_ERROR;
+					return -ENOMEM;
+				}
+				if (bkt->nb_entry) {
+					*(uint64_t *)((struct tim_mem_entry *)
+							bkt->current_chunk +
+							nb_chunk_slots) =
+						(uint64_t) chunk;
+				} else {
+					bkt->first_chunk = (uint64_t) chunk;
+				}
+				*(uint64_t *)(chunk + nb_chunk_slots) = 0;
+			} else {
+				chunk = timr_clr_bkt(timr, bkt);
+				*(uint64_t *)(chunk + nb_chunk_slots) = 0;
+				bkt->first_chunk = (uint64_t) chunk;
+			}
+			bkt->current_chunk = (uint64_t) chunk;
+			timr_bkt_set_rem(bkt, nb_chunk_slots - 1);
+		} else {
+			chunk = (struct tim_mem_entry *)bkt->current_chunk;
+			chunk += nb_chunk_slots - rem;
+		}
+		/* Copy work entry. */
+		*chunk = *pent;
+		timr_bkt_inc_nent(bkt);
+		timr_bkt_dec_lock(bkt);
+		break;
+	}
+	tim->impl_opaque[0] = (uint64_t)chunk;
+	tim->impl_opaque[1] = (uint64_t)bkt;
+	tim->state = RTE_EVENT_TIMER_ARMED;
+	return 0;
+}
-- 
2.16.1