[dpdk-dev] [PATCH v4 30/34] event/cnxk: add timer arm timeout burst

[<pbhagavatula@marvell.com>]

From: Pavan Nikhilesh <pbhagavatula@marvell.com>

Add event timer arm timeout burst function.
All the timers requested to be armed have the same timeout.

Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com>
Signed-off-by: Shijith Thotton <sthotton@marvell.com>
---
 drivers/event/cnxk/cnxk_tim_evdev.c  |   7 ++
 drivers/event/cnxk/cnxk_tim_evdev.h  |  12 +++
 drivers/event/cnxk/cnxk_tim_worker.c |  53 ++++++++++
 drivers/event/cnxk/cnxk_tim_worker.h | 141 +++++++++++++++++++++++++++
 4 files changed, 213 insertions(+)

diff --git a/drivers/event/cnxk/cnxk_tim_evdev.c b/drivers/event/cnxk/cnxk_tim_evdev.c
index ecc952a6a..68c3b3049 100644
--- a/drivers/event/cnxk/cnxk_tim_evdev.c
+++ b/drivers/event/cnxk/cnxk_tim_evdev.c
@@ -88,7 +88,14 @@ cnxk_tim_set_fp_ops(struct cnxk_tim_ring *tim_ring)
 #undef FP
 	};
 
+	const rte_event_timer_arm_tmo_tick_burst_t arm_tmo_burst[2] = {
+#define FP(_name, _f1, flags) [_f1] = cnxk_tim_arm_tmo_tick_burst_##_name,
+		TIM_ARM_TMO_FASTPATH_MODES
+#undef FP
+	};
+
 	cnxk_tim_ops.arm_burst = arm_burst[tim_ring->ena_dfb][prod_flag];
+	cnxk_tim_ops.arm_tmo_tick_burst = arm_tmo_burst[tim_ring->ena_dfb];
 }
 
 static void
diff --git a/drivers/event/cnxk/cnxk_tim_evdev.h b/drivers/event/cnxk/cnxk_tim_evdev.h
index 1f2aad17a..b66aac17c 100644
--- a/drivers/event/cnxk/cnxk_tim_evdev.h
+++ b/drivers/event/cnxk/cnxk_tim_evdev.h
@@ -217,6 +217,10 @@ cnxk_tim_cntfrq(void)
 	FP(fb_sp, 1, 0, CNXK_TIM_ENA_FB | CNXK_TIM_SP)                         \
 	FP(fb_mp, 1, 1, CNXK_TIM_ENA_FB | CNXK_TIM_MP)
 
+#define TIM_ARM_TMO_FASTPATH_MODES                                             \
+	FP(dfb, 0, CNXK_TIM_ENA_DFB)                                           \
+	FP(fb, 1, CNXK_TIM_ENA_FB)
+
 #define FP(_name, _f2, _f1, flags)                                             \
 	uint16_t cnxk_tim_arm_burst_##_name(                                   \
 		const struct rte_event_timer_adapter *adptr,                   \
@@ -224,6 +228,14 @@ cnxk_tim_cntfrq(void)
 TIM_ARM_FASTPATH_MODES
 #undef FP
 
+#define FP(_name, _f1, flags)                                                  \
+	uint16_t cnxk_tim_arm_tmo_tick_burst_##_name(                          \
+		const struct rte_event_timer_adapter *adptr,                   \
+		struct rte_event_timer **tim, const uint64_t timeout_tick,     \
+		const uint16_t nb_timers);
+TIM_ARM_TMO_FASTPATH_MODES
+#undef FP
+
 int cnxk_tim_caps_get(const struct rte_eventdev *dev, uint64_t flags,
 		      uint32_t *caps,
 		      const struct rte_event_timer_adapter_ops **ops);
diff --git a/drivers/event/cnxk/cnxk_tim_worker.c b/drivers/event/cnxk/cnxk_tim_worker.c
index 268f845c8..717c53fb7 100644
--- a/drivers/event/cnxk/cnxk_tim_worker.c
+++ b/drivers/event/cnxk/cnxk_tim_worker.c
@@ -99,3 +99,56 @@ cnxk_tim_timer_arm_burst(const struct rte_event_timer_adapter *adptr,
 	}
 TIM_ARM_FASTPATH_MODES
 #undef FP
+
+static __rte_always_inline uint16_t
+cnxk_tim_timer_arm_tmo_brst(const struct rte_event_timer_adapter *adptr,
+			    struct rte_event_timer **tim,
+			    const uint64_t timeout_tick,
+			    const uint16_t nb_timers, const uint8_t flags)
+{
+	struct cnxk_tim_ent entry[CNXK_TIM_MAX_BURST] __rte_cache_aligned;
+	struct cnxk_tim_ring *tim_ring = adptr->data->adapter_priv;
+	uint16_t set_timers = 0;
+	uint16_t arr_idx = 0;
+	uint16_t idx;
+	int ret;
+
+	if (unlikely(!timeout_tick || timeout_tick > tim_ring->nb_bkts)) {
+		const enum rte_event_timer_state state =
+			timeout_tick ? RTE_EVENT_TIMER_ERROR_TOOLATE :
+					     RTE_EVENT_TIMER_ERROR_TOOEARLY;
+		for (idx = 0; idx < nb_timers; idx++)
+			tim[idx]->state = state;
+
+		rte_errno = EINVAL;
+		return 0;
+	}
+
+	cnxk_tim_sync_start_cyc(tim_ring);
+	while (arr_idx < nb_timers) {
+		for (idx = 0; idx < CNXK_TIM_MAX_BURST && (arr_idx < nb_timers);
+		     idx++, arr_idx++) {
+			cnxk_tim_format_event(tim[arr_idx], &entry[idx]);
+		}
+		ret = cnxk_tim_add_entry_brst(tim_ring, timeout_tick,
+					      &tim[set_timers], entry, idx,
+					      flags);
+		set_timers += ret;
+		if (ret != idx)
+			break;
+	}
+
+	return set_timers;
+}
+
+#define FP(_name, _f1, _flags)                                                 \
+	uint16_t __rte_noinline cnxk_tim_arm_tmo_tick_burst_##_name(           \
+		const struct rte_event_timer_adapter *adptr,                   \
+		struct rte_event_timer **tim, const uint64_t timeout_tick,     \
+		const uint16_t nb_timers)                                      \
+	{                                                                      \
+		return cnxk_tim_timer_arm_tmo_brst(adptr, tim, timeout_tick,   \
+						   nb_timers, _flags);         \
+	}
+TIM_ARM_TMO_FASTPATH_MODES
+#undef FP
diff --git a/drivers/event/cnxk/cnxk_tim_worker.h b/drivers/event/cnxk/cnxk_tim_worker.h
index de8464e33..56cb4cdd9 100644
--- a/drivers/event/cnxk/cnxk_tim_worker.h
+++ b/drivers/event/cnxk/cnxk_tim_worker.h
@@ -420,4 +420,145 @@ cnxk_tim_add_entry_mp(struct cnxk_tim_ring *const tim_ring,
 	return 0;
 }
 
+static inline uint16_t
+cnxk_tim_cpy_wrk(uint16_t index, uint16_t cpy_lmt, struct cnxk_tim_ent *chunk,
+		 struct rte_event_timer **const tim,
+		 const struct cnxk_tim_ent *const ents,
+		 const struct cnxk_tim_bkt *const bkt)
+{
+	for (; index < cpy_lmt; index++) {
+		*chunk = *(ents + index);
+		tim[index]->impl_opaque[0] = (uintptr_t)chunk++;
+		tim[index]->impl_opaque[1] = (uintptr_t)bkt;
+		tim[index]->state = RTE_EVENT_TIMER_ARMED;
+	}
+
+	return index;
+}
+
+/* Burst mode functions */
+static inline int
+cnxk_tim_add_entry_brst(struct cnxk_tim_ring *const tim_ring,
+			const uint16_t rel_bkt,
+			struct rte_event_timer **const tim,
+			const struct cnxk_tim_ent *ents,
+			const uint16_t nb_timers, const uint8_t flags)
+{
+	struct cnxk_tim_ent *chunk = NULL;
+	struct cnxk_tim_bkt *mirr_bkt;
+	struct cnxk_tim_bkt *bkt;
+	uint16_t chunk_remainder;
+	uint16_t index = 0;
+	uint64_t lock_sema;
+	int16_t rem, crem;
+	uint8_t lock_cnt;
+
+__retry:
+	cnxk_tim_get_target_bucket(tim_ring, rel_bkt, &bkt, &mirr_bkt);
+
+	/* Only one thread beyond this. */
+	lock_sema = cnxk_tim_bkt_inc_lock(bkt);
+	lock_cnt = (uint8_t)((lock_sema >> TIM_BUCKET_W1_S_LOCK) &
+			     TIM_BUCKET_W1_M_LOCK);
+
+	if (lock_cnt) {
+		cnxk_tim_bkt_dec_lock(bkt);
+#ifdef RTE_ARCH_ARM64
+		asm volatile(PLT_CPU_FEATURE_PREAMBLE
+			     "		ldxrb %w[lock_cnt], [%[lock]]	\n"
+			     "		tst %w[lock_cnt], 255		\n"
+			     "		beq dne%=			\n"
+			     "		sevl				\n"
+			     "rty%=:	wfe				\n"
+			     "		ldxrb %w[lock_cnt], [%[lock]]	\n"
+			     "		tst %w[lock_cnt], 255		\n"
+			     "		bne rty%=			\n"
+			     "dne%=:					\n"
+			     : [lock_cnt] "=&r"(lock_cnt)
+			     : [lock] "r"(&bkt->lock)
+			     : "memory");
+#else
+		while (__atomic_load_n(&bkt->lock, __ATOMIC_RELAXED))
+			;
+#endif
+		goto __retry;
+	}
+
+	/* Bucket related checks. */
+	if (unlikely(cnxk_tim_bkt_get_hbt(lock_sema))) {
+		if (cnxk_tim_bkt_get_nent(lock_sema) != 0) {
+			uint64_t hbt_state;
+#ifdef RTE_ARCH_ARM64
+			asm volatile(PLT_CPU_FEATURE_PREAMBLE
+				     "		ldxr %[hbt], [%[w1]]	\n"
+				     "		tbz %[hbt], 33, dne%=	\n"
+				     "		sevl			\n"
+				     "rty%=:	wfe			\n"
+				     "		ldxr %[hbt], [%[w1]]	\n"
+				     "		tbnz %[hbt], 33, rty%=	\n"
+				     "dne%=:				\n"
+				     : [hbt] "=&r"(hbt_state)
+				     : [w1] "r"((&bkt->w1))
+				     : "memory");
+#else
+			do {
+				hbt_state = __atomic_load_n(&bkt->w1,
+							    __ATOMIC_RELAXED);
+			} while (hbt_state & BIT_ULL(33));
+#endif
+
+			if (!(hbt_state & BIT_ULL(34))) {
+				cnxk_tim_bkt_dec_lock(bkt);
+				goto __retry;
+			}
+		}
+	}
+
+	chunk_remainder = cnxk_tim_bkt_fetch_rem(lock_sema);
+	rem = chunk_remainder - nb_timers;
+	if (rem < 0) {
+		crem = tim_ring->nb_chunk_slots - chunk_remainder;
+		if (chunk_remainder && crem) {
+			chunk = ((struct cnxk_tim_ent *)
+					 mirr_bkt->current_chunk) +
+				crem;
+
+			index = cnxk_tim_cpy_wrk(index, chunk_remainder, chunk,
+						 tim, ents, bkt);
+			cnxk_tim_bkt_sub_rem(bkt, chunk_remainder);
+			cnxk_tim_bkt_add_nent(bkt, chunk_remainder);
+		}
+
+		if (flags & CNXK_TIM_ENA_FB)
+			chunk = cnxk_tim_refill_chunk(bkt, mirr_bkt, tim_ring);
+		if (flags & CNXK_TIM_ENA_DFB)
+			chunk = cnxk_tim_insert_chunk(bkt, mirr_bkt, tim_ring);
+
+		if (unlikely(chunk == NULL)) {
+			cnxk_tim_bkt_dec_lock(bkt);
+			rte_errno = ENOMEM;
+			tim[index]->state = RTE_EVENT_TIMER_ERROR;
+			return crem;
+		}
+		*(uint64_t *)(chunk + tim_ring->nb_chunk_slots) = 0;
+		mirr_bkt->current_chunk = (uintptr_t)chunk;
+		cnxk_tim_cpy_wrk(index, nb_timers, chunk, tim, ents, bkt);
+
+		rem = nb_timers - chunk_remainder;
+		cnxk_tim_bkt_set_rem(bkt, tim_ring->nb_chunk_slots - rem);
+		cnxk_tim_bkt_add_nent(bkt, rem);
+	} else {
+		chunk = (struct cnxk_tim_ent *)mirr_bkt->current_chunk;
+		chunk += (tim_ring->nb_chunk_slots - chunk_remainder);
+
+		cnxk_tim_cpy_wrk(index, nb_timers, chunk, tim, ents, bkt);
+		cnxk_tim_bkt_sub_rem(bkt, nb_timers);
+		cnxk_tim_bkt_add_nent(bkt, nb_timers);
+	}
+
+	cnxk_tim_bkt_dec_lock(bkt);
+
+	return nb_timers;
+}
+
 #endif /* __CNXK_TIM_WORKER_H__ */
-- 
2.17.1