From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id C5A2CA0C45; Mon, 13 Sep 2021 18:46:35 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 19F3E4118C; Mon, 13 Sep 2021 18:45:07 +0200 (CEST) Received: from mga12.intel.com (mga12.intel.com [192.55.52.136]) by mails.dpdk.org (Postfix) with ESMTP id 475354116A for ; Mon, 13 Sep 2021 18:45:02 +0200 (CEST) X-IronPort-AV: E=McAfee;i="6200,9189,10106"; a="201239152" X-IronPort-AV: E=Sophos;i="5.85,290,1624345200"; d="scan'208";a="201239152" Received: from fmsmga008.fm.intel.com ([10.253.24.58]) by fmsmga106.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 13 Sep 2021 09:45:01 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.85,290,1624345200"; d="scan'208";a="507429180" Received: from silpixa00400573.ir.intel.com (HELO silpixa00400573.ger.corp.intel.com) ([10.237.223.107]) by fmsmga008.fm.intel.com with ESMTP; 13 Sep 2021 09:44:59 -0700 From: Cristian Dumitrescu To: dev@dpdk.org Date: Mon, 13 Sep 2021 17:44:33 +0100 Message-Id: <20210913164443.16875-14-cristian.dumitrescu@intel.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20210913164443.16875-1-cristian.dumitrescu@intel.com> References: <20210910133713.93103-1-cristian.dumitrescu@intel.com> <20210913164443.16875-1-cristian.dumitrescu@intel.com> Subject: [dpdk-dev] [PATCH V3 14/24] pipeline: create inline functions for meter instructions X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Create inline functions for the meter instructions. Signed-off-by: Cristian Dumitrescu --- lib/pipeline/rte_swx_pipeline.c | 457 +------------------ lib/pipeline/rte_swx_pipeline_internal.h | 541 +++++++++++++++++++++++ 2 files changed, 558 insertions(+), 440 deletions(-) diff --git a/lib/pipeline/rte_swx_pipeline.c b/lib/pipeline/rte_swx_pipeline.c index c7117bb6da..8b64c57652 100644 --- a/lib/pipeline/rte_swx_pipeline.c +++ b/lib/pipeline/rte_swx_pipeline.c @@ -4470,119 +4470,14 @@ instr_meter_translate(struct rte_swx_pipeline *p, CHECK(0, EINVAL); } -static inline struct meter * -instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip) -{ - struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id]; - - uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id]; - uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset]; - uint64_t idx64 = *idx64_ptr; - uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits); - uint64_t idx = idx64 & idx64_mask & r->size_mask; - - return &r->metarray[idx]; -} - -#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN - -static inline struct meter * -instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip) -{ - struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id]; - - uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id]; - uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset]; - uint64_t idx64 = *idx64_ptr; - uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask; - - return &r->metarray[idx]; -} - -#else - -#define instr_meter_idx_nbo instr_meter_idx_hbo - -#endif - -static inline struct meter * -instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip) -{ - struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id]; - - uint64_t idx = ip->meter.idx_val & r->size_mask; - - return &r->metarray[idx]; -} - -static inline uint32_t -instr_meter_length_hbo(struct thread *t, struct instruction *ip) -{ - uint8_t *src_struct = t->structs[ip->meter.length.struct_id]; - uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset]; - uint64_t src64 = *src64_ptr; - uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits); - uint64_t src = src64 & src64_mask; - - return (uint32_t)src; -} - -#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN - -static inline uint32_t -instr_meter_length_nbo(struct thread *t, struct instruction *ip) -{ - uint8_t *src_struct = t->structs[ip->meter.length.struct_id]; - uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset]; - uint64_t src64 = *src64_ptr; - uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits); - - return (uint32_t)src; -} - -#else - -#define instr_meter_length_nbo instr_meter_length_hbo - -#endif - -static inline enum rte_color -instr_meter_color_in_hbo(struct thread *t, struct instruction *ip) -{ - uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id]; - uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset]; - uint64_t src64 = *src64_ptr; - uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits); - uint64_t src = src64 & src64_mask; - - return (enum rte_color)src; -} - -static inline void -instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out) -{ - uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id]; - uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset]; - uint64_t dst64 = *dst64_ptr; - uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits); - - uint64_t src = (uint64_t)color_out; - - *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); -} - static inline void instr_metprefetch_h_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - struct meter *m; - - TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id); /* Structs. */ - m = instr_meter_idx_nbo(p, t, ip); - rte_prefetch0(m); + __instr_metprefetch_h_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4593,13 +4488,9 @@ instr_metprefetch_m_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - struct meter *m; - - TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id); /* Structs. */ - m = instr_meter_idx_hbo(p, t, ip); - rte_prefetch0(m); + __instr_metprefetch_m_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4610,13 +4501,9 @@ instr_metprefetch_i_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - struct meter *m; - - TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id); /* Structs. */ - m = instr_meter_idx_imm(p, ip); - rte_prefetch0(m); + __instr_metprefetch_i_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4627,35 +4514,9 @@ instr_meter_hhm_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - struct meter *m; - uint64_t time, n_pkts, n_bytes; - uint32_t length; - enum rte_color color_in, color_out; - - TRACE("[Thread %2u] meter (hhm)\n", p->thread_id); /* Structs. */ - m = instr_meter_idx_nbo(p, t, ip); - rte_prefetch0(m->n_pkts); - time = rte_get_tsc_cycles(); - length = instr_meter_length_nbo(t, ip); - color_in = instr_meter_color_in_hbo(t, ip); - - color_out = rte_meter_trtcm_color_aware_check(&m->m, - &m->profile->profile, - time, - length, - color_in); - - color_out &= m->color_mask; - - n_pkts = m->n_pkts[color_out]; - n_bytes = m->n_bytes[color_out]; - - instr_meter_color_out_hbo_set(t, ip, color_out); - - m->n_pkts[color_out] = n_pkts + 1; - m->n_bytes[color_out] = n_bytes + length; + __instr_meter_hhm_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4666,35 +4527,9 @@ instr_meter_hhi_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - struct meter *m; - uint64_t time, n_pkts, n_bytes; - uint32_t length; - enum rte_color color_in, color_out; - - TRACE("[Thread %2u] meter (hhi)\n", p->thread_id); /* Structs. */ - m = instr_meter_idx_nbo(p, t, ip); - rte_prefetch0(m->n_pkts); - time = rte_get_tsc_cycles(); - length = instr_meter_length_nbo(t, ip); - color_in = (enum rte_color)ip->meter.color_in_val; - - color_out = rte_meter_trtcm_color_aware_check(&m->m, - &m->profile->profile, - time, - length, - color_in); - - color_out &= m->color_mask; - - n_pkts = m->n_pkts[color_out]; - n_bytes = m->n_bytes[color_out]; - - instr_meter_color_out_hbo_set(t, ip, color_out); - - m->n_pkts[color_out] = n_pkts + 1; - m->n_bytes[color_out] = n_bytes + length; + __instr_meter_hhi_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4705,73 +4540,22 @@ instr_meter_hmm_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - struct meter *m; - uint64_t time, n_pkts, n_bytes; - uint32_t length; - enum rte_color color_in, color_out; - - TRACE("[Thread %2u] meter (hmm)\n", p->thread_id); /* Structs. */ - m = instr_meter_idx_nbo(p, t, ip); - rte_prefetch0(m->n_pkts); - time = rte_get_tsc_cycles(); - length = instr_meter_length_hbo(t, ip); - color_in = instr_meter_color_in_hbo(t, ip); - - color_out = rte_meter_trtcm_color_aware_check(&m->m, - &m->profile->profile, - time, - length, - color_in); - - color_out &= m->color_mask; - - n_pkts = m->n_pkts[color_out]; - n_bytes = m->n_bytes[color_out]; - - instr_meter_color_out_hbo_set(t, ip, color_out); - - m->n_pkts[color_out] = n_pkts + 1; - m->n_bytes[color_out] = n_bytes + length; + __instr_meter_hmm_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); } + static inline void instr_meter_hmi_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - struct meter *m; - uint64_t time, n_pkts, n_bytes; - uint32_t length; - enum rte_color color_in, color_out; - - TRACE("[Thread %2u] meter (hmi)\n", p->thread_id); /* Structs. */ - m = instr_meter_idx_nbo(p, t, ip); - rte_prefetch0(m->n_pkts); - time = rte_get_tsc_cycles(); - length = instr_meter_length_hbo(t, ip); - color_in = (enum rte_color)ip->meter.color_in_val; - - color_out = rte_meter_trtcm_color_aware_check(&m->m, - &m->profile->profile, - time, - length, - color_in); - - color_out &= m->color_mask; - - n_pkts = m->n_pkts[color_out]; - n_bytes = m->n_bytes[color_out]; - - instr_meter_color_out_hbo_set(t, ip, color_out); - - m->n_pkts[color_out] = n_pkts + 1; - m->n_bytes[color_out] = n_bytes + length; + __instr_meter_hmi_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4782,35 +4566,9 @@ instr_meter_mhm_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - struct meter *m; - uint64_t time, n_pkts, n_bytes; - uint32_t length; - enum rte_color color_in, color_out; - - TRACE("[Thread %2u] meter (mhm)\n", p->thread_id); /* Structs. */ - m = instr_meter_idx_hbo(p, t, ip); - rte_prefetch0(m->n_pkts); - time = rte_get_tsc_cycles(); - length = instr_meter_length_nbo(t, ip); - color_in = instr_meter_color_in_hbo(t, ip); - - color_out = rte_meter_trtcm_color_aware_check(&m->m, - &m->profile->profile, - time, - length, - color_in); - - color_out &= m->color_mask; - - n_pkts = m->n_pkts[color_out]; - n_bytes = m->n_bytes[color_out]; - - instr_meter_color_out_hbo_set(t, ip, color_out); - - m->n_pkts[color_out] = n_pkts + 1; - m->n_bytes[color_out] = n_bytes + length; + __instr_meter_mhm_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4821,35 +4579,9 @@ instr_meter_mhi_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - struct meter *m; - uint64_t time, n_pkts, n_bytes; - uint32_t length; - enum rte_color color_in, color_out; - - TRACE("[Thread %2u] meter (mhi)\n", p->thread_id); /* Structs. */ - m = instr_meter_idx_hbo(p, t, ip); - rte_prefetch0(m->n_pkts); - time = rte_get_tsc_cycles(); - length = instr_meter_length_nbo(t, ip); - color_in = (enum rte_color)ip->meter.color_in_val; - - color_out = rte_meter_trtcm_color_aware_check(&m->m, - &m->profile->profile, - time, - length, - color_in); - - color_out &= m->color_mask; - - n_pkts = m->n_pkts[color_out]; - n_bytes = m->n_bytes[color_out]; - - instr_meter_color_out_hbo_set(t, ip, color_out); - - m->n_pkts[color_out] = n_pkts + 1; - m->n_bytes[color_out] = n_bytes + length; + __instr_meter_mhi_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4860,35 +4592,9 @@ instr_meter_mmm_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - struct meter *m; - uint64_t time, n_pkts, n_bytes; - uint32_t length; - enum rte_color color_in, color_out; - - TRACE("[Thread %2u] meter (mmm)\n", p->thread_id); /* Structs. */ - m = instr_meter_idx_hbo(p, t, ip); - rte_prefetch0(m->n_pkts); - time = rte_get_tsc_cycles(); - length = instr_meter_length_hbo(t, ip); - color_in = instr_meter_color_in_hbo(t, ip); - - color_out = rte_meter_trtcm_color_aware_check(&m->m, - &m->profile->profile, - time, - length, - color_in); - - color_out &= m->color_mask; - - n_pkts = m->n_pkts[color_out]; - n_bytes = m->n_bytes[color_out]; - - instr_meter_color_out_hbo_set(t, ip, color_out); - - m->n_pkts[color_out] = n_pkts + 1; - m->n_bytes[color_out] = n_bytes + length; + __instr_meter_mmm_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4899,35 +4605,9 @@ instr_meter_mmi_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - struct meter *m; - uint64_t time, n_pkts, n_bytes; - uint32_t length; - enum rte_color color_in, color_out; - - TRACE("[Thread %2u] meter (mmi)\n", p->thread_id); /* Structs. */ - m = instr_meter_idx_hbo(p, t, ip); - rte_prefetch0(m->n_pkts); - time = rte_get_tsc_cycles(); - length = instr_meter_length_hbo(t, ip); - color_in = (enum rte_color)ip->meter.color_in_val; - - color_out = rte_meter_trtcm_color_aware_check(&m->m, - &m->profile->profile, - time, - length, - color_in); - - color_out &= m->color_mask; - - n_pkts = m->n_pkts[color_out]; - n_bytes = m->n_bytes[color_out]; - - instr_meter_color_out_hbo_set(t, ip, color_out); - - m->n_pkts[color_out] = n_pkts + 1; - m->n_bytes[color_out] = n_bytes + length; + __instr_meter_mmi_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4938,35 +4618,9 @@ instr_meter_ihm_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - struct meter *m; - uint64_t time, n_pkts, n_bytes; - uint32_t length; - enum rte_color color_in, color_out; - - TRACE("[Thread %2u] meter (ihm)\n", p->thread_id); /* Structs. */ - m = instr_meter_idx_imm(p, ip); - rte_prefetch0(m->n_pkts); - time = rte_get_tsc_cycles(); - length = instr_meter_length_nbo(t, ip); - color_in = instr_meter_color_in_hbo(t, ip); - - color_out = rte_meter_trtcm_color_aware_check(&m->m, - &m->profile->profile, - time, - length, - color_in); - - color_out &= m->color_mask; - - n_pkts = m->n_pkts[color_out]; - n_bytes = m->n_bytes[color_out]; - - instr_meter_color_out_hbo_set(t, ip, color_out); - - m->n_pkts[color_out] = n_pkts + 1; - m->n_bytes[color_out] = n_bytes + length; + __instr_meter_ihm_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4977,35 +4631,9 @@ instr_meter_ihi_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - struct meter *m; - uint64_t time, n_pkts, n_bytes; - uint32_t length; - enum rte_color color_in, color_out; - - TRACE("[Thread %2u] meter (ihi)\n", p->thread_id); /* Structs. */ - m = instr_meter_idx_imm(p, ip); - rte_prefetch0(m->n_pkts); - time = rte_get_tsc_cycles(); - length = instr_meter_length_nbo(t, ip); - color_in = (enum rte_color)ip->meter.color_in_val; - - color_out = rte_meter_trtcm_color_aware_check(&m->m, - &m->profile->profile, - time, - length, - color_in); - - color_out &= m->color_mask; - - n_pkts = m->n_pkts[color_out]; - n_bytes = m->n_bytes[color_out]; - - instr_meter_color_out_hbo_set(t, ip, color_out); - - m->n_pkts[color_out] = n_pkts + 1; - m->n_bytes[color_out] = n_bytes + length; + __instr_meter_ihi_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -5016,73 +4644,22 @@ instr_meter_imm_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - struct meter *m; - uint64_t time, n_pkts, n_bytes; - uint32_t length; - enum rte_color color_in, color_out; - - TRACE("[Thread %2u] meter (imm)\n", p->thread_id); /* Structs. */ - m = instr_meter_idx_imm(p, ip); - rte_prefetch0(m->n_pkts); - time = rte_get_tsc_cycles(); - length = instr_meter_length_hbo(t, ip); - color_in = instr_meter_color_in_hbo(t, ip); - - color_out = rte_meter_trtcm_color_aware_check(&m->m, - &m->profile->profile, - time, - length, - color_in); - - color_out &= m->color_mask; - - n_pkts = m->n_pkts[color_out]; - n_bytes = m->n_bytes[color_out]; - - instr_meter_color_out_hbo_set(t, ip, color_out); - - m->n_pkts[color_out] = n_pkts + 1; - m->n_bytes[color_out] = n_bytes + length; + __instr_meter_imm_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); } + static inline void instr_meter_imi_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - struct meter *m; - uint64_t time, n_pkts, n_bytes; - uint32_t length; - enum rte_color color_in, color_out; - - TRACE("[Thread %2u] meter (imi)\n", p->thread_id); /* Structs. */ - m = instr_meter_idx_imm(p, ip); - rte_prefetch0(m->n_pkts); - time = rte_get_tsc_cycles(); - length = instr_meter_length_hbo(t, ip); - color_in = (enum rte_color)ip->meter.color_in_val; - - color_out = rte_meter_trtcm_color_aware_check(&m->m, - &m->profile->profile, - time, - length, - color_in); - - color_out &= m->color_mask; - - n_pkts = m->n_pkts[color_out]; - n_bytes = m->n_bytes[color_out]; - - instr_meter_color_out_hbo_set(t, ip, color_out); - - m->n_pkts[color_out] = n_pkts + 1; - m->n_bytes[color_out] = n_bytes + length; + __instr_meter_imi_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); diff --git a/lib/pipeline/rte_swx_pipeline_internal.h b/lib/pipeline/rte_swx_pipeline_internal.h index 2526c2f4c7..791adfb471 100644 --- a/lib/pipeline/rte_swx_pipeline_internal.h +++ b/lib/pipeline/rte_swx_pipeline_internal.h @@ -3302,4 +3302,545 @@ __instr_regadd_rii_exec(struct rte_swx_pipeline *p, regarray[idx] += src; } +/* + * metarray. + */ +static inline struct meter * +instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id]; + + uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id]; + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset]; + uint64_t idx64 = *idx64_ptr; + uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits); + uint64_t idx = idx64 & idx64_mask & r->size_mask; + + return &r->metarray[idx]; +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline struct meter * +instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id]; + + uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id]; + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset]; + uint64_t idx64 = *idx64_ptr; + uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask; + + return &r->metarray[idx]; +} + +#else + +#define instr_meter_idx_nbo instr_meter_idx_hbo + +#endif + +static inline struct meter * +instr_meter_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip) +{ + struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id]; + + uint64_t idx = ip->meter.idx_val & r->size_mask; + + return &r->metarray[idx]; +} + +static inline uint32_t +instr_meter_length_hbo(struct thread *t, const struct instruction *ip) +{ + uint8_t *src_struct = t->structs[ip->meter.length.struct_id]; + uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset]; + uint64_t src64 = *src64_ptr; + uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits); + uint64_t src = src64 & src64_mask; + + return (uint32_t)src; +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline uint32_t +instr_meter_length_nbo(struct thread *t, const struct instruction *ip) +{ + uint8_t *src_struct = t->structs[ip->meter.length.struct_id]; + uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset]; + uint64_t src64 = *src64_ptr; + uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits); + + return (uint32_t)src; +} + +#else + +#define instr_meter_length_nbo instr_meter_length_hbo + +#endif + +static inline enum rte_color +instr_meter_color_in_hbo(struct thread *t, const struct instruction *ip) +{ + uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id]; + uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset]; + uint64_t src64 = *src64_ptr; + uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits); + uint64_t src = src64 & src64_mask; + + return (enum rte_color)src; +} + +static inline void +instr_meter_color_out_hbo_set(struct thread *t, + const struct instruction *ip, + enum rte_color color_out) +{ + uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id]; + uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset]; + uint64_t dst64 = *dst64_ptr; + uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits); + + uint64_t src = (uint64_t)color_out; + + *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); +} + +static inline void +__instr_metprefetch_h_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + struct meter *m; + + TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id); + + m = instr_meter_idx_nbo(p, t, ip); + rte_prefetch0(m); +} + +static inline void +__instr_metprefetch_m_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + struct meter *m; + + TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id); + + m = instr_meter_idx_hbo(p, t, ip); + rte_prefetch0(m); +} + +static inline void +__instr_metprefetch_i_exec(struct rte_swx_pipeline *p, + struct thread *t __rte_unused, + const struct instruction *ip) +{ + struct meter *m; + + TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id); + + m = instr_meter_idx_imm(p, ip); + rte_prefetch0(m); +} + +static inline void +__instr_meter_hhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (hhm)\n", p->thread_id); + + m = instr_meter_idx_nbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_hhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (hhi)\n", p->thread_id); + + m = instr_meter_idx_nbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_hmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (hmm)\n", p->thread_id); + + m = instr_meter_idx_nbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_hmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (hmi)\n", p->thread_id); + + m = instr_meter_idx_nbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_mhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (mhm)\n", p->thread_id); + + m = instr_meter_idx_hbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_mhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (mhi)\n", p->thread_id); + + m = instr_meter_idx_hbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_mmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (mmm)\n", p->thread_id); + + m = instr_meter_idx_hbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_mmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (mmi)\n", p->thread_id); + + m = instr_meter_idx_hbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_ihm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (ihm)\n", p->thread_id); + + m = instr_meter_idx_imm(p, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_ihi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (ihi)\n", p->thread_id); + + m = instr_meter_idx_imm(p, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_imm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (imm)\n", p->thread_id); + + m = instr_meter_idx_imm(p, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_imi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (imi)\n", p->thread_id); + + m = instr_meter_idx_imm(p, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + #endif -- 2.17.1