From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 4486CA0C45; Mon, 13 Sep 2021 18:46:19 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 35EC6410E9; Mon, 13 Sep 2021 18:45:04 +0200 (CEST) Received: from mga12.intel.com (mga12.intel.com [192.55.52.136]) by mails.dpdk.org (Postfix) with ESMTP id 0E768410EB for ; Mon, 13 Sep 2021 18:45:00 +0200 (CEST) X-IronPort-AV: E=McAfee;i="6200,9189,10106"; a="201239143" X-IronPort-AV: E=Sophos;i="5.85,290,1624345200"; d="scan'208";a="201239143" Received: from fmsmga008.fm.intel.com ([10.253.24.58]) by fmsmga106.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 13 Sep 2021 09:44:59 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.85,290,1624345200"; d="scan'208";a="507429168" Received: from silpixa00400573.ir.intel.com (HELO silpixa00400573.ger.corp.intel.com) ([10.237.223.107]) by fmsmga008.fm.intel.com with ESMTP; 13 Sep 2021 09:44:58 -0700 From: Cristian Dumitrescu To: dev@dpdk.org Date: Mon, 13 Sep 2021 17:44:32 +0100 Message-Id: <20210913164443.16875-13-cristian.dumitrescu@intel.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20210913164443.16875-1-cristian.dumitrescu@intel.com> References: <20210910133713.93103-1-cristian.dumitrescu@intel.com> <20210913164443.16875-1-cristian.dumitrescu@intel.com> Subject: [dpdk-dev] [PATCH V3 13/24] pipeline: create inline functions for register instructions X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Create inline functions for the register instructions. Signed-off-by: Cristian Dumitrescu --- lib/pipeline/rte_swx_pipeline.c | 320 ++------------- lib/pipeline/rte_swx_pipeline_internal.h | 475 +++++++++++++++++++++++ 2 files changed, 502 insertions(+), 293 deletions(-) diff --git a/lib/pipeline/rte_swx_pipeline.c b/lib/pipeline/rte_swx_pipeline.c index 8956b6de27..c7117bb6da 100644 --- a/lib/pipeline/rte_swx_pipeline.c +++ b/lib/pipeline/rte_swx_pipeline.c @@ -3901,134 +3901,14 @@ instr_regadd_translate(struct rte_swx_pipeline *p, return 0; } -static inline uint64_t * -instr_regarray_regarray(struct rte_swx_pipeline *p, struct instruction *ip) -{ - struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; - return r->regarray; -} - -static inline uint64_t -instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip) -{ - struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; - - uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id]; - uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset]; - uint64_t idx64 = *idx64_ptr; - uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits); - uint64_t idx = idx64 & idx64_mask & r->size_mask; - - return idx; -} - -#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN - -static inline uint64_t -instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip) -{ - struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; - - uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id]; - uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset]; - uint64_t idx64 = *idx64_ptr; - uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask; - - return idx; -} - -#else - -#define instr_regarray_idx_nbo instr_regarray_idx_hbo - -#endif - -static inline uint64_t -instr_regarray_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip) -{ - struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; - - uint64_t idx = ip->regarray.idx_val & r->size_mask; - - return idx; -} - -static inline uint64_t -instr_regarray_src_hbo(struct thread *t, struct instruction *ip) -{ - uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id]; - uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset]; - uint64_t src64 = *src64_ptr; - uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits); - uint64_t src = src64 & src64_mask; - - return src; -} - -#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN - -static inline uint64_t -instr_regarray_src_nbo(struct thread *t, struct instruction *ip) -{ - uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id]; - uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset]; - uint64_t src64 = *src64_ptr; - uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits); - - return src; -} - -#else - -#define instr_regarray_src_nbo instr_regarray_src_hbo - -#endif - -static inline void -instr_regarray_dst_hbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src) -{ - uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id]; - uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset]; - uint64_t dst64 = *dst64_ptr; - uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits); - - *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); - -} - -#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN - -static inline void -instr_regarray_dst_nbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src) -{ - uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id]; - uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset]; - uint64_t dst64 = *dst64_ptr; - uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits); - - src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits); - *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); -} - -#else - -#define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set - -#endif - static inline void instr_regprefetch_rh_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint64_t *regarray, idx; - - TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id); /* Structs. */ - regarray = instr_regarray_regarray(p, ip); - idx = instr_regarray_idx_nbo(p, t, ip); - rte_prefetch0(®array[idx]); + __instr_regprefetch_rh_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4039,14 +3919,9 @@ instr_regprefetch_rm_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint64_t *regarray, idx; - - TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id); /* Structs. */ - regarray = instr_regarray_regarray(p, ip); - idx = instr_regarray_idx_hbo(p, t, ip); - rte_prefetch0(®array[idx]); + __instr_regprefetch_rm_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4057,14 +3932,9 @@ instr_regprefetch_ri_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint64_t *regarray, idx; - - TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id); /* Structs. */ - regarray = instr_regarray_regarray(p, ip); - idx = instr_regarray_idx_imm(p, ip); - rte_prefetch0(®array[idx]); + __instr_regprefetch_ri_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4075,14 +3945,9 @@ instr_regrd_hrh_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint64_t *regarray, idx; - - TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id); /* Structs. */ - regarray = instr_regarray_regarray(p, ip); - idx = instr_regarray_idx_nbo(p, t, ip); - instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]); + __instr_regrd_hrh_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4093,14 +3958,9 @@ instr_regrd_hrm_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint64_t *regarray, idx; - - TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id); /* Structs. */ - regarray = instr_regarray_regarray(p, ip); - idx = instr_regarray_idx_hbo(p, t, ip); - instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]); + __instr_regrd_hrm_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4111,14 +3971,9 @@ instr_regrd_mrh_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint64_t *regarray, idx; - - TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id); /* Structs. */ - regarray = instr_regarray_regarray(p, ip); - idx = instr_regarray_idx_nbo(p, t, ip); - instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]); + __instr_regrd_mrh_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4129,12 +3984,9 @@ instr_regrd_mrm_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint64_t *regarray, idx; /* Structs. */ - regarray = instr_regarray_regarray(p, ip); - idx = instr_regarray_idx_hbo(p, t, ip); - instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]); + __instr_regrd_mrm_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4145,14 +3997,9 @@ instr_regrd_hri_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint64_t *regarray, idx; - - TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id); /* Structs. */ - regarray = instr_regarray_regarray(p, ip); - idx = instr_regarray_idx_imm(p, ip); - instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]); + __instr_regrd_hri_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4163,14 +4010,9 @@ instr_regrd_mri_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint64_t *regarray, idx; - - TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id); /* Structs. */ - regarray = instr_regarray_regarray(p, ip); - idx = instr_regarray_idx_imm(p, ip); - instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]); + __instr_regrd_mri_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4181,15 +4023,9 @@ instr_regwr_rhh_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint64_t *regarray, idx, src; - - TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id); /* Structs. */ - regarray = instr_regarray_regarray(p, ip); - idx = instr_regarray_idx_nbo(p, t, ip); - src = instr_regarray_src_nbo(t, ip); - regarray[idx] = src; + __instr_regwr_rhh_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4200,15 +4036,9 @@ instr_regwr_rhm_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint64_t *regarray, idx, src; - - TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id); /* Structs. */ - regarray = instr_regarray_regarray(p, ip); - idx = instr_regarray_idx_nbo(p, t, ip); - src = instr_regarray_src_hbo(t, ip); - regarray[idx] = src; + __instr_regwr_rhm_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4219,15 +4049,9 @@ instr_regwr_rmh_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint64_t *regarray, idx, src; - - TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id); /* Structs. */ - regarray = instr_regarray_regarray(p, ip); - idx = instr_regarray_idx_hbo(p, t, ip); - src = instr_regarray_src_nbo(t, ip); - regarray[idx] = src; + __instr_regwr_rmh_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4238,15 +4062,9 @@ instr_regwr_rmm_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint64_t *regarray, idx, src; - - TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id); /* Structs. */ - regarray = instr_regarray_regarray(p, ip); - idx = instr_regarray_idx_hbo(p, t, ip); - src = instr_regarray_src_hbo(t, ip); - regarray[idx] = src; + __instr_regwr_rmm_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4257,15 +4075,9 @@ instr_regwr_rhi_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint64_t *regarray, idx, src; - - TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id); /* Structs. */ - regarray = instr_regarray_regarray(p, ip); - idx = instr_regarray_idx_nbo(p, t, ip); - src = ip->regarray.dstsrc_val; - regarray[idx] = src; + __instr_regwr_rhi_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4276,15 +4088,9 @@ instr_regwr_rmi_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint64_t *regarray, idx, src; - - TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id); /* Structs. */ - regarray = instr_regarray_regarray(p, ip); - idx = instr_regarray_idx_hbo(p, t, ip); - src = ip->regarray.dstsrc_val; - regarray[idx] = src; + __instr_regwr_rmi_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4295,15 +4101,9 @@ instr_regwr_rih_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint64_t *regarray, idx, src; - - TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id); /* Structs. */ - regarray = instr_regarray_regarray(p, ip); - idx = instr_regarray_idx_imm(p, ip); - src = instr_regarray_src_nbo(t, ip); - regarray[idx] = src; + __instr_regwr_rih_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4314,15 +4114,9 @@ instr_regwr_rim_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint64_t *regarray, idx, src; - - TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id); /* Structs. */ - regarray = instr_regarray_regarray(p, ip); - idx = instr_regarray_idx_imm(p, ip); - src = instr_regarray_src_hbo(t, ip); - regarray[idx] = src; + __instr_regwr_rim_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4333,15 +4127,9 @@ instr_regwr_rii_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint64_t *regarray, idx, src; - - TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id); /* Structs. */ - regarray = instr_regarray_regarray(p, ip); - idx = instr_regarray_idx_imm(p, ip); - src = ip->regarray.dstsrc_val; - regarray[idx] = src; + __instr_regwr_rii_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4352,15 +4140,9 @@ instr_regadd_rhh_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint64_t *regarray, idx, src; - - TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id); /* Structs. */ - regarray = instr_regarray_regarray(p, ip); - idx = instr_regarray_idx_nbo(p, t, ip); - src = instr_regarray_src_nbo(t, ip); - regarray[idx] += src; + __instr_regadd_rhh_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4371,15 +4153,9 @@ instr_regadd_rhm_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint64_t *regarray, idx, src; - - TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id); /* Structs. */ - regarray = instr_regarray_regarray(p, ip); - idx = instr_regarray_idx_nbo(p, t, ip); - src = instr_regarray_src_hbo(t, ip); - regarray[idx] += src; + __instr_regadd_rhm_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4390,15 +4166,9 @@ instr_regadd_rmh_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint64_t *regarray, idx, src; - - TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id); /* Structs. */ - regarray = instr_regarray_regarray(p, ip); - idx = instr_regarray_idx_hbo(p, t, ip); - src = instr_regarray_src_nbo(t, ip); - regarray[idx] += src; + __instr_regadd_rmh_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4409,15 +4179,9 @@ instr_regadd_rmm_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint64_t *regarray, idx, src; - - TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id); /* Structs. */ - regarray = instr_regarray_regarray(p, ip); - idx = instr_regarray_idx_hbo(p, t, ip); - src = instr_regarray_src_hbo(t, ip); - regarray[idx] += src; + __instr_regadd_rmm_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4428,15 +4192,9 @@ instr_regadd_rhi_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint64_t *regarray, idx, src; - - TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id); /* Structs. */ - regarray = instr_regarray_regarray(p, ip); - idx = instr_regarray_idx_nbo(p, t, ip); - src = ip->regarray.dstsrc_val; - regarray[idx] += src; + __instr_regadd_rhi_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4447,15 +4205,9 @@ instr_regadd_rmi_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint64_t *regarray, idx, src; - - TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id); /* Structs. */ - regarray = instr_regarray_regarray(p, ip); - idx = instr_regarray_idx_hbo(p, t, ip); - src = ip->regarray.dstsrc_val; - regarray[idx] += src; + __instr_regadd_rmi_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4466,15 +4218,9 @@ instr_regadd_rih_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint64_t *regarray, idx, src; - - TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id); /* Structs. */ - regarray = instr_regarray_regarray(p, ip); - idx = instr_regarray_idx_imm(p, ip); - src = instr_regarray_src_nbo(t, ip); - regarray[idx] += src; + __instr_regadd_rih_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4485,15 +4231,9 @@ instr_regadd_rim_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint64_t *regarray, idx, src; - - TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id); /* Structs. */ - regarray = instr_regarray_regarray(p, ip); - idx = instr_regarray_idx_imm(p, ip); - src = instr_regarray_src_hbo(t, ip); - regarray[idx] += src; + __instr_regadd_rim_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); @@ -4504,15 +4244,9 @@ instr_regadd_rii_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint64_t *regarray, idx, src; - - TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id); /* Structs. */ - regarray = instr_regarray_regarray(p, ip); - idx = instr_regarray_idx_imm(p, ip); - src = ip->regarray.dstsrc_val; - regarray[idx] += src; + __instr_regadd_rii_exec(p, t, ip); /* Thread. */ thread_ip_inc(p); diff --git a/lib/pipeline/rte_swx_pipeline_internal.h b/lib/pipeline/rte_swx_pipeline_internal.h index 7c4a2c05ef..2526c2f4c7 100644 --- a/lib/pipeline/rte_swx_pipeline_internal.h +++ b/lib/pipeline/rte_swx_pipeline_internal.h @@ -2827,4 +2827,479 @@ __instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p __rte_unused, *dst16_ptr = (uint16_t)r; } +/* + * Register array. + */ +static inline uint64_t * +instr_regarray_regarray(struct rte_swx_pipeline *p, const struct instruction *ip) +{ + struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; + return r->regarray; +} + +static inline uint64_t +instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; + + uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id]; + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset]; + uint64_t idx64 = *idx64_ptr; + uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits); + uint64_t idx = idx64 & idx64_mask & r->size_mask; + + return idx; +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline uint64_t +instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; + + uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id]; + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset]; + uint64_t idx64 = *idx64_ptr; + uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask; + + return idx; +} + +#else + +#define instr_regarray_idx_nbo instr_regarray_idx_hbo + +#endif + +static inline uint64_t +instr_regarray_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip) +{ + struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; + + uint64_t idx = ip->regarray.idx_val & r->size_mask; + + return idx; +} + +static inline uint64_t +instr_regarray_src_hbo(struct thread *t, const struct instruction *ip) +{ + uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id]; + uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset]; + uint64_t src64 = *src64_ptr; + uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits); + uint64_t src = src64 & src64_mask; + + return src; +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline uint64_t +instr_regarray_src_nbo(struct thread *t, const struct instruction *ip) +{ + uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id]; + uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset]; + uint64_t src64 = *src64_ptr; + uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits); + + return src; +} + +#else + +#define instr_regarray_src_nbo instr_regarray_src_hbo + +#endif + +static inline void +instr_regarray_dst_hbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src) +{ + uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id]; + uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset]; + uint64_t dst64 = *dst64_ptr; + uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits); + + *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); + +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline void +instr_regarray_dst_nbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src) +{ + uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id]; + uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset]; + uint64_t dst64 = *dst64_ptr; + uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits); + + src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits); + *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); +} + +#else + +#define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set + +#endif + +static inline void +__instr_regprefetch_rh_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + rte_prefetch0(®array[idx]); +} + +static inline void +__instr_regprefetch_rm_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + rte_prefetch0(®array[idx]); +} + +static inline void +__instr_regprefetch_ri_exec(struct rte_swx_pipeline *p, + struct thread *t __rte_unused, + const struct instruction *ip) +{ + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + rte_prefetch0(®array[idx]); +} + +static inline void +__instr_regrd_hrh_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]); +} + +static inline void +__instr_regrd_hrm_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]); +} + +static inline void +__instr_regrd_mrh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]); +} + +static inline void +__instr_regrd_mrm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (m = r[m])\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]); +} + +static inline void +__instr_regrd_hri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]); +} + +static inline void +__instr_regrd_mri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]); +} + +static inline void +__instr_regwr_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] = src; +} + +static inline void +__instr_regwr_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] = src; +} + +static inline void +__instr_regwr_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] = src; +} + +static inline void +__instr_regwr_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] = src; +} + +static inline void +__instr_regwr_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] = src; +} + +static inline void +__instr_regwr_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] = src; +} + +static inline void +__instr_regwr_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] = src; +} + +static inline void +__instr_regwr_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] = src; +} + +static inline void +__instr_regwr_rii_exec(struct rte_swx_pipeline *p, + struct thread *t __rte_unused, + const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] = src; +} + +static inline void +__instr_regadd_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] += src; +} + +static inline void +__instr_regadd_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] += src; +} + +static inline void +__instr_regadd_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] += src; +} + +static inline void +__instr_regadd_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] += src; +} + +static inline void +__instr_regadd_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] += src; +} + +static inline void +__instr_regadd_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] += src; +} + +static inline void +__instr_regadd_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] += src; +} + +static inline void +__instr_regadd_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] += src; +} + +static inline void +__instr_regadd_rii_exec(struct rte_swx_pipeline *p, + struct thread *t __rte_unused, + const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] += src; +} + #endif -- 2.17.1