From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id A103BA0546; Fri, 30 Apr 2021 17:07:59 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id ECF0E4125D; Fri, 30 Apr 2021 17:07:11 +0200 (CEST) Received: from mga07.intel.com (mga07.intel.com [134.134.136.100]) by mails.dpdk.org (Postfix) with ESMTP id 2BA3841153 for ; Fri, 30 Apr 2021 17:07:04 +0200 (CEST) IronPort-SDR: GVJm6JOzk8gNytnkUEaWMTaHhatmWUxixqE8wg/qMb0oYsktnP/jdh4PUS4erJIT7J5oxppGZK 0tL4OpumO2AA== X-IronPort-AV: E=McAfee;i="6200,9189,9970"; a="261240858" X-IronPort-AV: E=Sophos;i="5.82,263,1613462400"; d="scan'208";a="261240858" Received: from fmsmga003.fm.intel.com ([10.253.24.29]) by orsmga105.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 30 Apr 2021 08:07:03 -0700 IronPort-SDR: D6KmbrT9rGaewOnJxYCiGZfksGiOA5VWm7lrrWSnPTtwap/ZD8Sqv6NSajlpnlhgQy5GazA0HK M9dFc/BfR7uw== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.82,263,1613462400"; d="scan'208";a="456011382" Received: from silpixa00399126.ir.intel.com ([10.237.223.78]) by FMSMGA003.fm.intel.com with ESMTP; 30 Apr 2021 08:07:01 -0700 From: Bruce Richardson To: dev@dpdk.org Cc: kevin.laatz@intel.com, sunil.pai.g@intel.com, jiayu.hu@intel.com, Bruce Richardson Date: Fri, 30 Apr 2021 16:06:34 +0100 Message-Id: <20210430150637.362610-10-bruce.richardson@intel.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20210430150637.362610-1-bruce.richardson@intel.com> References: <20210318182042.43658-1-bruce.richardson@intel.com> <20210430150637.362610-1-bruce.richardson@intel.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Subject: [dpdk-dev] [PATCH v4 09/12] raw/ioat: move idxd functions to separate file X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Split the rte_ioat_rawdev_fns.h file into two separate headers, so that the data structures for the original ioat devices and the newer idxd ones can be kept separate from each other. This makes code management and rework easier. Signed-off-by: Bruce Richardson --- drivers/raw/ioat/meson.build | 1 + drivers/raw/ioat/rte_idxd_rawdev_fns.h | 275 ++++++++++++++++++++++ drivers/raw/ioat/rte_ioat_rawdev_fns.h | 310 ++----------------------- 3 files changed, 299 insertions(+), 287 deletions(-) create mode 100644 drivers/raw/ioat/rte_idxd_rawdev_fns.h diff --git a/drivers/raw/ioat/meson.build b/drivers/raw/ioat/meson.build index 6382a826e7..0e81cb5951 100644 --- a/drivers/raw/ioat/meson.build +++ b/drivers/raw/ioat/meson.build @@ -13,5 +13,6 @@ sources = files( deps += ['bus_pci', 'mbuf', 'rawdev'] headers = files( 'rte_ioat_rawdev.h', + 'rte_idxd_rawdev_fns.h', 'rte_ioat_rawdev_fns.h', ) diff --git a/drivers/raw/ioat/rte_idxd_rawdev_fns.h b/drivers/raw/ioat/rte_idxd_rawdev_fns.h new file mode 100644 index 0000000000..c2a12ebef0 --- /dev/null +++ b/drivers/raw/ioat/rte_idxd_rawdev_fns.h @@ -0,0 +1,275 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ +#ifndef _RTE_IDXD_RAWDEV_FNS_H_ +#define _RTE_IDXD_RAWDEV_FNS_H_ + +#include + +/* + * Defines used in the data path for interacting with hardware. + */ +#define IDXD_CMD_OP_SHIFT 24 +enum rte_idxd_ops { + idxd_op_nop = 0, + idxd_op_batch, + idxd_op_drain, + idxd_op_memmove, + idxd_op_fill +}; + +#define IDXD_FLAG_FENCE (1 << 0) +#define IDXD_FLAG_COMPLETION_ADDR_VALID (1 << 2) +#define IDXD_FLAG_REQUEST_COMPLETION (1 << 3) +#define IDXD_FLAG_CACHE_CONTROL (1 << 8) + +#define IOAT_COMP_UPDATE_SHIFT 3 +#define IOAT_CMD_OP_SHIFT 24 +enum rte_ioat_ops { + ioat_op_copy = 0, /* Standard DMA Operation */ + ioat_op_fill /* Block Fill */ +}; + +/** + * Hardware descriptor used by DSA hardware, for both bursts and + * for individual operations. + */ +struct rte_idxd_hw_desc { + uint32_t pasid; + uint32_t op_flags; + rte_iova_t completion; + + RTE_STD_C11 + union { + rte_iova_t src; /* source address for copy ops etc. */ + rte_iova_t desc_addr; /* descriptor pointer for batch */ + }; + rte_iova_t dst; + + uint32_t size; /* length of data for op, or batch size */ + + uint16_t intr_handle; /* completion interrupt handle */ + + /* remaining 26 bytes are reserved */ + uint16_t __reserved[13]; +} __rte_aligned(64); + +/** + * Completion record structure written back by DSA + */ +struct rte_idxd_completion { + uint8_t status; + uint8_t result; + /* 16-bits pad here */ + uint32_t completed_size; /* data length, or descriptors for batch */ + + rte_iova_t fault_address; + uint32_t invalid_flags; +} __rte_aligned(32); + +#define BATCH_SIZE 64 + +/** + * Structure used inside the driver for building up and submitting + * a batch of operations to the DSA hardware. + */ +struct rte_idxd_desc_batch { + struct rte_idxd_completion comp; /* the completion record for batch */ + + uint16_t submitted; + uint16_t op_count; + uint16_t hdl_end; + + struct rte_idxd_hw_desc batch_desc; + + /* batches must always have 2 descriptors, so put a null at the start */ + struct rte_idxd_hw_desc null_desc; + struct rte_idxd_hw_desc ops[BATCH_SIZE]; +}; + +/** + * structure used to save the "handles" provided by the user to be + * returned to the user on job completion. + */ +struct rte_idxd_user_hdl { + uint64_t src; + uint64_t dst; +}; + +/** + * @internal + * Structure representing an IDXD device instance + */ +struct rte_idxd_rawdev { + enum rte_ioat_dev_type type; + struct rte_ioat_xstats xstats; + + void *portal; /* address to write the batch descriptor */ + + /* counters to track the batches and the individual op handles */ + uint16_t batch_ring_sz; /* size of batch ring */ + uint16_t hdl_ring_sz; /* size of the user hdl ring */ + + uint16_t next_batch; /* where we write descriptor ops */ + uint16_t next_completed; /* batch where we read completions */ + uint16_t next_ret_hdl; /* the next user hdl to return */ + uint16_t last_completed_hdl; /* the last user hdl that has completed */ + uint16_t next_free_hdl; /* where the handle for next op will go */ + uint16_t hdls_disable; /* disable tracking completion handles */ + + struct rte_idxd_user_hdl *hdl_ring; + struct rte_idxd_desc_batch *batch_ring; +}; + +static __rte_always_inline int +__idxd_write_desc(int dev_id, const struct rte_idxd_hw_desc *desc, + const struct rte_idxd_user_hdl *hdl) +{ + struct rte_idxd_rawdev *idxd = + (struct rte_idxd_rawdev *)rte_rawdevs[dev_id].dev_private; + struct rte_idxd_desc_batch *b = &idxd->batch_ring[idxd->next_batch]; + + /* check for room in the handle ring */ + if (((idxd->next_free_hdl + 1) & (idxd->hdl_ring_sz - 1)) == idxd->next_ret_hdl) + goto failed; + + /* check for space in current batch */ + if (b->op_count >= BATCH_SIZE) + goto failed; + + /* check that we can actually use the current batch */ + if (b->submitted) + goto failed; + + /* write the descriptor */ + b->ops[b->op_count++] = *desc; + + /* store the completion details */ + if (!idxd->hdls_disable) + idxd->hdl_ring[idxd->next_free_hdl] = *hdl; + if (++idxd->next_free_hdl == idxd->hdl_ring_sz) + idxd->next_free_hdl = 0; + + idxd->xstats.enqueued++; + return 1; + +failed: + idxd->xstats.enqueue_failed++; + rte_errno = ENOSPC; + return 0; +} + +static __rte_always_inline int +__idxd_enqueue_fill(int dev_id, uint64_t pattern, rte_iova_t dst, + unsigned int length, uintptr_t dst_hdl) +{ + const struct rte_idxd_hw_desc desc = { + .op_flags = (idxd_op_fill << IDXD_CMD_OP_SHIFT) | + IDXD_FLAG_CACHE_CONTROL, + .src = pattern, + .dst = dst, + .size = length + }; + const struct rte_idxd_user_hdl hdl = { + .dst = dst_hdl + }; + return __idxd_write_desc(dev_id, &desc, &hdl); +} + +static __rte_always_inline int +__idxd_enqueue_copy(int dev_id, rte_iova_t src, rte_iova_t dst, + unsigned int length, uintptr_t src_hdl, uintptr_t dst_hdl) +{ + const struct rte_idxd_hw_desc desc = { + .op_flags = (idxd_op_memmove << IDXD_CMD_OP_SHIFT) | + IDXD_FLAG_CACHE_CONTROL, + .src = src, + .dst = dst, + .size = length + }; + const struct rte_idxd_user_hdl hdl = { + .src = src_hdl, + .dst = dst_hdl + }; + return __idxd_write_desc(dev_id, &desc, &hdl); +} + +static __rte_always_inline int +__idxd_fence(int dev_id) +{ + static const struct rte_idxd_hw_desc fence = { + .op_flags = IDXD_FLAG_FENCE + }; + static const struct rte_idxd_user_hdl null_hdl; + return __idxd_write_desc(dev_id, &fence, &null_hdl); +} + +static __rte_always_inline void +__idxd_movdir64b(volatile void *dst, const void *src) +{ + asm volatile (".byte 0x66, 0x0f, 0x38, 0xf8, 0x02" + : + : "a" (dst), "d" (src)); +} + +static __rte_always_inline int +__idxd_perform_ops(int dev_id) +{ + struct rte_idxd_rawdev *idxd = + (struct rte_idxd_rawdev *)rte_rawdevs[dev_id].dev_private; + struct rte_idxd_desc_batch *b = &idxd->batch_ring[idxd->next_batch]; + + if (b->submitted || b->op_count == 0) + return 0; + b->hdl_end = idxd->next_free_hdl; + b->comp.status = 0; + b->submitted = 1; + b->batch_desc.size = b->op_count + 1; + __idxd_movdir64b(idxd->portal, &b->batch_desc); + + if (++idxd->next_batch == idxd->batch_ring_sz) + idxd->next_batch = 0; + idxd->xstats.started = idxd->xstats.enqueued; + return 0; +} + +static __rte_always_inline int +__idxd_completed_ops(int dev_id, uint8_t max_ops, + uintptr_t *src_hdls, uintptr_t *dst_hdls) +{ + struct rte_idxd_rawdev *idxd = + (struct rte_idxd_rawdev *)rte_rawdevs[dev_id].dev_private; + struct rte_idxd_desc_batch *b = &idxd->batch_ring[idxd->next_completed]; + uint16_t h_idx = idxd->next_ret_hdl; + int n = 0; + + while (b->submitted && b->comp.status != 0) { + idxd->last_completed_hdl = b->hdl_end; + b->submitted = 0; + b->op_count = 0; + if (++idxd->next_completed == idxd->batch_ring_sz) + idxd->next_completed = 0; + b = &idxd->batch_ring[idxd->next_completed]; + } + + if (!idxd->hdls_disable) + for (n = 0; n < max_ops && h_idx != idxd->last_completed_hdl; n++) { + src_hdls[n] = idxd->hdl_ring[h_idx].src; + dst_hdls[n] = idxd->hdl_ring[h_idx].dst; + if (++h_idx == idxd->hdl_ring_sz) + h_idx = 0; + } + else + while (h_idx != idxd->last_completed_hdl) { + n++; + if (++h_idx == idxd->hdl_ring_sz) + h_idx = 0; + } + + idxd->next_ret_hdl = h_idx; + + idxd->xstats.completed += n; + return n; +} + +#endif diff --git a/drivers/raw/ioat/rte_ioat_rawdev_fns.h b/drivers/raw/ioat/rte_ioat_rawdev_fns.h index 477c1b7b41..598852b1fa 100644 --- a/drivers/raw/ioat/rte_ioat_rawdev_fns.h +++ b/drivers/raw/ioat/rte_ioat_rawdev_fns.h @@ -9,6 +9,29 @@ #include #include +/** + * @internal + * Identify the data path to use. + * Must be first field of rte_ioat_rawdev and rte_idxd_rawdev structs + */ +enum rte_ioat_dev_type { + RTE_IOAT_DEV, + RTE_IDXD_DEV, +}; + +/** + * @internal + * some statistics for tracking, if added/changed update xstats fns + */ +struct rte_ioat_xstats { + uint64_t enqueue_failed; + uint64_t enqueued; + uint64_t started; + uint64_t completed; +}; + +#include "rte_idxd_rawdev_fns.h" + /** * @internal * Structure representing a device descriptor @@ -39,27 +62,6 @@ struct rte_ioat_generic_hw_desc { uint64_t op_specific[4]; }; -/** - * @internal - * Identify the data path to use. - * Must be first field of rte_ioat_rawdev and rte_idxd_rawdev structs - */ -enum rte_ioat_dev_type { - RTE_IOAT_DEV, - RTE_IDXD_DEV, -}; - -/** - * @internal - * some statistics for tracking, if added/changed update xstats fns - */ -struct rte_ioat_xstats { - uint64_t enqueue_failed; - uint64_t enqueued; - uint64_t started; - uint64_t completed; -}; - /** * @internal * Structure representing an IOAT device instance @@ -98,121 +100,6 @@ struct rte_ioat_rawdev { #define RTE_IOAT_CHANSTS_HALTED 0x3 #define RTE_IOAT_CHANSTS_ARMED 0x4 -/* - * Defines used in the data path for interacting with hardware. - */ -#define IDXD_CMD_OP_SHIFT 24 -enum rte_idxd_ops { - idxd_op_nop = 0, - idxd_op_batch, - idxd_op_drain, - idxd_op_memmove, - idxd_op_fill -}; - -#define IDXD_FLAG_FENCE (1 << 0) -#define IDXD_FLAG_COMPLETION_ADDR_VALID (1 << 2) -#define IDXD_FLAG_REQUEST_COMPLETION (1 << 3) -#define IDXD_FLAG_CACHE_CONTROL (1 << 8) - -#define IOAT_COMP_UPDATE_SHIFT 3 -#define IOAT_CMD_OP_SHIFT 24 -enum rte_ioat_ops { - ioat_op_copy = 0, /* Standard DMA Operation */ - ioat_op_fill /* Block Fill */ -}; - -/** - * Hardware descriptor used by DSA hardware, for both bursts and - * for individual operations. - */ -struct rte_idxd_hw_desc { - uint32_t pasid; - uint32_t op_flags; - rte_iova_t completion; - - RTE_STD_C11 - union { - rte_iova_t src; /* source address for copy ops etc. */ - rte_iova_t desc_addr; /* descriptor pointer for batch */ - }; - rte_iova_t dst; - - uint32_t size; /* length of data for op, or batch size */ - - uint16_t intr_handle; /* completion interrupt handle */ - - /* remaining 26 bytes are reserved */ - uint16_t __reserved[13]; -} __rte_aligned(64); - -/** - * Completion record structure written back by DSA - */ -struct rte_idxd_completion { - uint8_t status; - uint8_t result; - /* 16-bits pad here */ - uint32_t completed_size; /* data length, or descriptors for batch */ - - rte_iova_t fault_address; - uint32_t invalid_flags; -} __rte_aligned(32); - -#define BATCH_SIZE 64 - -/** - * Structure used inside the driver for building up and submitting - * a batch of operations to the DSA hardware. - */ -struct rte_idxd_desc_batch { - struct rte_idxd_completion comp; /* the completion record for batch */ - - uint16_t submitted; - uint16_t op_count; - uint16_t hdl_end; - - struct rte_idxd_hw_desc batch_desc; - - /* batches must always have 2 descriptors, so put a null at the start */ - struct rte_idxd_hw_desc null_desc; - struct rte_idxd_hw_desc ops[BATCH_SIZE]; -}; - -/** - * structure used to save the "handles" provided by the user to be - * returned to the user on job completion. - */ -struct rte_idxd_user_hdl { - uint64_t src; - uint64_t dst; -}; - -/** - * @internal - * Structure representing an IDXD device instance - */ -struct rte_idxd_rawdev { - enum rte_ioat_dev_type type; - struct rte_ioat_xstats xstats; - - void *portal; /* address to write the batch descriptor */ - - /* counters to track the batches and the individual op handles */ - uint16_t batch_ring_sz; /* size of batch ring */ - uint16_t hdl_ring_sz; /* size of the user hdl ring */ - - uint16_t next_batch; /* where we write descriptor ops */ - uint16_t next_completed; /* batch where we read completions */ - uint16_t next_ret_hdl; /* the next user hdl to return */ - uint16_t last_completed_hdl; /* the last user hdl that has completed */ - uint16_t next_free_hdl; /* where the handle for next op will go */ - uint16_t hdls_disable; /* disable tracking completion handles */ - - struct rte_idxd_user_hdl *hdl_ring; - struct rte_idxd_desc_batch *batch_ring; -}; - static __rte_always_inline int __ioat_write_desc(int dev_id, uint32_t op, uint64_t src, phys_addr_t dst, unsigned int length, uintptr_t src_hdl, uintptr_t dst_hdl) @@ -373,157 +260,6 @@ __ioat_completed_ops(int dev_id, uint8_t max_copies, return count; } -static __rte_always_inline int -__idxd_write_desc(int dev_id, const struct rte_idxd_hw_desc *desc, - const struct rte_idxd_user_hdl *hdl) -{ - struct rte_idxd_rawdev *idxd = - (struct rte_idxd_rawdev *)rte_rawdevs[dev_id].dev_private; - struct rte_idxd_desc_batch *b = &idxd->batch_ring[idxd->next_batch]; - - /* check for room in the handle ring */ - if (((idxd->next_free_hdl + 1) & (idxd->hdl_ring_sz - 1)) == idxd->next_ret_hdl) - goto failed; - - /* check for space in current batch */ - if (b->op_count >= BATCH_SIZE) - goto failed; - - /* check that we can actually use the current batch */ - if (b->submitted) - goto failed; - - /* write the descriptor */ - b->ops[b->op_count++] = *desc; - - /* store the completion details */ - if (!idxd->hdls_disable) - idxd->hdl_ring[idxd->next_free_hdl] = *hdl; - if (++idxd->next_free_hdl == idxd->hdl_ring_sz) - idxd->next_free_hdl = 0; - - idxd->xstats.enqueued++; - return 1; - -failed: - idxd->xstats.enqueue_failed++; - rte_errno = ENOSPC; - return 0; -} - -static __rte_always_inline int -__idxd_enqueue_fill(int dev_id, uint64_t pattern, rte_iova_t dst, - unsigned int length, uintptr_t dst_hdl) -{ - const struct rte_idxd_hw_desc desc = { - .op_flags = (idxd_op_fill << IDXD_CMD_OP_SHIFT) | - IDXD_FLAG_CACHE_CONTROL, - .src = pattern, - .dst = dst, - .size = length - }; - const struct rte_idxd_user_hdl hdl = { - .dst = dst_hdl - }; - return __idxd_write_desc(dev_id, &desc, &hdl); -} - -static __rte_always_inline int -__idxd_enqueue_copy(int dev_id, rte_iova_t src, rte_iova_t dst, - unsigned int length, uintptr_t src_hdl, uintptr_t dst_hdl) -{ - const struct rte_idxd_hw_desc desc = { - .op_flags = (idxd_op_memmove << IDXD_CMD_OP_SHIFT) | - IDXD_FLAG_CACHE_CONTROL, - .src = src, - .dst = dst, - .size = length - }; - const struct rte_idxd_user_hdl hdl = { - .src = src_hdl, - .dst = dst_hdl - }; - return __idxd_write_desc(dev_id, &desc, &hdl); -} - -static __rte_always_inline int -__idxd_fence(int dev_id) -{ - static const struct rte_idxd_hw_desc fence = { - .op_flags = IDXD_FLAG_FENCE - }; - static const struct rte_idxd_user_hdl null_hdl; - return __idxd_write_desc(dev_id, &fence, &null_hdl); -} - -static __rte_always_inline void -__idxd_movdir64b(volatile void *dst, const void *src) -{ - asm volatile (".byte 0x66, 0x0f, 0x38, 0xf8, 0x02" - : - : "a" (dst), "d" (src)); -} - -static __rte_always_inline int -__idxd_perform_ops(int dev_id) -{ - struct rte_idxd_rawdev *idxd = - (struct rte_idxd_rawdev *)rte_rawdevs[dev_id].dev_private; - struct rte_idxd_desc_batch *b = &idxd->batch_ring[idxd->next_batch]; - - if (b->submitted || b->op_count == 0) - return 0; - b->hdl_end = idxd->next_free_hdl; - b->comp.status = 0; - b->submitted = 1; - b->batch_desc.size = b->op_count + 1; - __idxd_movdir64b(idxd->portal, &b->batch_desc); - - if (++idxd->next_batch == idxd->batch_ring_sz) - idxd->next_batch = 0; - idxd->xstats.started = idxd->xstats.enqueued; - return 0; -} - -static __rte_always_inline int -__idxd_completed_ops(int dev_id, uint8_t max_ops, - uintptr_t *src_hdls, uintptr_t *dst_hdls) -{ - struct rte_idxd_rawdev *idxd = - (struct rte_idxd_rawdev *)rte_rawdevs[dev_id].dev_private; - struct rte_idxd_desc_batch *b = &idxd->batch_ring[idxd->next_completed]; - uint16_t h_idx = idxd->next_ret_hdl; - int n = 0; - - while (b->submitted && b->comp.status != 0) { - idxd->last_completed_hdl = b->hdl_end; - b->submitted = 0; - b->op_count = 0; - if (++idxd->next_completed == idxd->batch_ring_sz) - idxd->next_completed = 0; - b = &idxd->batch_ring[idxd->next_completed]; - } - - if (!idxd->hdls_disable) - for (n = 0; n < max_ops && h_idx != idxd->last_completed_hdl; n++) { - src_hdls[n] = idxd->hdl_ring[h_idx].src; - dst_hdls[n] = idxd->hdl_ring[h_idx].dst; - if (++h_idx == idxd->hdl_ring_sz) - h_idx = 0; - } - else - while (h_idx != idxd->last_completed_hdl) { - n++; - if (++h_idx == idxd->hdl_ring_sz) - h_idx = 0; - } - - idxd->next_ret_hdl = h_idx; - - idxd->xstats.completed += n; - return n; -} - static inline int rte_ioat_enqueue_fill(int dev_id, uint64_t pattern, phys_addr_t dst, unsigned int len, uintptr_t dst_hdl) -- 2.30.2