[PATCH 2/5] common/mlx5: add CQE validity iteration count

[Alexander Kozyrev <akozyrev@nvidia.com>]

The validity iteration count replaces the functionality of the owner bit
in terms of indicating that a new CQE was written to buffer.
On iteration=k on the CQ buffer, only entries with the iteration_count=k
should be treated as new CQEs or mini CQE arrays.
The validity iteration count is used when the Enhanced CQE compression
is selected. Add this CQE field and the method to check it.

Signed-off-by: Alexander Kozyrev <akozyrev@nvidia.com>
---
 drivers/common/mlx5/mlx5_common.h      | 57 ++++++++++++++++++++++----
 drivers/common/mlx5/mlx5_common_devx.c |  4 +-
 drivers/common/mlx5/mlx5_prm.h         | 12 ++++--
 3 files changed, 62 insertions(+), 11 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index f8d07d6c6b..9fb85ddefb 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -180,7 +180,26 @@ enum mlx5_cqe_status {
 };
 
 /**
- * Check whether CQE is valid.
+ * Check whether CQE has an error opcode.
+ *
+ * @param op_code
+ *   Opcode to check.
+ *
+ * @return
+ *   The CQE status.
+ */
+static __rte_always_inline enum mlx5_cqe_status
+check_cqe_error(const uint8_t op_code)
+{
+	rte_io_rmb();
+	if (unlikely(op_code == MLX5_CQE_RESP_ERR ||
+		     op_code == MLX5_CQE_REQ_ERR))
+		return MLX5_CQE_STATUS_ERR;
+	return MLX5_CQE_STATUS_SW_OWN;
+}
+
+/**
+ * Check whether CQE is valid using owner bit.
  *
  * @param cqe
  *   Pointer to CQE.
@@ -201,13 +220,37 @@ check_cqe(volatile struct mlx5_cqe *cqe, const uint16_t cqes_n,
 	const uint8_t op_owner = MLX5_CQE_OWNER(op_own);
 	const uint8_t op_code = MLX5_CQE_OPCODE(op_own);
 
-	if (unlikely((op_owner != (!!(idx))) || (op_code == MLX5_CQE_INVALID)))
+	if (unlikely((op_owner != (!!(idx))) ||
+		     (op_code == MLX5_CQE_INVALID)))
 		return MLX5_CQE_STATUS_HW_OWN;
-	rte_io_rmb();
-	if (unlikely(op_code == MLX5_CQE_RESP_ERR ||
-		     op_code == MLX5_CQE_REQ_ERR))
-		return MLX5_CQE_STATUS_ERR;
-	return MLX5_CQE_STATUS_SW_OWN;
+	return check_cqe_error(op_code);
+}
+
+/**
+ * Check whether CQE is valid using validity iteration count.
+ *
+ * @param cqe
+ *   Pointer to CQE.
+ * @param cqes_n
+ *   Log 2 of completion queue size.
+ * @param ci
+ *   Consumer index.
+ *
+ * @return
+ *   The CQE status.
+ */
+static __rte_always_inline enum mlx5_cqe_status
+check_cqe_iteration(volatile struct mlx5_cqe *cqe, const uint16_t cqes_n,
+		    const uint32_t ci)
+{
+	const uint8_t op_own = cqe->op_own;
+	const uint8_t op_code = MLX5_CQE_OPCODE(op_own);
+	const uint8_t vic = ci >> cqes_n;
+
+	if (unlikely((cqe->validity_iteration_count != vic) ||
+		     (op_code == MLX5_CQE_INVALID)))
+		return MLX5_CQE_STATUS_HW_OWN;
+	return check_cqe_error(op_code);
 }
 
 /*
diff --git a/drivers/common/mlx5/mlx5_common_devx.c b/drivers/common/mlx5/mlx5_common_devx.c
index 5f53996b72..431d8361ce 100644
--- a/drivers/common/mlx5/mlx5_common_devx.c
+++ b/drivers/common/mlx5/mlx5_common_devx.c
@@ -41,8 +41,10 @@ mlx5_cq_init(struct mlx5_devx_cq *cq_obj, uint16_t cq_size)
 	volatile struct mlx5_cqe *cqe = cq_obj->cqes;
 	uint16_t i;
 
-	for (i = 0; i < cq_size; i++, cqe++)
+	for (i = 0; i < cq_size; i++, cqe++) {
 		cqe->op_own = (MLX5_CQE_INVALID << 4) | MLX5_CQE_OWNER_MASK;
+		cqe->validity_iteration_count = MLX5_CQE_VIC_INIT;
+	}
 }
 
 /**
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index aa291f19a6..a52feba7e4 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -26,12 +26,18 @@
 /* Get CQE opcode. */
 #define MLX5_CQE_OPCODE(op_own) (((op_own) & 0xf0) >> 4)
 
+/* Get CQE number of mini CQEs. */
+#define MLX5_CQE_NUM_MINIS(op_own) (((op_own) & 0xf0) >> 4)
+
 /* Get CQE solicited event. */
 #define MLX5_CQE_SE(op_own) (((op_own) >> 1) & 1)
 
 /* Invalidate a CQE. */
 #define MLX5_CQE_INVALIDATE (MLX5_CQE_INVALID << 4)
 
+/* Initialize CQE validity iteration count. */
+#define MLX5_CQE_VIC_INIT 0xffu
+
 /* Hardware index widths. */
 #define MLX5_CQ_INDEX_WIDTH 24
 #define MLX5_WQ_INDEX_WIDTH 16
@@ -442,7 +448,7 @@ struct mlx5_cqe {
 	uint64_t timestamp;
 	uint32_t sop_drop_qpn;
 	uint16_t wqe_counter;
-	uint8_t rsvd5;
+	uint8_t validity_iteration_count;
 	uint8_t op_own;
 };
 
@@ -450,7 +456,7 @@ struct mlx5_cqe_ts {
 	uint64_t timestamp;
 	uint32_t sop_drop_qpn;
 	uint16_t wqe_counter;
-	uint8_t rsvd5;
+	uint8_t validity_iteration_count;
 	uint8_t op_own;
 };
 
@@ -5041,8 +5047,8 @@ struct mlx5_mini_cqe8 {
 		};
 		struct {
 			uint16_t wqe_counter;
+			uint8_t  validity_iteration_count;
 			uint8_t  s_wqe_opcode;
-			uint8_t  reserved;
 		} s_wqe_info;
 	};
 	union {
-- 
2.18.2