[PATCH v5 06/21] pdcp: add pre and post process for UL

[Anoob Joseph <anoobj@marvell.com>]

Add routines to perform pre & post processing based on the type of
entity. To avoid checks in datapath, there are different function
pointers registered based on the following,
1. Control plane v/s user plane
2. 12 bit v/s 18 bit SN

For control plane only 12 bit SN need to be supported (as per PDCP
specification).

Signed-off-by: Anoob Joseph <anoobj@marvell.com>
Signed-off-by: Kiran Kumar K <kirankumark@marvell.com>
Signed-off-by: Volodymyr Fialko <vfialko@marvell.com>
Acked-by: Akhil Goyal <gakhil@marvell.com>
---
 lib/pdcp/pdcp_entity.h  |  24 +++
 lib/pdcp/pdcp_process.c | 334 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 358 insertions(+)

diff --git a/lib/pdcp/pdcp_entity.h b/lib/pdcp/pdcp_entity.h
index 000297588f..23628ebad4 100644
--- a/lib/pdcp/pdcp_entity.h
+++ b/lib/pdcp/pdcp_entity.h
@@ -92,22 +92,46 @@ pdcp_hdr_size_get(enum rte_security_pdcp_sn_size sn_size)
 	return RTE_ALIGN_MUL_CEIL(sn_size, 8) / 8;
 }
 
+static inline uint32_t
+pdcp_window_size_get(enum rte_security_pdcp_sn_size sn_size)
+{
+	return 1 << (sn_size - 1);
+}
+
 static inline uint32_t
 pdcp_sn_mask_get(enum rte_security_pdcp_sn_size sn_size)
 {
 	return (1 << sn_size) - 1;
 }
 
+static inline uint32_t
+pdcp_sn_from_count_get(uint32_t count, enum rte_security_pdcp_sn_size sn_size)
+{
+	return (count & pdcp_sn_mask_get(sn_size));
+}
+
 static inline uint32_t
 pdcp_hfn_mask_get(enum rte_security_pdcp_sn_size sn_size)
 {
 	return ~pdcp_sn_mask_get(sn_size);
 }
 
+static inline uint32_t
+pdcp_hfn_from_count_get(uint32_t count, enum rte_security_pdcp_sn_size sn_size)
+{
+	return (count & pdcp_hfn_mask_get(sn_size)) >> sn_size;
+}
+
 static inline uint32_t
 pdcp_count_from_hfn_sn_get(uint32_t hfn, uint32_t sn, enum rte_security_pdcp_sn_size sn_size)
 {
 	return (((hfn << sn_size) & pdcp_hfn_mask_get(sn_size)) | (sn & pdcp_sn_mask_get(sn_size)));
 }
 
+static inline uint32_t
+pdcp_hfn_max(enum rte_security_pdcp_sn_size sn_size)
+{
+	return (1 << (32 - sn_size)) - 1;
+}
+
 #endif /* PDCP_ENTITY_H */
diff --git a/lib/pdcp/pdcp_process.c b/lib/pdcp/pdcp_process.c
index 79f5dce5db..9b7de39db6 100644
--- a/lib/pdcp/pdcp_process.c
+++ b/lib/pdcp/pdcp_process.c
@@ -36,6 +36,336 @@ pdcp_crypto_xfrm_get(const struct rte_pdcp_entity_conf *conf, struct rte_crypto_
 	return 0;
 }
 
+static inline void
+cop_prepare(const struct entity_priv *en_priv, struct rte_mbuf *mb, struct rte_crypto_op *cop,
+	    uint8_t data_offset, uint32_t count, const bool is_auth)
+{
+	const struct rte_crypto_op cop_init = {
+		.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED,
+		.sess_type = RTE_CRYPTO_OP_WITH_SESSION,
+	};
+	struct rte_crypto_sym_op *op;
+	uint32_t pkt_len;
+
+	const uint8_t cipher_shift = 3 * en_priv->flags.is_cipher_in_bits;
+	const uint8_t auth_shift = 3 * en_priv->flags.is_auth_in_bits;
+
+	op = cop->sym;
+	cop->raw = cop_init.raw;
+	op->m_src = mb;
+	op->m_dst = mb;
+
+	/* Set IV */
+	en_priv->iv_gen(cop, en_priv, count);
+
+	/* Prepare op */
+	pkt_len = rte_pktmbuf_pkt_len(mb);
+	op->cipher.data.offset = data_offset << cipher_shift;
+	op->cipher.data.length = (pkt_len - data_offset) << cipher_shift;
+
+	if (is_auth) {
+		op->auth.data.offset = 0;
+		op->auth.data.length = (pkt_len - RTE_PDCP_MAC_I_LEN) << auth_shift;
+		op->auth.digest.data = rte_pktmbuf_mtod_offset(mb, uint8_t *,
+							       (pkt_len - RTE_PDCP_MAC_I_LEN));
+	}
+
+	__rte_crypto_sym_op_attach_sym_session(op, en_priv->crypto_sess);
+}
+
+static inline bool
+pdcp_pre_process_uplane_sn_12_ul_set_sn(struct entity_priv *en_priv, struct rte_mbuf *mb,
+					uint32_t *count)
+{
+	struct rte_pdcp_up_data_pdu_sn_12_hdr *pdu_hdr;
+	const uint8_t hdr_sz = en_priv->hdr_sz;
+	uint32_t sn;
+
+	/* Prepend PDU header */
+	pdu_hdr = (struct rte_pdcp_up_data_pdu_sn_12_hdr *)rte_pktmbuf_prepend(mb, hdr_sz);
+	if (unlikely(pdu_hdr == NULL))
+		return false;
+
+	/* Update sequence num in the PDU header */
+	*count = en_priv->state.tx_next++;
+	sn = pdcp_sn_from_count_get(*count, RTE_SECURITY_PDCP_SN_SIZE_12);
+
+	pdu_hdr->d_c = RTE_PDCP_PDU_TYPE_DATA;
+	pdu_hdr->sn_11_8 = ((sn & 0xf00) >> 8);
+	pdu_hdr->sn_7_0 = (sn & 0xff);
+	pdu_hdr->r = 0;
+	return true;
+}
+
+static uint16_t
+pdcp_pre_process_uplane_sn_12_ul(const struct rte_pdcp_entity *entity, struct rte_mbuf *in_mb[],
+				 struct rte_crypto_op *cop[], uint16_t num, uint16_t *nb_err_ret)
+{
+	struct entity_priv *en_priv = entity_priv_get(entity);
+	uint16_t nb_cop, nb_prep = 0, nb_err = 0;
+	struct rte_mbuf *mb;
+	uint32_t count;
+	uint8_t *mac_i;
+	int i;
+
+	const uint8_t data_offset = en_priv->hdr_sz + en_priv->aad_sz;
+	const int is_null_auth = en_priv->flags.is_null_auth;
+
+	nb_cop = rte_crypto_op_bulk_alloc(en_priv->cop_pool, RTE_CRYPTO_OP_TYPE_SYMMETRIC, cop,
+					  num);
+
+	if (en_priv->flags.is_authenticated) {
+		for (i = 0; i < nb_cop; i++) {
+			mb = in_mb[i];
+			mac_i = (uint8_t *)rte_pktmbuf_append(mb, RTE_PDCP_MAC_I_LEN);
+			if (unlikely(mac_i == NULL)) {
+				in_mb[nb_err++] = mb;
+				continue;
+			}
+
+			/* Clear MAC-I field for NULL auth */
+			if (is_null_auth)
+				memset(mac_i, 0, RTE_PDCP_MAC_I_LEN);
+
+			if (unlikely(!pdcp_pre_process_uplane_sn_12_ul_set_sn(en_priv, mb,
+									      &count))) {
+				in_mb[nb_err++] = mb;
+				continue;
+			}
+
+			cop_prepare(en_priv, mb, cop[nb_prep++], data_offset, count, true);
+		}
+	} else {
+		for (i = 0; i < nb_cop; i++) {
+			mb = in_mb[i];
+			if (unlikely(!pdcp_pre_process_uplane_sn_12_ul_set_sn(en_priv, mb,
+									      &count))) {
+				in_mb[nb_err++] = mb;
+				continue;
+			}
+
+			cop_prepare(en_priv, mb, cop[nb_prep++], data_offset, count, false);
+		}
+	}
+
+	if (unlikely(nb_err))
+		/* Using mempool API since crypto API is not providing bulk free */
+		rte_mempool_put_bulk(en_priv->cop_pool, (void *)&cop[nb_prep], nb_cop - nb_prep);
+
+	*nb_err_ret = num - nb_prep;
+
+	return nb_prep;
+}
+
+static inline bool
+pdcp_pre_process_uplane_sn_18_ul_set_sn(struct entity_priv *en_priv, struct rte_mbuf *mb,
+					uint32_t *count)
+{
+	struct rte_pdcp_up_data_pdu_sn_18_hdr *pdu_hdr;
+	const uint8_t hdr_sz = en_priv->hdr_sz;
+	uint32_t sn;
+
+	/* Prepend PDU header */
+	pdu_hdr = (struct rte_pdcp_up_data_pdu_sn_18_hdr *)rte_pktmbuf_prepend(mb, hdr_sz);
+	if (unlikely(pdu_hdr == NULL))
+		return false;
+
+	/* Update sequence num in the PDU header */
+	*count = en_priv->state.tx_next++;
+	sn = pdcp_sn_from_count_get(*count, RTE_SECURITY_PDCP_SN_SIZE_18);
+
+	pdu_hdr->d_c = RTE_PDCP_PDU_TYPE_DATA;
+	pdu_hdr->sn_17_16 = ((sn & 0x30000) >> 16);
+	pdu_hdr->sn_15_8 = ((sn & 0xff00) >> 8);
+	pdu_hdr->sn_7_0 = (sn & 0xff);
+	pdu_hdr->r = 0;
+
+	return true;
+}
+
+static inline uint16_t
+pdcp_pre_process_uplane_sn_18_ul(const struct rte_pdcp_entity *entity, struct rte_mbuf *in_mb[],
+				 struct rte_crypto_op *cop[], uint16_t num, uint16_t *nb_err_ret)
+{
+	struct entity_priv *en_priv = entity_priv_get(entity);
+	uint16_t nb_cop, nb_prep = 0, nb_err = 0;
+	struct rte_mbuf *mb;
+	uint32_t count;
+	uint8_t *mac_i;
+	int i;
+
+	const uint8_t data_offset = en_priv->hdr_sz + en_priv->aad_sz;
+	const int is_null_auth = en_priv->flags.is_null_auth;
+
+	nb_cop = rte_crypto_op_bulk_alloc(en_priv->cop_pool, RTE_CRYPTO_OP_TYPE_SYMMETRIC, cop,
+					  num);
+
+	if (en_priv->flags.is_authenticated) {
+		for (i = 0; i < nb_cop; i++) {
+			mb = in_mb[i];
+			mac_i = (uint8_t *)rte_pktmbuf_append(mb, RTE_PDCP_MAC_I_LEN);
+			if (unlikely(mac_i == NULL)) {
+				in_mb[nb_err++] = mb;
+				continue;
+			}
+
+			/* Clear MAC-I field for NULL auth */
+			if (is_null_auth)
+				memset(mac_i, 0, RTE_PDCP_MAC_I_LEN);
+
+			if (unlikely(!pdcp_pre_process_uplane_sn_18_ul_set_sn(en_priv, mb,
+									      &count))) {
+				in_mb[nb_err++] = mb;
+				continue;
+			}
+
+			cop_prepare(en_priv, mb, cop[nb_prep++], data_offset, count, true);
+		}
+	} else {
+		for (i = 0; i < nb_cop; i++) {
+			mb = in_mb[i];
+			if (unlikely(!pdcp_pre_process_uplane_sn_18_ul_set_sn(en_priv, mb,
+									      &count))) {
+
+				in_mb[nb_err++] = mb;
+				continue;
+			}
+
+			cop_prepare(en_priv, mb, cop[nb_prep++], data_offset, count, false);
+		}
+	}
+
+	if (unlikely(nb_err))
+		/* Using mempool API since crypto API is not providing bulk free */
+		rte_mempool_put_bulk(en_priv->cop_pool, (void *)&cop[nb_prep], nb_cop - nb_prep);
+
+	*nb_err_ret = num - nb_prep;
+
+	return nb_prep;
+}
+
+static uint16_t
+pdcp_pre_process_cplane_sn_12_ul(const struct rte_pdcp_entity *entity, struct rte_mbuf *in_mb[],
+				 struct rte_crypto_op *cop[], uint16_t num, uint16_t *nb_err_ret)
+{
+	struct entity_priv *en_priv = entity_priv_get(entity);
+	struct rte_pdcp_cp_data_pdu_sn_12_hdr *pdu_hdr;
+	uint16_t nb_cop, nb_prep = 0, nb_err = 0;
+	struct rte_mbuf *mb;
+	uint32_t count, sn;
+	uint8_t *mac_i;
+	int i;
+
+	const uint8_t hdr_sz = en_priv->hdr_sz;
+	const uint8_t data_offset = hdr_sz + en_priv->aad_sz;
+	const int is_null_auth = en_priv->flags.is_null_auth;
+
+	nb_cop = rte_crypto_op_bulk_alloc(en_priv->cop_pool, RTE_CRYPTO_OP_TYPE_SYMMETRIC, cop,
+					  num);
+
+	for (i = 0; i < nb_cop; i++) {
+		mb = in_mb[i];
+		/* Prepend PDU header */
+		pdu_hdr = (struct rte_pdcp_cp_data_pdu_sn_12_hdr *)rte_pktmbuf_prepend(mb, hdr_sz);
+		if (unlikely(pdu_hdr == NULL)) {
+			in_mb[nb_err++] = mb;
+			continue;
+		}
+
+		mac_i = (uint8_t *)rte_pktmbuf_append(mb, RTE_PDCP_MAC_I_LEN);
+		if (unlikely(mac_i == NULL)) {
+			in_mb[nb_err++] = mb;
+			continue;
+		}
+
+		/* Clear MAC-I field for NULL auth */
+		if (is_null_auth)
+			memset(mac_i, 0, RTE_PDCP_MAC_I_LEN);
+
+		/* Update sequence number in the PDU header */
+		count = en_priv->state.tx_next++;
+		sn = pdcp_sn_from_count_get(count, RTE_SECURITY_PDCP_SN_SIZE_12);
+
+		pdu_hdr->sn_11_8 = ((sn & 0xf00) >> 8);
+		pdu_hdr->sn_7_0 = (sn & 0xff);
+		pdu_hdr->r = 0;
+
+		cop_prepare(en_priv, mb, cop[nb_prep++], data_offset, count, true);
+	}
+
+	if (unlikely(nb_err))
+		/* Using mempool API since crypto API is not providing bulk free */
+		rte_mempool_put_bulk(en_priv->cop_pool, (void *)&cop[nb_prep], nb_cop - nb_prep);
+
+	*nb_err_ret = num - nb_prep;
+
+	return nb_prep;
+}
+
+static uint16_t
+pdcp_post_process_ul(const struct rte_pdcp_entity *entity,
+		     struct rte_mbuf *in_mb[], struct rte_mbuf *out_mb[],
+		     uint16_t num, uint16_t *nb_err_ret)
+{
+	struct entity_priv *en_priv = entity_priv_get(entity);
+	const uint32_t hdr_trim_sz = en_priv->aad_sz;
+	int i, nb_success = 0, nb_err = 0;
+	struct rte_mbuf *mb, *err_mb[num];
+
+	for (i = 0; i < num; i++) {
+		mb = in_mb[i];
+		if (unlikely(mb->ol_flags & RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED)) {
+			err_mb[nb_err++] = mb;
+			continue;
+		}
+
+		if (hdr_trim_sz)
+			rte_pktmbuf_adj(mb, hdr_trim_sz);
+
+		out_mb[nb_success++] = mb;
+	}
+
+	if (unlikely(nb_err != 0))
+		rte_memcpy(&out_mb[nb_success], err_mb, nb_err * sizeof(struct rte_mbuf *));
+
+	*nb_err_ret = nb_err;
+	return nb_success;
+}
+
+static int
+pdcp_pre_post_func_set(struct rte_pdcp_entity *entity, const struct rte_pdcp_entity_conf *conf)
+{
+	entity->pre_process = NULL;
+	entity->post_process = NULL;
+
+	if ((conf->pdcp_xfrm.domain == RTE_SECURITY_PDCP_MODE_CONTROL) &&
+	    (conf->pdcp_xfrm.sn_size == RTE_SECURITY_PDCP_SN_SIZE_12) &&
+	    (conf->pdcp_xfrm.pkt_dir == RTE_SECURITY_PDCP_UPLINK)) {
+		entity->pre_process = pdcp_pre_process_cplane_sn_12_ul;
+		entity->post_process = pdcp_post_process_ul;
+	}
+
+	if ((conf->pdcp_xfrm.domain == RTE_SECURITY_PDCP_MODE_DATA) &&
+	    (conf->pdcp_xfrm.sn_size == RTE_SECURITY_PDCP_SN_SIZE_12) &&
+	    (conf->pdcp_xfrm.pkt_dir == RTE_SECURITY_PDCP_UPLINK)) {
+		entity->pre_process = pdcp_pre_process_uplane_sn_12_ul;
+		entity->post_process = pdcp_post_process_ul;
+	}
+
+	if ((conf->pdcp_xfrm.domain == RTE_SECURITY_PDCP_MODE_DATA) &&
+	    (conf->pdcp_xfrm.sn_size == RTE_SECURITY_PDCP_SN_SIZE_18) &&
+	    (conf->pdcp_xfrm.pkt_dir == RTE_SECURITY_PDCP_UPLINK)) {
+		entity->pre_process = pdcp_pre_process_uplane_sn_18_ul;
+		entity->post_process = pdcp_post_process_ul;
+	}
+
+	if (entity->pre_process == NULL || entity->post_process == NULL)
+		return -ENOTSUP;
+
+	return 0;
+}
+
 static int
 pdcp_entity_priv_populate(struct entity_priv *en_priv, const struct rte_pdcp_entity_conf *conf)
 {
@@ -134,5 +464,9 @@ pdcp_process_func_set(struct rte_pdcp_entity *entity, const struct rte_pdcp_enti
 	if (ret)
 		return ret;
 
+	ret = pdcp_pre_post_func_set(entity, conf);
+	if (ret)
+		return ret;
+
 	return 0;
 }
-- 
2.25.1