RE: [PATCH 3/6] examples/ipsec-secgw: add lookaside event mode

[Suanming Mou <suanmingm@nvidia.com>]

Hi,

> -----Original Message-----
> From: Volodymyr Fialko <vfialko@marvell.com>
> Sent: Thursday, August 4, 2022 6:36 PM
> To: dev@dpdk.org; Radu Nicolau <radu.nicolau@intel.com>; Akhil Goyal
> <gakhil@marvell.com>
> Cc: jerinj@marvell.com; anoobj@marvell.com; Volodymyr Fialko
> <vfialko@marvell.com>
> Subject: [PATCH 3/6] examples/ipsec-secgw: add lookaside event mode
> 
> Add base support for lookaside event mode. Events that are coming from ethdev
> will be enqueued to the event crypto adapter, processed and enqueued back to
> ethdev for the transmission.
> 
> Signed-off-by: Volodymyr Fialko <vfialko@marvell.com>
> ---
>  doc/guides/sample_app_ug/ipsec_secgw.rst |   4 +-
>  examples/ipsec-secgw/ipsec-secgw.c       |   3 +-
>  examples/ipsec-secgw/ipsec.c             |  35 +++-
>  examples/ipsec-secgw/ipsec.h             |   8 +-
>  examples/ipsec-secgw/ipsec_worker.c      | 224 +++++++++++++++++++++--
>  examples/ipsec-secgw/sa.c                |  23 ++-
>  6 files changed, 262 insertions(+), 35 deletions(-)
> 

snip

>  create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa, diff --git
> a/examples/ipsec-secgw/ipsec_worker.c b/examples/ipsec-
> secgw/ipsec_worker.c
> index 803157d8ee..2661f0275f 100644
> --- a/examples/ipsec-secgw/ipsec_worker.c
> +++ b/examples/ipsec-secgw/ipsec_worker.c
> @@ -3,6 +3,7 @@
>   * Copyright (C) 2020 Marvell International Ltd.
>   */
>  #include <rte_acl.h>
> +#include <rte_event_crypto_adapter.h>
>  #include <rte_event_eth_tx_adapter.h>
>  #include <rte_lpm.h>
>  #include <rte_lpm6.h>
> @@ -11,6 +12,7 @@
>  #include "ipsec.h"
>  #include "ipsec-secgw.h"
>  #include "ipsec_worker.h"
> +#include "sad.h"
> 
>  #if defined(__ARM_NEON)
>  #include "ipsec_lpm_neon.h"
> @@ -228,6 +230,43 @@ check_sp_sa_bulk(struct sp_ctx *sp, struct sa_ctx
> *sa_ctx,
>  	ip->num = j;
>  }
> 
> +static inline void
> +pkt_l3_len_set(struct rte_mbuf *pkt)
> +{
> +	struct rte_ipv4_hdr *ipv4;
> +	struct rte_ipv6_hdr *ipv6;
> +	size_t l3len, ext_len;
> +	uint32_t l3_type;
> +	int next_proto;
> +	uint8_t *p;
> +
> +	l3_type = pkt->packet_type & RTE_PTYPE_L3_MASK;
> +	if (l3_type == RTE_PTYPE_L3_IPV4) {
> +		ipv4 = rte_pktmbuf_mtod(pkt, struct rte_ipv4_hdr *);
> +		pkt->l3_len = ipv4->ihl * 4;
> +	} else if (l3_type & RTE_PTYPE_L3_IPV6) {
> +		ipv6 = rte_pktmbuf_mtod(pkt, struct rte_ipv6_hdr *);
> +		l3len = sizeof(struct rte_ipv6_hdr);
> +		if (l3_type == RTE_PTYPE_L3_IPV6_EXT ||
> +		     l3_type == RTE_PTYPE_L3_IPV6_EXT_UNKNOWN) {
> +			p = rte_pktmbuf_mtod(pkt, uint8_t *);
> +			next_proto = ipv6->proto;
> +			while (next_proto != IPPROTO_ESP &&
> +			       l3len < pkt->data_len &&
> +			       (next_proto = rte_ipv6_get_next_ext(p + l3len,
> +						next_proto, &ext_len)) >= 0)
> +				l3len += ext_len;
> +
> +			/* Drop pkt when IPv6 header exceeds first seg size */
> +			if (unlikely(l3len > pkt->data_len)) {
> +				free_pkts(&pkt, 1);
> +				return;
> +			}
> +		}
> +		pkt->l3_len = l3len;
> +	}
> +}
> +
>  static inline uint16_t
>  route4_pkt(struct rte_mbuf *pkt, struct rt_ctx *rt_ctx)  { @@ -287,9 +326,67
> @@ get_route(struct rte_mbuf *pkt, struct route_table *rt, enum pkt_type type)
>  	return RTE_MAX_ETHPORTS;
>  }
> 
> +static inline void
> +crypto_op_reset(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
> +		struct rte_crypto_op *cop[], uint16_t num) {
> +	struct rte_crypto_sym_op *sop;
> +	uint32_t i;
> +
> +	const struct rte_crypto_op unproc_cop = {
> +		.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
> +		.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED,
> +		.sess_type = RTE_CRYPTO_OP_SECURITY_SESSION,
> +	};
> +
> +	for (i = 0; i != num; i++) {
> +		cop[i]->raw = unproc_cop.raw;
> +		sop = cop[i]->sym;
> +		sop->m_src = mb[i];
> +		sop->m_dst = NULL;
> +		__rte_security_attach_session(sop, ss->security.ses);
> +	}
> +}
> +
> +static inline int
> +event_crypto_enqueue(struct ipsec_ctx *ctx __rte_unused, struct rte_mbuf
> *pkt,
> +		struct ipsec_sa *sa, const struct eh_event_link_info *ev_link) {
> +	struct ipsec_mbuf_metadata *priv;
> +	struct rte_ipsec_session *sess;
> +	struct rte_crypto_op *cop;
> +	struct rte_event cev;
> +	int ret;
> +
> +	/* Get IPsec session */
> +	sess = ipsec_get_primary_session(sa);
> +
> +	/* Get pkt private data */
> +	priv = get_priv(pkt);
> +	cop = &priv->cop;
> +
> +	/* Reset crypto operation data */
> +	crypto_op_reset(sess, &pkt, &cop, 1);
> +
> +	/* Update event_ptr with rte_crypto_op */
> +	cev.event = 0;
> +	cev.event_ptr = cop;
> +
> +	/* Enqueue event to crypto adapter */
> +	ret = rte_event_crypto_adapter_enqueue(ev_link->eventdev_id,
> +			ev_link->event_port_id, &cev, 1);
> +	if (unlikely(ret <= 0)) {
> +		/* pkt will be freed by the caller */
> +		RTE_LOG_DP(DEBUG, IPSEC, "Cannot enqueue event: %i
> (errno: %i)\n", ret, rte_errno);
> +		return rte_errno;
> +	}
> +
> +	return 0;
> +}
> +
>  static inline int
>  process_ipsec_ev_inbound(struct ipsec_ctx *ctx, struct route_table *rt,
> -		struct rte_event *ev)
> +	const struct eh_event_link_info *ev_link, struct rte_event *ev)
>  {
>  	struct ipsec_sa *sa = NULL;
>  	struct rte_mbuf *pkt;
> @@ -340,7 +437,22 @@ process_ipsec_ev_inbound(struct ipsec_ctx *ctx, struct
> route_table *rt,
>  			goto drop_pkt_and_exit;
>  		}
>  		break;
> +	case PKT_TYPE_IPSEC_IPV4:
> +	case PKT_TYPE_IPSEC_IPV6:
> +		rte_pktmbuf_adj(pkt, RTE_ETHER_HDR_LEN);
> +		pkt_l3_len_set(pkt);

One small question, in case free_pkts() happens inside pkt_l3_len_set(pkt),  can the pkt still be used in sad_lookup() below?
Should return value be added to pkt_l3_len_set() in case pkt free?

> +
> +		sad_lookup(&ctx->sa_ctx->sad, &pkt, (void **)&sa, 1);
> +		sa = ipsec_mask_saptr(sa);
> +		if (unlikely(sa == NULL)) {
> +			RTE_LOG_DP(DEBUG, IPSEC, "Cannot find sa\n");
> +			goto drop_pkt_and_exit;
> +		}
> 
snip