[dpdk-dev] [PATCH v1 0/2] examples/ipsec-secgw: add fallback session

[dpdk-dev] [PATCH v1 0/2] examples/ipsec-secgw: add fallback session

[Marcin Smoczynski]

Inline processing is limited to a specified subset of traffic. It is
often unable to handle more complicated situations, such as fragmented
traffic. When using inline processing such traffic is dropped.

Introduce multiple sessions per SA allowing to configure a fallback
lookaside session for packets that normally would be dropped.
A fallback session type in the SA configuration by adding 'fallback'
with 'lookaside-none' or 'lookaside-protocol' parameter to determine
type of session.

Fallback session feature is available only when using librte_ipsec.

Marcin Smoczynski (2):
  examples/ipsec-secgw: ipsec_sa structure cleanup
  examples/ipsec-secgw: add fallback session feature

 doc/guides/sample_app_ug/ipsec_secgw.rst |  17 ++-
 examples/ipsec-secgw/esp.c               |  35 ++++--
 examples/ipsec-secgw/ipsec-secgw.c       |  16 ++-
 examples/ipsec-secgw/ipsec.c             |  99 ++++++++-------
 examples/ipsec-secgw/ipsec.h             |  61 +++++++--
 examples/ipsec-secgw/ipsec_process.c     | 113 ++++++++++-------
 examples/ipsec-secgw/sa.c                | 153 +++++++++++++++++------
 7 files changed, 334 insertions(+), 160 deletions(-)

--
2.17.1

[dpdk-dev] [PATCH v1 1/2] examples/ipsec-secgw: ipsec_sa structure cleanup

[Marcin Smoczynski]

Cleanup ipsec_sa structure by removing every field that is already in
the rte_ipsec_session structure:
 * cryptodev/security session union
 * action type
 * offload flags
 * security context
References to abovementioned fields are changed to direct references
to matching fields of rte_ipsec_session structure.

Such refactoring is needed to introduce many sessions per SA feature,
e.g. fallback session for inline offload processing.

Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
---
 examples/ipsec-secgw/esp.c           | 35 +++++++----
 examples/ipsec-secgw/ipsec.c         | 91 +++++++++++++++-------------
 examples/ipsec-secgw/ipsec.h         | 27 ++++++---
 examples/ipsec-secgw/ipsec_process.c | 30 ++++-----
 examples/ipsec-secgw/sa.c            | 66 ++++++++++----------
 5 files changed, 137 insertions(+), 112 deletions(-)

diff --git a/examples/ipsec-secgw/esp.c b/examples/ipsec-secgw/esp.c
index d6d7b1256..c1b49da1e 100644
--- a/examples/ipsec-secgw/esp.c
+++ b/examples/ipsec-secgw/esp.c
@@ -30,7 +30,8 @@ esp_inbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 	int32_t payload_len, ip_hdr_len;
 
 	RTE_ASSERT(sa != NULL);
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
+	if (ipsec_get_action_type(sa) ==
+			RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
 		return 0;
 
 	RTE_ASSERT(m != NULL);
@@ -148,13 +149,16 @@ esp_inbound_post(struct rte_mbuf *m, struct ipsec_sa *sa,
 	uint8_t *nexthdr, *pad_len;
 	uint8_t *padding;
 	uint16_t i;
+	struct rte_ipsec_session *ips;
 
 	RTE_ASSERT(m != NULL);
 	RTE_ASSERT(sa != NULL);
 	RTE_ASSERT(cop != NULL);
 
-	if ((sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
-			(sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
+	ips = ipsec_get_session(sa);
+
+	if ((ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
+			(ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
 		if (m->ol_flags & PKT_RX_SEC_OFFLOAD) {
 			if (m->ol_flags & PKT_RX_SEC_OFFLOAD_FAILED)
 				cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
@@ -169,8 +173,8 @@ esp_inbound_post(struct rte_mbuf *m, struct ipsec_sa *sa,
 		return -1;
 	}
 
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO &&
-	    sa->ol_flags & RTE_SECURITY_RX_HW_TRAILER_OFFLOAD) {
+	if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO &&
+		ips->security.ol_flags & RTE_SECURITY_RX_HW_TRAILER_OFFLOAD) {
 		nexthdr = &m->inner_esp_next_proto;
 	} else {
 		nexthdr = rte_pktmbuf_mtod_offset(m, uint8_t*,
@@ -225,10 +229,12 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 	struct rte_crypto_sym_op *sym_cop;
 	int32_t i;
 	uint16_t pad_payload_len, pad_len, ip_hdr_len;
+	struct rte_ipsec_session *ips;
 
 	RTE_ASSERT(m != NULL);
 	RTE_ASSERT(sa != NULL);
 
+	ips = ipsec_get_session(sa);
 	ip_hdr_len = 0;
 
 	ip4 = rte_pktmbuf_mtod(m, struct ip *);
@@ -277,9 +283,10 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 	}
 
 	/* Add trailer padding if it is not constructed by HW */
-	if (sa->type != RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
-	    (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO &&
-	     !(sa->ol_flags & RTE_SECURITY_TX_HW_TRAILER_OFFLOAD))) {
+	if (ips->type != RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+		(ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO &&
+		 !(ips->security.ol_flags &
+			 RTE_SECURITY_TX_HW_TRAILER_OFFLOAD))) {
 		padding = (uint8_t *)rte_pktmbuf_append(m, pad_len +
 							sa->digest_len);
 		if (unlikely(padding == NULL)) {
@@ -344,8 +351,9 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 		}
 	}
 
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
-		if (sa->ol_flags & RTE_SECURITY_TX_HW_TRAILER_OFFLOAD) {
+	if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		if (ips->security.ol_flags &
+				RTE_SECURITY_TX_HW_TRAILER_OFFLOAD) {
 			/* Set the inner esp next protocol for HW trailer */
 			m->inner_esp_next_proto = nlp;
 			m->packet_type |= RTE_PTYPE_TUNNEL_ESP;
@@ -448,11 +456,14 @@ esp_outbound_post(struct rte_mbuf *m,
 		  struct ipsec_sa *sa,
 		  struct rte_crypto_op *cop)
 {
+	enum rte_security_session_action_type type;
 	RTE_ASSERT(m != NULL);
 	RTE_ASSERT(sa != NULL);
 
-	if ((sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
-			(sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
+	type = ipsec_get_action_type(sa);
+
+	if ((type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
+			(type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
 		m->ol_flags |= PKT_TX_SEC_OFFLOAD;
 	} else {
 		RTE_ASSERT(cop != NULL);
diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index dc85adfe5..8c60bd84b 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -40,7 +40,8 @@ set_ipsec_conf(struct ipsec_sa *sa, struct rte_security_ipsec_xform *ipsec)
 }
 
 int
-create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips)
 {
 	struct rte_cryptodev_info cdev_info;
 	unsigned long cdev_id_qp = 0;
@@ -71,9 +72,9 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			ipsec_ctx->tbl[cdev_id_qp].id,
 			ipsec_ctx->tbl[cdev_id_qp].qp);
 
-	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
+	if (ips->type != RTE_SECURITY_ACTION_TYPE_NONE) {
 		struct rte_security_session_conf sess_conf = {
-			.action_type = sa->type,
+			.action_type = ips->type,
 			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 			{.ipsec = {
 				.spi = sa->spi,
@@ -90,7 +91,7 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 
 		};
 
-		if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
+		if (ips->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
 			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
 							rte_cryptodev_get_sec_ctx(
 							ipsec_ctx->tbl[cdev_id_qp].id);
@@ -98,9 +99,9 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			/* Set IPsec parameters in conf */
 			set_ipsec_conf(sa, &(sess_conf.ipsec));
 
-			sa->sec_session = rte_security_session_create(ctx,
+			ips->security.ses = rte_security_session_create(ctx,
 					&sess_conf, ipsec_ctx->session_priv_pool);
-			if (sa->sec_session == NULL) {
+			if (ips->security.ses == NULL) {
 				RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 				return -1;
@@ -110,10 +111,10 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			return -1;
 		}
 	} else {
-		sa->crypto_session = rte_cryptodev_sym_session_create(
+		ips->crypto.ses = rte_cryptodev_sym_session_create(
 				ipsec_ctx->session_pool);
 		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
-				sa->crypto_session, sa->xforms,
+				ips->crypto.ses, sa->xforms,
 				ipsec_ctx->session_priv_pool);
 
 		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
@@ -126,12 +127,13 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 }
 
 int
-create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips)
 {
 	int32_t ret = 0;
 	struct rte_security_ctx *sec_ctx;
 	struct rte_security_session_conf sess_conf = {
-		.action_type = sa->type,
+		.action_type = ips->type,
 		.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 		{.ipsec = {
 			.spi = sa->spi,
@@ -151,7 +153,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on port %u\n",
 		sa->spi, sa->portid);
 
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+	if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
 		struct rte_flow_error err;
 		const struct rte_security_capability *sec_cap;
 		int ret = 0;
@@ -165,9 +167,9 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 			return -1;
 		}
 
-		sa->sec_session = rte_security_session_create(sec_ctx,
+		ips->security.ses = rte_security_session_create(sec_ctx,
 				&sess_conf, skt_ctx->session_pool);
-		if (sa->sec_session == NULL) {
+		if (ips->security.ses == NULL) {
 			RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 			return -1;
@@ -177,7 +179,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 
 		/* iterate until ESP tunnel*/
 		while (sec_cap->action != RTE_SECURITY_ACTION_TYPE_NONE) {
-			if (sec_cap->action == sa->type &&
+			if (sec_cap->action == ips->type &&
 			    sec_cap->protocol ==
 				RTE_SECURITY_PROTOCOL_IPSEC &&
 			    sec_cap->ipsec.mode ==
@@ -193,8 +195,8 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 			return -1;
 		}
 
-		sa->ol_flags = sec_cap->ol_flags;
-		sa->security_ctx = sec_ctx;
+		ips->security.ol_flags = sec_cap->ol_flags;
+		ips->security.ctx = sec_ctx;
 		sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
 
 		if (IS_IP6(sa->flags)) {
@@ -223,7 +225,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 		sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
 
 		sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
-		sa->action[0].conf = sa->sec_session;
+		sa->action[0].conf = ips->security.ses;
 
 		sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
 
@@ -282,8 +284,8 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 			if (ret)
 				goto flow_create_failure;
 		} else if (sa->attr.egress &&
-			   (sa->ol_flags &
-				    RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
+				(ips->security.ol_flags &
+					RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
 			sa->action[1].type =
 					RTE_FLOW_ACTION_TYPE_PASSTHRU;
 			sa->action[2].type =
@@ -299,7 +301,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 				err.message);
 			return -1;
 		}
-	} else if (sa->type ==	RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
+	} else if (ips->type ==	RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
 		const struct rte_security_capability *sec_cap;
 
 		sec_ctx = (struct rte_security_ctx *)
@@ -327,9 +329,9 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 
 		sess_conf.userdata = (void *) sa;
 
-		sa->sec_session = rte_security_session_create(sec_ctx,
+		ips->security.ses = rte_security_session_create(sec_ctx,
 					&sess_conf, skt_ctx->session_pool);
-		if (sa->sec_session == NULL) {
+		if (ips->security.ses == NULL) {
 			RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 			return -1;
@@ -345,7 +347,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 		/* iterate until ESP tunnel*/
 		while (sec_cap->action !=
 				RTE_SECURITY_ACTION_TYPE_NONE) {
-			if (sec_cap->action == sa->type &&
+			if (sec_cap->action == ips->type &&
 			    sec_cap->protocol ==
 				RTE_SECURITY_PROTOCOL_IPSEC &&
 			    sec_cap->ipsec.mode ==
@@ -361,8 +363,8 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 			return -1;
 		}
 
-		sa->ol_flags = sec_cap->ol_flags;
-		sa->security_ctx = sec_ctx;
+		ips->security.ol_flags = sec_cap->ol_flags;
+		ips->security.ctx = sec_ctx;
 	}
 	sa->cdev_id_qp = 0;
 
@@ -409,6 +411,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 	struct ipsec_mbuf_metadata *priv;
 	struct rte_crypto_sym_op *sym_cop;
 	struct ipsec_sa *sa;
+	struct rte_ipsec_session *ips;
 
 	for (i = 0; i < nb_pkts; i++) {
 		if (unlikely(sas[i] == NULL)) {
@@ -422,16 +425,17 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 		priv = get_priv(pkts[i]);
 		sa = sas[i];
 		priv->sa = sa;
+		ips = ipsec_get_session(sa);
 
-		switch (sa->type) {
+		switch (ips->type) {
 		case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL:
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
 
-			if ((unlikely(sa->sec_session == NULL)) &&
-				create_lookaside_session(ipsec_ctx, sa)) {
+			if ((unlikely(ips->security.ses == NULL)) &&
+				create_lookaside_session(ipsec_ctx, sa, ips)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -440,7 +444,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			sym_cop->m_src = pkts[i];
 
 			rte_security_attach_session(&priv->cop,
-					sa->sec_session);
+				ips->security.ses);
 			break;
 		case RTE_SECURITY_ACTION_TYPE_NONE:
 
@@ -449,14 +453,14 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			rte_prefetch0(&priv->sym_cop);
 
-			if ((unlikely(sa->crypto_session == NULL)) &&
-				create_lookaside_session(ipsec_ctx, sa)) {
+			if ((unlikely(ips->crypto.ses == NULL)) &&
+				create_lookaside_session(ipsec_ctx, sa, ips)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
 
 			rte_crypto_op_attach_sym_session(&priv->cop,
-					sa->crypto_session);
+					ips->crypto.ses);
 
 			ret = xform_func(pkts[i], sa, &priv->cop);
 			if (unlikely(ret)) {
@@ -465,21 +469,22 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 			break;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
-			RTE_ASSERT(sa->sec_session != NULL);
+			RTE_ASSERT(ips->security.ses != NULL);
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
-			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
+			if (ips->security.ol_flags &
+				RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
-						sa->security_ctx,
-						sa->sec_session, pkts[i], NULL);
+					ips->security.ctx, ips->security.ses,
+					pkts[i], NULL);
 			continue;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
-			RTE_ASSERT(sa->sec_session != NULL);
+			RTE_ASSERT(ips->security.ses != NULL);
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
 			rte_security_attach_session(&priv->cop,
-					sa->sec_session);
+					ips->security.ses);
 
 			ret = xform_func(pkts[i], sa, &priv->cop);
 			if (unlikely(ret)) {
@@ -488,10 +493,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
-			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
+			if (ips->security.ol_flags &
+				RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
-						sa->security_ctx,
-						sa->sec_session, pkts[i], NULL);
+					ips->security.ctx, ips->security.ses,
+					pkts[i], NULL);
 			continue;
 		}
 
@@ -560,7 +566,8 @@ ipsec_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			RTE_ASSERT(sa != NULL);
 
-			if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {
+			if (ipsec_get_action_type(sa) ==
+				RTE_SECURITY_ACTION_TYPE_NONE) {
 				ret = xform_func(pkt, sa, cops[j]);
 				if (unlikely(ret)) {
 					rte_pktmbuf_free(pkt);
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index 1efa6e488..a4ad81b0e 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -87,10 +87,6 @@ struct ipsec_sa {
 	uint32_t cdev_id_qp;
 	uint64_t seq;
 	uint32_t salt;
-	union {
-		struct rte_cryptodev_sym_session *crypto_session;
-		struct rte_security_session *sec_session;
-	};
 	enum rte_crypto_cipher_algorithm cipher_algo;
 	enum rte_crypto_auth_algorithm auth_algo;
 	enum rte_crypto_aead_algorithm aead_algo;
@@ -114,11 +110,8 @@ struct ipsec_sa {
 		struct rte_crypto_sym_xform *xforms;
 		struct rte_security_ipsec_xform *sec_xform;
 	};
-	enum rte_security_session_action_type type;
 	enum rte_security_ipsec_sa_direction direction;
 	uint16_t portid;
-	struct rte_security_ctx *security_ctx;
-	uint32_t ol_flags;
 
 #define MAX_RTE_FLOW_PATTERN (4)
 #define MAX_RTE_FLOW_ACTIONS (3)
@@ -284,6 +277,20 @@ get_sym_cop(struct rte_crypto_op *cop)
 	return (cop + 1);
 }
 
+static inline struct rte_ipsec_session *
+ipsec_get_session(struct ipsec_sa *sa)
+{
+	return &sa->ips;
+}
+
+static inline enum rte_security_session_action_type
+ipsec_get_action_type(struct ipsec_sa *sa)
+{
+	struct rte_ipsec_session *ips;
+	ips = ipsec_get_session(sa);
+	return ips->type;
+}
+
 int
 inbound_sa_check(struct sa_ctx *sa_ctx, struct rte_mbuf *m, uint32_t sa_idx);
 
@@ -338,9 +345,11 @@ void
 enqueue_cop_burst(struct cdev_qp *cqp);
 
 int
-create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips);
 
 int
-create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa);
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips);
 
 #endif /* __IPSEC_H__ */
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index 868f1a28d..239d81ef6 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -94,22 +94,16 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
 
 	/* setup crypto section */
 	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		if (sa->crypto_session == NULL) {
-			rc = create_lookaside_session(ctx, sa);
-			if (rc != 0)
-				return rc;
-		}
-		ss->crypto.ses = sa->crypto_session;
+		RTE_ASSERT(ss->crypto.ses == NULL);
+		rc = create_lookaside_session(ctx, sa, ss);
+		if (rc != 0)
+			return rc;
 	/* setup session action type */
-	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
-		if (sa->sec_session == NULL) {
-			rc = create_lookaside_session(ctx, sa);
-			if (rc != 0)
-				return rc;
-		}
-		ss->security.ses = sa->sec_session;
-		ss->security.ctx = sa->security_ctx;
-		ss->security.ol_flags = sa->ol_flags;
+	} else if (ss->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
+		RTE_ASSERT(ss->security.ses == NULL);
+		rc = create_lookaside_session(ctx, sa, ss);
+		if (rc != 0)
+			return rc;
 	} else
 		RTE_ASSERT(0);
 
@@ -218,7 +212,7 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 		pg = grp + i;
 		sa = pg->id.ptr;
 
-		ips = &sa->ips;
+		ips = ipsec_get_session(sa);
 
 		/* no valid HW session for that SA, try to create one */
 		if (sa == NULL || (ips->crypto.ses == NULL &&
@@ -226,8 +220,8 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 			k = 0;
 
 		/* process packets inline */
-		else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
-				sa->type ==
+		else if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+				ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
 
 			satp = rte_ipsec_sa_type(ips->sa);
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index c3cf3bd1f..a3d8e4545 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -224,6 +224,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	struct parse_status *status)
 {
 	struct ipsec_sa *rule = NULL;
+	struct rte_ipsec_session *ips;
 	uint32_t ti; /*token index*/
 	uint32_t *ri /*rule index*/;
 	uint32_t cipher_algo_p = 0;
@@ -262,6 +263,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	if (atoi(tokens[1]) == INVALID_SPI)
 		return;
 	rule->spi = atoi(tokens[1]);
+	ips = ipsec_get_session(rule);
 
 	for (ti = 2; ti < n_tokens; ti++) {
 		if (strcmp(tokens[ti], "mode") == 0) {
@@ -558,18 +560,18 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 				return;
 
 			if (strcmp(tokens[ti], "inline-crypto-offload") == 0)
-				rule->type =
+				ips->type =
 					RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO;
 			else if (strcmp(tokens[ti],
 					"inline-protocol-offload") == 0)
-				rule->type =
+				ips->type =
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL;
 			else if (strcmp(tokens[ti],
 					"lookaside-protocol-offload") == 0)
-				rule->type =
+				ips->type =
 				RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL;
 			else if (strcmp(tokens[ti], "no-offload") == 0)
-				rule->type = RTE_SECURITY_ACTION_TYPE_NONE;
+				ips->type = RTE_SECURITY_ACTION_TYPE_NONE;
 			else {
 				APP_CHECK(0, status, "Invalid input \"%s\"",
 						tokens[ti]);
@@ -624,11 +626,11 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	if (status->status < 0)
 		return;
 
-	if ((rule->type != RTE_SECURITY_ACTION_TYPE_NONE) && (portid_p == 0))
+	if ((ips->type != RTE_SECURITY_ACTION_TYPE_NONE) && (portid_p == 0))
 		printf("Missing portid option, falling back to non-offload\n");
 
 	if (!type_p || !portid_p) {
-		rule->type = RTE_SECURITY_ACTION_TYPE_NONE;
+		ips->type = RTE_SECURITY_ACTION_TYPE_NONE;
 		rule->portid = -1;
 	}
 
@@ -640,6 +642,7 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 {
 	uint32_t i;
 	uint8_t a, b, c, d;
+	const struct rte_ipsec_session *ips;
 
 	printf("\tspi_%s(%3u):", inbound?"in":"out", sa->spi);
 
@@ -695,8 +698,10 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 		printf("Transport ");
 		break;
 	}
+
+	ips = &sa->ips;
 	printf(" type:");
-	switch (sa->type) {
+	switch (ips->type) {
 	case RTE_SECURITY_ACTION_TYPE_NONE:
 		printf("no-offload ");
 		break;
@@ -876,6 +881,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 	uint16_t iv_length, aad_length;
 	int inline_status;
 	int32_t rc;
+	struct rte_ipsec_session *ips;
 
 	/* for ESN upper 32 bits of SQN also need to be part of AAD */
 	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
@@ -890,9 +896,10 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 		}
 		*sa = entries[i];
 		sa->seq = 0;
+		ips = ipsec_get_session(sa);
 
-		if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
-			sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
+			ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
 			if (check_eth_dev_caps(sa->portid, inbound))
 				return -EINVAL;
 		}
@@ -907,7 +914,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 			sa->dst.ip.ip4 = rte_cpu_to_be_32(sa->dst.ip.ip4);
 			break;
 		case TRANSPORT:
-			if (sa->type ==
+			if (ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
 				inline_status =
 					sa_add_address_inline_crypto(sa);
@@ -918,6 +925,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 		}
 
 		if (sa->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) {
+			struct rte_ipsec_session *ips;
 			iv_length = 16;
 
 			sa_ctx->xf[idx].a.type = RTE_CRYPTO_SYM_XFORM_AEAD;
@@ -938,11 +946,12 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			sa->xforms = &sa_ctx->xf[idx].a;
 
-			if (sa->type ==
+			ips = ipsec_get_session(sa);
+			if (ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
-				sa->type ==
+				ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
-				rc = create_inline_session(skt_ctx, sa);
+				rc = create_inline_session(skt_ctx, sa, ips);
 				if (rc != 0) {
 					RTE_LOG(ERR, IPSEC_ESP,
 						"create_inline_session() failed\n");
@@ -1100,23 +1109,11 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm, const struct ipsec_sa *ss,
 }
 
 static int
-fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
-	const struct ipsec_sa *lsa)
+fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa)
 {
 	int32_t rc = 0;
 
 	ss->sa = sa;
-	ss->type = lsa->type;
-
-	/* setup crypto section */
-	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		ss->crypto.ses = lsa->crypto_session;
-	/* setup session action type */
-	} else {
-		ss->security.ses = lsa->sec_session;
-		ss->security.ctx = lsa->security_ctx;
-		ss->security.ol_flags = lsa->ol_flags;
-	}
 
 	if (ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
 		ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
@@ -1138,6 +1135,7 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 {
 	int rc;
 	struct rte_ipsec_sa_prm prm;
+	struct rte_ipsec_session *ips;
 	struct rte_ipv4_hdr v4  = {
 		.version_ihl = IPVERSION << 4 |
 			sizeof(v4) / RTE_IPV4_IHL_MULTIPLIER,
@@ -1162,7 +1160,10 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
+	/* init processing session */
+	ips = ipsec_get_session(lsa);
+	rc = fill_ipsec_session(ips, sa);
+
 	return rc;
 }
 
@@ -1386,6 +1387,7 @@ sa_check_offloads(uint16_t port_id, uint64_t *rx_offloads,
 {
 	struct ipsec_sa *rule;
 	uint32_t idx_sa;
+	enum rte_security_session_action_type rule_type;
 
 	*rx_offloads = 0;
 	*tx_offloads = 0;
@@ -1393,8 +1395,9 @@ sa_check_offloads(uint16_t port_id, uint64_t *rx_offloads,
 	/* Check for inbound rules that use offloads and use this port */
 	for (idx_sa = 0; idx_sa < nb_sa_in; idx_sa++) {
 		rule = &sa_in[idx_sa];
-		if ((rule->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
-				rule->type ==
+		rule_type = ipsec_get_action_type(rule);
+		if ((rule_type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+				rule_type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
 				&& rule->portid == port_id)
 			*rx_offloads |= DEV_RX_OFFLOAD_SECURITY;
@@ -1403,8 +1406,9 @@ sa_check_offloads(uint16_t port_id, uint64_t *rx_offloads,
 	/* Check for outbound rules that use offloads and use this port */
 	for (idx_sa = 0; idx_sa < nb_sa_out; idx_sa++) {
 		rule = &sa_out[idx_sa];
-		if ((rule->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
-				rule->type ==
+		rule_type = ipsec_get_action_type(rule);
+		if ((rule_type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+				rule_type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
 				&& rule->portid == port_id)
 			*tx_offloads |= DEV_TX_OFFLOAD_SECURITY;
-- 
2.17.1

[dpdk-dev] [PATCH v1 2/2] examples/ipsec-secgw: add fallback session feature

[Marcin Smoczynski]

Inline processing is limited to a specified subset of traffic. It is
often unable to handle more complicated situations, such as fragmented
traffic. When using inline processing such traffic is dropped.

Introduce multiple sessions per SA allowing to configure a fallback
lookaside session for packets that normally would be dropped.
A fallback session type in the SA configuration by adding 'fallback'
with 'lookaside-none' or 'lookaside-protocol' parameter to determine
type of session.

Fallback session feature is available only when using librte_ipsec.

Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
---
 doc/guides/sample_app_ug/ipsec_secgw.rst | 17 +++-
 examples/ipsec-secgw/esp.c               |  4 +-
 examples/ipsec-secgw/ipsec-secgw.c       | 16 ++--
 examples/ipsec-secgw/ipsec.c             | 10 +--
 examples/ipsec-secgw/ipsec.h             | 40 ++++++++--
 examples/ipsec-secgw/ipsec_process.c     | 85 +++++++++++++-------
 examples/ipsec-secgw/sa.c                | 99 ++++++++++++++++++++----
 7 files changed, 210 insertions(+), 61 deletions(-)

diff --git a/doc/guides/sample_app_ug/ipsec_secgw.rst b/doc/guides/sample_app_ug/ipsec_secgw.rst
index ad2d79e75..9ded5fb70 100644
--- a/doc/guides/sample_app_ug/ipsec_secgw.rst
+++ b/doc/guides/sample_app_ug/ipsec_secgw.rst
@@ -401,7 +401,7 @@ The SA rule syntax is shown as follows:
 .. code-block:: console
 
     sa <dir> <spi> <cipher_algo> <cipher_key> <auth_algo> <auth_key>
-    <mode> <src_ip> <dst_ip> <action_type> <port_id>
+    <mode> <src_ip> <dst_ip> <action_type> <port_id> <fallback>
 
 where each options means:
 
@@ -573,6 +573,21 @@ where each options means:
 
    * *port_id X* X is a valid device number in decimal
 
+ ``<fallback>``
+
+ * Action type for packets that inline processor failed to process.
+
+ * Optional: Yes, by default there is *no fallback* session type.
+
+ * Available options:
+
+   * *lookaside-none*: use automatically chosen cryptodev to process packets
+   * *lookaside-protocol*: lookaside protocol offload to HW accelerator
+
+ * Syntax:
+
+   * *fallback lookaside-none*
+   * *fallback lookaside-protocol*
 
 Example SA rules:
 
diff --git a/examples/ipsec-secgw/esp.c b/examples/ipsec-secgw/esp.c
index c1b49da1e..bfa7ff721 100644
--- a/examples/ipsec-secgw/esp.c
+++ b/examples/ipsec-secgw/esp.c
@@ -155,7 +155,7 @@ esp_inbound_post(struct rte_mbuf *m, struct ipsec_sa *sa,
 	RTE_ASSERT(sa != NULL);
 	RTE_ASSERT(cop != NULL);
 
-	ips = ipsec_get_session(sa);
+	ips = ipsec_get_primary_session(sa);
 
 	if ((ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
 			(ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
@@ -234,7 +234,7 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 	RTE_ASSERT(m != NULL);
 	RTE_ASSERT(sa != NULL);
 
-	ips = ipsec_get_session(sa);
+	ips = ipsec_get_primary_session(sa);
 	ip_hdr_len = 0;
 
 	ip4 = rte_pktmbuf_mtod(m, struct ip *);
diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index 0d1fd6af6..641ed3767 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -189,6 +189,7 @@ static uint32_t mtu_size = RTE_ETHER_MTU;
 
 /* application wide librte_ipsec/SA parameters */
 struct app_sa_prm app_sa_prm = {.enable = 0};
+static const char *cfgfile;
 
 struct lcore_rx_queue {
 	uint16_t port_id;
@@ -1462,12 +1463,7 @@ parse_args(int32_t argc, char **argv)
 				print_usage(prgname);
 				return -1;
 			}
-			if (parse_cfg_file(optarg) < 0) {
-				printf("parsing file \"%s\" failed\n",
-					optarg);
-				print_usage(prgname);
-				return -1;
-			}
+			cfgfile = optarg;
 			f_present = 1;
 			break;
 		case 'j':
@@ -2398,6 +2394,14 @@ main(int32_t argc, char **argv)
 	if (ret < 0)
 		rte_exit(EXIT_FAILURE, "Invalid parameters\n");
 
+	/* parse configuration file */
+	if (parse_cfg_file(cfgfile) < 0) {
+		printf("parsing file \"%s\" failed\n",
+			optarg);
+		print_usage(argv[0]);
+		return -1;
+	}
+
 	if ((unprotected_port_mask & enabled_port_mask) !=
 			unprotected_port_mask)
 		rte_exit(EXIT_FAILURE, "Invalid unprotected portmask 0x%x\n",
diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index 8c60bd84b..8b0441028 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -404,7 +404,7 @@ enqueue_cop(struct cdev_qp *cqp, struct rte_crypto_op *cop)
 
 static inline void
 ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
-		struct rte_mbuf *pkts[], struct ipsec_sa *sas[],
+		struct rte_mbuf *pkts[], void *sas[],
 		uint16_t nb_pkts)
 {
 	int32_t ret = 0, i;
@@ -423,9 +423,9 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 		rte_prefetch0(pkts[i]);
 
 		priv = get_priv(pkts[i]);
-		sa = sas[i];
+		sa = ipsec_mask_saptr(sas[i]);
 		priv->sa = sa;
-		ips = ipsec_get_session(sa);
+		ips = ipsec_get_primary_session(sa);
 
 		switch (ips->type) {
 		case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL:
@@ -586,7 +586,7 @@ uint16_t
 ipsec_inbound(struct ipsec_ctx *ctx, struct rte_mbuf *pkts[],
 		uint16_t nb_pkts, uint16_t len)
 {
-	struct ipsec_sa *sas[nb_pkts];
+	void *sas[nb_pkts];
 
 	inbound_sa_lookup(ctx->sa_ctx, pkts, sas, nb_pkts);
 
@@ -606,7 +606,7 @@ uint16_t
 ipsec_outbound(struct ipsec_ctx *ctx, struct rte_mbuf *pkts[],
 		uint32_t sa_idx[], uint16_t nb_pkts, uint16_t len)
 {
-	struct ipsec_sa *sas[nb_pkts];
+	void *sas[nb_pkts];
 
 	outbound_sa_lookup(ctx->sa_ctx, sa_idx, sas, nb_pkts);
 
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index a4ad81b0e..8e075216c 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -81,12 +81,32 @@ struct app_sa_prm {
 
 extern struct app_sa_prm app_sa_prm;
 
+enum {
+	IPSEC_SESSION_PRIMARY = 0,
+	IPSEC_SESSION_FALLBACK = 1,
+	IPSEC_SESSION_MAX
+};
+
+#define IPSEC_SA_OFFLOAD_FALLBACK_FLAG (1)
+
+static inline struct ipsec_sa *
+ipsec_mask_saptr(void *ptr)
+{
+	uintptr_t i = (uintptr_t)ptr;
+	static const uintptr_t mask = IPSEC_SA_OFFLOAD_FALLBACK_FLAG;
+
+	i &= ~mask;
+
+	return (struct ipsec_sa *)i;
+}
+
 struct ipsec_sa {
-	struct rte_ipsec_session ips; /* one session per sa for now */
+	struct rte_ipsec_session sessions[IPSEC_SESSION_MAX];
 	uint32_t spi;
 	uint32_t cdev_id_qp;
 	uint64_t seq;
 	uint32_t salt;
+	uint32_t fallback_sessions;
 	enum rte_crypto_cipher_algorithm cipher_algo;
 	enum rte_crypto_auth_algorithm auth_algo;
 	enum rte_crypto_aead_algorithm aead_algo;
@@ -210,7 +230,7 @@ struct cnt_blk {
 struct traffic_type {
 	const uint8_t *data[MAX_PKT_BURST * 2];
 	struct rte_mbuf *pkts[MAX_PKT_BURST * 2];
-	struct ipsec_sa *saptr[MAX_PKT_BURST * 2];
+	void *saptr[MAX_PKT_BURST * 2];
 	uint32_t res[MAX_PKT_BURST * 2];
 	uint32_t num;
 };
@@ -278,16 +298,22 @@ get_sym_cop(struct rte_crypto_op *cop)
 }
 
 static inline struct rte_ipsec_session *
-ipsec_get_session(struct ipsec_sa *sa)
+ipsec_get_primary_session(struct ipsec_sa *sa)
+{
+	return &sa->sessions[IPSEC_SESSION_PRIMARY];
+}
+
+static inline struct rte_ipsec_session *
+ipsec_get_fallback_session(struct ipsec_sa *sa)
 {
-	return &sa->ips;
+	return &sa->sessions[IPSEC_SESSION_FALLBACK];
 }
 
 static inline enum rte_security_session_action_type
 ipsec_get_action_type(struct ipsec_sa *sa)
 {
 	struct rte_ipsec_session *ips;
-	ips = ipsec_get_session(sa);
+	ips = ipsec_get_primary_session(sa);
 	return ips->type;
 }
 
@@ -296,11 +322,11 @@ inbound_sa_check(struct sa_ctx *sa_ctx, struct rte_mbuf *m, uint32_t sa_idx);
 
 void
 inbound_sa_lookup(struct sa_ctx *sa_ctx, struct rte_mbuf *pkts[],
-		struct ipsec_sa *sa[], uint16_t nb_pkts);
+		void *sa[], uint16_t nb_pkts);
 
 void
 outbound_sa_lookup(struct sa_ctx *sa_ctx, uint32_t sa_idx[],
-		struct ipsec_sa *sa[], uint16_t nb_pkts);
+		void *sa[], uint16_t nb_pkts);
 
 void
 sp4_init(struct socket_ctx *ctx, int32_t socket_id);
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index 239d81ef6..2eb5c8b34 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -18,7 +18,6 @@
 	(((t) & RTE_IPSEC_SATP_IPV_MASK) == RTE_IPSEC_SATP_IPV4)) || \
 	((t) & RTE_IPSEC_SATP_MODE_MASK) == RTE_IPSEC_SATP_MODE_TUNLV4)
 
-
 /* helper routine to free bulk of packets */
 static inline void
 free_pkts(struct rte_mbuf *mb[], uint32_t n)
@@ -118,7 +117,7 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
  * group input packets byt the SA they belong to.
  */
 static uint32_t
-sa_group(struct ipsec_sa *sa_ptr[], struct rte_mbuf *pkts[],
+sa_group(void *sa_ptr[], struct rte_mbuf *pkts[],
 	struct rte_ipsec_group grp[], uint32_t num)
 {
 	uint32_t i, n, spi;
@@ -185,6 +184,37 @@ copy_to_trf(struct ipsec_traffic *trf, uint64_t satp, struct rte_mbuf *mb[],
 	out->num += num;
 }
 
+static uint32_t
+ipsec_prepare_crypto_group(struct ipsec_ctx *ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips, struct rte_mbuf **m,
+		unsigned int cnt)
+{
+	struct cdev_qp *cqp;
+	struct rte_crypto_op *cop[cnt];
+	uint32_t j, k;
+	struct ipsec_mbuf_metadata *priv;
+
+	cqp = &ctx->tbl[sa->cdev_id_qp];
+
+	/* for that app each mbuf has it's own crypto op */
+	for (j = 0; j != cnt; j++) {
+		priv = get_priv(m[j]);
+		cop[j] = &priv->cop;
+		/*
+		 * this is just to satisfy inbound_sa_check()
+		 * should be removed in future.
+		 */
+		priv->sa = sa;
+	}
+
+	/* prepare and enqueue crypto ops */
+	k = rte_ipsec_pkt_crypto_prepare(ips, m, cop, cnt);
+	if (k != 0)
+		enqueue_cop_bulk(cqp, cop, k);
+
+	return k;
+}
+
 /*
  * Process ipsec packets.
  * If packet belong to SA that is subject of inline-crypto,
@@ -201,18 +231,15 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 	struct ipsec_mbuf_metadata *priv;
 	struct rte_ipsec_group *pg;
 	struct rte_ipsec_session *ips;
-	struct cdev_qp *cqp;
-	struct rte_crypto_op *cop[RTE_DIM(trf->ipsec.pkts)];
 	struct rte_ipsec_group grp[RTE_DIM(trf->ipsec.pkts)];
 
 	n = sa_group(trf->ipsec.saptr, trf->ipsec.pkts, grp, trf->ipsec.num);
 
 	for (i = 0; i != n; i++) {
-
 		pg = grp + i;
-		sa = pg->id.ptr;
+		sa = ipsec_mask_saptr(pg->id.ptr);
 
-		ips = ipsec_get_session(sa);
+		ips = ipsec_get_primary_session(sa);
 
 		/* no valid HW session for that SA, try to create one */
 		if (sa == NULL || (ips->crypto.ses == NULL &&
@@ -224,6 +251,7 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 				ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
 
+			/* get SA type */
 			satp = rte_ipsec_sa_type(ips->sa);
 
 			/*
@@ -236,30 +264,33 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 				priv->sa = sa;
 			}
 
-			k = rte_ipsec_pkt_process(ips, pg->m, pg->cnt);
-			copy_to_trf(trf, satp, pg->m, k);
+			/* fallback to cryptodev with RX packets which inline
+			 * processor was unable to process
+			 */
+			if (pg->id.val & IPSEC_SA_OFFLOAD_FALLBACK_FLAG) {
+				/* offload packets to cryptodev */
+				struct rte_ipsec_session *fallback;
+
+				fallback = ipsec_get_fallback_session(sa);
+				if (fallback->crypto.ses == NULL &&
+					fill_ipsec_session(fallback, ctx, sa)
+					!= 0)
+					k = 0;
+				else
+					k = ipsec_prepare_crypto_group(ctx, sa,
+						fallback, pg->m, pg->cnt);
+			} else {
+				/* finish processing of packets successfully
+				 * decrypted by an inline processor
+				 */
+				k = rte_ipsec_pkt_process(ips, pg->m, pg->cnt);
+				copy_to_trf(trf, satp, pg->m, k);
 
+			}
 		/* enqueue packets to crypto dev */
 		} else {
-
-			cqp = &ctx->tbl[sa->cdev_id_qp];
-
-			/* for that app each mbuf has it's own crypto op */
-			for (j = 0; j != pg->cnt; j++) {
-				priv = get_priv(pg->m[j]);
-				cop[j] = &priv->cop;
-				/*
-				 * this is just to satisfy inbound_sa_check()
-				 * should be removed in future.
-				 */
-				priv->sa = sa;
-			}
-
-			/* prepare and enqueue crypto ops */
-			k = rte_ipsec_pkt_crypto_prepare(ips, pg->m, cop,
+			k = ipsec_prepare_crypto_group(ctx, sa, ips, pg->m,
 				pg->cnt);
-			if (k != 0)
-				enqueue_cop_bulk(cqp, cop, k);
 		}
 
 		/* drop packets that cannot be enqueued/processed */
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index a3d8e4545..0c8db6349 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -31,6 +31,8 @@
 
 #define IP6_FULL_MASK (sizeof(((struct ip_addr *)NULL)->ip.ip6.ip6) * CHAR_BIT)
 
+#define MBUF_NO_SEC_OFFLOAD(m) ((m->ol_flags & PKT_RX_SEC_OFFLOAD) == 0)
+
 struct supported_cipher_algo {
 	const char *keyword;
 	enum rte_crypto_cipher_algorithm algo;
@@ -235,6 +237,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	uint32_t mode_p = 0;
 	uint32_t type_p = 0;
 	uint32_t portid_p = 0;
+	uint32_t fallback_p = 0;
 
 	if (strcmp(tokens[0], "in") == 0) {
 		ri = &nb_sa_in;
@@ -263,7 +266,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	if (atoi(tokens[1]) == INVALID_SPI)
 		return;
 	rule->spi = atoi(tokens[1]);
-	ips = ipsec_get_session(rule);
+	ips = ipsec_get_primary_session(rule);
 
 	for (ti = 2; ti < n_tokens; ti++) {
 		if (strcmp(tokens[ti], "mode") == 0) {
@@ -596,6 +599,37 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 			continue;
 		}
 
+		if (strcmp(tokens[ti], "fallback") == 0) {
+			struct rte_ipsec_session *fb;
+
+			APP_CHECK(app_sa_prm.enable, status, "Fallback session "
+				"not allowed for legacy mode.");
+			if (status->status < 0)
+				return;
+			APP_CHECK_PRESENCE(fallback_p, tokens[ti], status);
+			if (status->status < 0)
+				return;
+			INCREMENT_TOKEN_INDEX(ti, n_tokens, status);
+			if (status->status < 0)
+				return;
+			fb = ipsec_get_fallback_session(rule);
+			if (strcmp(tokens[ti], "lookaside-none") == 0) {
+				fb->type = RTE_SECURITY_ACTION_TYPE_NONE;
+			} else if (strcmp(tokens[ti], "lookaside-protocol")
+				== 0) {
+				fb->type =
+				RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL;
+			} else {
+				APP_CHECK(0, status, "unrecognized fallback "
+					"type %s.", tokens[ti]);
+				return;
+			}
+
+			rule->fallback_sessions = 1;
+			fallback_p = 1;
+			continue;
+		}
+
 		/* unrecognizeable input */
 		APP_CHECK(0, status, "unrecognized input \"%s\"",
 			tokens[ti]);
@@ -643,6 +677,7 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 	uint32_t i;
 	uint8_t a, b, c, d;
 	const struct rte_ipsec_session *ips;
+	const struct rte_ipsec_session *fallback_ips;
 
 	printf("\tspi_%s(%3u):", inbound?"in":"out", sa->spi);
 
@@ -699,7 +734,7 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 		break;
 	}
 
-	ips = &sa->ips;
+	ips = &sa->sessions[IPSEC_SESSION_PRIMARY];
 	printf(" type:");
 	switch (ips->type) {
 	case RTE_SECURITY_ACTION_TYPE_NONE:
@@ -715,6 +750,22 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 		printf("lookaside-protocol-offload ");
 		break;
 	}
+
+	fallback_ips = &sa->sessions[IPSEC_SESSION_FALLBACK];
+	if (fallback_ips != NULL && sa->fallback_sessions > 0) {
+		printf("inline fallback:");
+		switch (fallback_ips->type) {
+		case RTE_SECURITY_ACTION_TYPE_NONE:
+			printf("lookaside-none ");
+			break;
+		case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL:
+			printf("lookaside-protocol ");
+			break;
+		default:
+			printf("invalid ");
+			break;
+		}
+	}
 	printf("\n");
 }
 
@@ -896,7 +947,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 		}
 		*sa = entries[i];
 		sa->seq = 0;
-		ips = ipsec_get_session(sa);
+		ips = ipsec_get_primary_session(sa);
 
 		if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
 			ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
@@ -946,7 +997,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			sa->xforms = &sa_ctx->xf[idx].a;
 
-			ips = ipsec_get_session(sa);
+			ips = ipsec_get_primary_session(sa);
 			if (ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
 				ips->type ==
@@ -1160,9 +1211,15 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	/* init processing session */
-	ips = ipsec_get_session(lsa);
+	/* init primary processing session */
+	ips = ipsec_get_primary_session(lsa);
 	rc = fill_ipsec_session(ips, sa);
+	if (rc != 0)
+		return rc;
+
+	/* init inline fallback processing session */
+	if (lsa->fallback_sessions == 1)
+		rc = fill_ipsec_session(ipsec_get_fallback_session(lsa), sa);
 
 	return rc;
 }
@@ -1318,13 +1375,14 @@ inbound_sa_check(struct sa_ctx *sa_ctx, struct rte_mbuf *m, uint32_t sa_idx)
 
 static inline void
 single_inbound_lookup(struct ipsec_sa *sadb, struct rte_mbuf *pkt,
-		struct ipsec_sa **sa_ret)
+		void **sa_ret)
 {
 	struct rte_esp_hdr *esp;
 	struct ip *ip;
 	uint32_t *src4_addr;
 	uint8_t *src6_addr;
 	struct ipsec_sa *sa;
+	void *result_sa;
 
 	*sa_ret = NULL;
 
@@ -1334,33 +1392,48 @@ single_inbound_lookup(struct ipsec_sa *sadb, struct rte_mbuf *pkt,
 	if (esp->spi == INVALID_SPI)
 		return;
 
-	sa = &sadb[SPI2IDX(rte_be_to_cpu_32(esp->spi))];
+	result_sa = sa = &sadb[SPI2IDX(rte_be_to_cpu_32(esp->spi))];
 	if (rte_be_to_cpu_32(esp->spi) != sa->spi)
 		return;
 
+	/*
+	 * Mark need for inline offload fallback on the LSB of SA pointer.
+	 * Thanks to packet grouping mechanism which ipsec_process is using
+	 * packets marked for fallback processing will form separate group.
+	 *
+	 * Because it is not safe to use SA pointer it is casted to generic
+	 * pointer to prevent from unintentional use. Use ipsec_mask_saptr
+	 * to get valid struct pointer.
+	 */
+	if (MBUF_NO_SEC_OFFLOAD(pkt) && sa->fallback_sessions > 0) {
+		uintptr_t intsa = (uintptr_t)sa;
+		intsa |= IPSEC_SA_OFFLOAD_FALLBACK_FLAG;
+		result_sa = (void *)intsa;
+	}
+
 	switch (WITHOUT_TRANSPORT_VERSION(sa->flags)) {
 	case IP4_TUNNEL:
 		src4_addr = RTE_PTR_ADD(ip, offsetof(struct ip, ip_src));
 		if ((ip->ip_v == IPVERSION) &&
 				(sa->src.ip.ip4 == *src4_addr) &&
 				(sa->dst.ip.ip4 == *(src4_addr + 1)))
-			*sa_ret = sa;
+			*sa_ret = result_sa;
 		break;
 	case IP6_TUNNEL:
 		src6_addr = RTE_PTR_ADD(ip, offsetof(struct ip6_hdr, ip6_src));
 		if ((ip->ip_v == IP6_VERSION) &&
 				!memcmp(&sa->src.ip.ip6.ip6, src6_addr, 16) &&
 				!memcmp(&sa->dst.ip.ip6.ip6, src6_addr + 16, 16))
-			*sa_ret = sa;
+			*sa_ret = result_sa;
 		break;
 	case TRANSPORT:
-		*sa_ret = sa;
+		*sa_ret = result_sa;
 	}
 }
 
 void
 inbound_sa_lookup(struct sa_ctx *sa_ctx, struct rte_mbuf *pkts[],
-		struct ipsec_sa *sa[], uint16_t nb_pkts)
+		void *sa[], uint16_t nb_pkts)
 {
 	uint32_t i;
 
@@ -1370,7 +1443,7 @@ inbound_sa_lookup(struct sa_ctx *sa_ctx, struct rte_mbuf *pkts[],
 
 void
 outbound_sa_lookup(struct sa_ctx *sa_ctx, uint32_t sa_idx[],
-		struct ipsec_sa *sa[], uint16_t nb_pkts)
+		void *sa[], uint16_t nb_pkts)
 {
 	uint32_t i;
 
-- 
2.17.1

[dpdk-dev] [PATCH v2 0/3] examples/ipsec-secgw: add fallback session

[Marcin Smoczynski]

Inline processing is limited to a specified subset of traffic. It is
often unable to handle more complicated situations, such as fragmented
traffic. When using inline processing such traffic is dropped.

Introduce multiple sessions per SA allowing to configure a fallback
lookaside session for packets that normally would be dropped.
A fallback session type in the SA configuration by adding 'fallback'
with 'lookaside-none' or 'lookaside-protocol' parameter to determine
type of session.

Fallback session feature is available only when using librte_ipsec.

v1 to v2 changes:
 - disable fallback offload for outbound SAs
 - add test scripts

Marcin Smoczynski (3):
  examples/ipsec-secgw: ipsec_sa structure cleanup
  examples/ipsec-secgw: add fallback session feature
  examples/ipsec-secgw: add offload fallback tests

 doc/guides/sample_app_ug/ipsec_secgw.rst      |  17 +-
 examples/ipsec-secgw/esp.c                    |  35 ++--
 examples/ipsec-secgw/ipsec-secgw.c            |  16 +-
 examples/ipsec-secgw/ipsec.c                  |  99 ++++++-----
 examples/ipsec-secgw/ipsec.h                  |  61 +++++--
 examples/ipsec-secgw/ipsec_process.c          | 113 +++++++-----
 examples/ipsec-secgw/sa.c                     | 164 +++++++++++++-----
 .../test/trs_aesgcm_common_defs.sh            |   4 +-
 .../trs_aesgcm_inline_crypto_fallback_defs.sh |   5 +
 .../test/tun_aesgcm_common_defs.sh            |   6 +-
 .../tun_aesgcm_inline_crypto_fallback_defs.sh |   5 +
 11 files changed, 358 insertions(+), 167 deletions(-)
 create mode 100644 examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
 create mode 100644 examples/ipsec-secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh

-- 
2.21.0.windows.1

[dpdk-dev] [PATCH v2 1/3] examples/ipsec-secgw: ipsec_sa structure cleanup

[Marcin Smoczynski]

Cleanup ipsec_sa structure by removing every field that is already in
the rte_ipsec_session structure:
 * cryptodev/security session union
 * action type
 * offload flags
 * security context
References to abovementioned fields are changed to direct references
to matching fields of rte_ipsec_session structure.

Such refactoring is needed to introduce many sessions per SA feature,
e.g. fallback session for inline offload processing.

Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
---
 examples/ipsec-secgw/esp.c           | 35 +++++++----
 examples/ipsec-secgw/ipsec.c         | 91 +++++++++++++++-------------
 examples/ipsec-secgw/ipsec.h         | 27 ++++++---
 examples/ipsec-secgw/ipsec_process.c | 30 ++++-----
 examples/ipsec-secgw/sa.c            | 66 ++++++++++----------
 5 files changed, 137 insertions(+), 112 deletions(-)

diff --git a/examples/ipsec-secgw/esp.c b/examples/ipsec-secgw/esp.c
index d6d7b1256..c1b49da1e 100644
--- a/examples/ipsec-secgw/esp.c
+++ b/examples/ipsec-secgw/esp.c
@@ -30,7 +30,8 @@ esp_inbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 	int32_t payload_len, ip_hdr_len;
 
 	RTE_ASSERT(sa != NULL);
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
+	if (ipsec_get_action_type(sa) ==
+			RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
 		return 0;
 
 	RTE_ASSERT(m != NULL);
@@ -148,13 +149,16 @@ esp_inbound_post(struct rte_mbuf *m, struct ipsec_sa *sa,
 	uint8_t *nexthdr, *pad_len;
 	uint8_t *padding;
 	uint16_t i;
+	struct rte_ipsec_session *ips;
 
 	RTE_ASSERT(m != NULL);
 	RTE_ASSERT(sa != NULL);
 	RTE_ASSERT(cop != NULL);
 
-	if ((sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
-			(sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
+	ips = ipsec_get_session(sa);
+
+	if ((ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
+			(ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
 		if (m->ol_flags & PKT_RX_SEC_OFFLOAD) {
 			if (m->ol_flags & PKT_RX_SEC_OFFLOAD_FAILED)
 				cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
@@ -169,8 +173,8 @@ esp_inbound_post(struct rte_mbuf *m, struct ipsec_sa *sa,
 		return -1;
 	}
 
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO &&
-	    sa->ol_flags & RTE_SECURITY_RX_HW_TRAILER_OFFLOAD) {
+	if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO &&
+		ips->security.ol_flags & RTE_SECURITY_RX_HW_TRAILER_OFFLOAD) {
 		nexthdr = &m->inner_esp_next_proto;
 	} else {
 		nexthdr = rte_pktmbuf_mtod_offset(m, uint8_t*,
@@ -225,10 +229,12 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 	struct rte_crypto_sym_op *sym_cop;
 	int32_t i;
 	uint16_t pad_payload_len, pad_len, ip_hdr_len;
+	struct rte_ipsec_session *ips;
 
 	RTE_ASSERT(m != NULL);
 	RTE_ASSERT(sa != NULL);
 
+	ips = ipsec_get_session(sa);
 	ip_hdr_len = 0;
 
 	ip4 = rte_pktmbuf_mtod(m, struct ip *);
@@ -277,9 +283,10 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 	}
 
 	/* Add trailer padding if it is not constructed by HW */
-	if (sa->type != RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
-	    (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO &&
-	     !(sa->ol_flags & RTE_SECURITY_TX_HW_TRAILER_OFFLOAD))) {
+	if (ips->type != RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+		(ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO &&
+		 !(ips->security.ol_flags &
+			 RTE_SECURITY_TX_HW_TRAILER_OFFLOAD))) {
 		padding = (uint8_t *)rte_pktmbuf_append(m, pad_len +
 							sa->digest_len);
 		if (unlikely(padding == NULL)) {
@@ -344,8 +351,9 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 		}
 	}
 
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
-		if (sa->ol_flags & RTE_SECURITY_TX_HW_TRAILER_OFFLOAD) {
+	if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		if (ips->security.ol_flags &
+				RTE_SECURITY_TX_HW_TRAILER_OFFLOAD) {
 			/* Set the inner esp next protocol for HW trailer */
 			m->inner_esp_next_proto = nlp;
 			m->packet_type |= RTE_PTYPE_TUNNEL_ESP;
@@ -448,11 +456,14 @@ esp_outbound_post(struct rte_mbuf *m,
 		  struct ipsec_sa *sa,
 		  struct rte_crypto_op *cop)
 {
+	enum rte_security_session_action_type type;
 	RTE_ASSERT(m != NULL);
 	RTE_ASSERT(sa != NULL);
 
-	if ((sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
-			(sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
+	type = ipsec_get_action_type(sa);
+
+	if ((type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
+			(type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
 		m->ol_flags |= PKT_TX_SEC_OFFLOAD;
 	} else {
 		RTE_ASSERT(cop != NULL);
diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index dc85adfe5..8c60bd84b 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -40,7 +40,8 @@ set_ipsec_conf(struct ipsec_sa *sa, struct rte_security_ipsec_xform *ipsec)
 }
 
 int
-create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips)
 {
 	struct rte_cryptodev_info cdev_info;
 	unsigned long cdev_id_qp = 0;
@@ -71,9 +72,9 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			ipsec_ctx->tbl[cdev_id_qp].id,
 			ipsec_ctx->tbl[cdev_id_qp].qp);
 
-	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
+	if (ips->type != RTE_SECURITY_ACTION_TYPE_NONE) {
 		struct rte_security_session_conf sess_conf = {
-			.action_type = sa->type,
+			.action_type = ips->type,
 			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 			{.ipsec = {
 				.spi = sa->spi,
@@ -90,7 +91,7 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 
 		};
 
-		if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
+		if (ips->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
 			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
 							rte_cryptodev_get_sec_ctx(
 							ipsec_ctx->tbl[cdev_id_qp].id);
@@ -98,9 +99,9 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			/* Set IPsec parameters in conf */
 			set_ipsec_conf(sa, &(sess_conf.ipsec));
 
-			sa->sec_session = rte_security_session_create(ctx,
+			ips->security.ses = rte_security_session_create(ctx,
 					&sess_conf, ipsec_ctx->session_priv_pool);
-			if (sa->sec_session == NULL) {
+			if (ips->security.ses == NULL) {
 				RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 				return -1;
@@ -110,10 +111,10 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			return -1;
 		}
 	} else {
-		sa->crypto_session = rte_cryptodev_sym_session_create(
+		ips->crypto.ses = rte_cryptodev_sym_session_create(
 				ipsec_ctx->session_pool);
 		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
-				sa->crypto_session, sa->xforms,
+				ips->crypto.ses, sa->xforms,
 				ipsec_ctx->session_priv_pool);
 
 		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
@@ -126,12 +127,13 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 }
 
 int
-create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips)
 {
 	int32_t ret = 0;
 	struct rte_security_ctx *sec_ctx;
 	struct rte_security_session_conf sess_conf = {
-		.action_type = sa->type,
+		.action_type = ips->type,
 		.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 		{.ipsec = {
 			.spi = sa->spi,
@@ -151,7 +153,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on port %u\n",
 		sa->spi, sa->portid);
 
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+	if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
 		struct rte_flow_error err;
 		const struct rte_security_capability *sec_cap;
 		int ret = 0;
@@ -165,9 +167,9 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 			return -1;
 		}
 
-		sa->sec_session = rte_security_session_create(sec_ctx,
+		ips->security.ses = rte_security_session_create(sec_ctx,
 				&sess_conf, skt_ctx->session_pool);
-		if (sa->sec_session == NULL) {
+		if (ips->security.ses == NULL) {
 			RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 			return -1;
@@ -177,7 +179,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 
 		/* iterate until ESP tunnel*/
 		while (sec_cap->action != RTE_SECURITY_ACTION_TYPE_NONE) {
-			if (sec_cap->action == sa->type &&
+			if (sec_cap->action == ips->type &&
 			    sec_cap->protocol ==
 				RTE_SECURITY_PROTOCOL_IPSEC &&
 			    sec_cap->ipsec.mode ==
@@ -193,8 +195,8 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 			return -1;
 		}
 
-		sa->ol_flags = sec_cap->ol_flags;
-		sa->security_ctx = sec_ctx;
+		ips->security.ol_flags = sec_cap->ol_flags;
+		ips->security.ctx = sec_ctx;
 		sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
 
 		if (IS_IP6(sa->flags)) {
@@ -223,7 +225,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 		sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
 
 		sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
-		sa->action[0].conf = sa->sec_session;
+		sa->action[0].conf = ips->security.ses;
 
 		sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
 
@@ -282,8 +284,8 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 			if (ret)
 				goto flow_create_failure;
 		} else if (sa->attr.egress &&
-			   (sa->ol_flags &
-				    RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
+				(ips->security.ol_flags &
+					RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
 			sa->action[1].type =
 					RTE_FLOW_ACTION_TYPE_PASSTHRU;
 			sa->action[2].type =
@@ -299,7 +301,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 				err.message);
 			return -1;
 		}
-	} else if (sa->type ==	RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
+	} else if (ips->type ==	RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
 		const struct rte_security_capability *sec_cap;
 
 		sec_ctx = (struct rte_security_ctx *)
@@ -327,9 +329,9 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 
 		sess_conf.userdata = (void *) sa;
 
-		sa->sec_session = rte_security_session_create(sec_ctx,
+		ips->security.ses = rte_security_session_create(sec_ctx,
 					&sess_conf, skt_ctx->session_pool);
-		if (sa->sec_session == NULL) {
+		if (ips->security.ses == NULL) {
 			RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 			return -1;
@@ -345,7 +347,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 		/* iterate until ESP tunnel*/
 		while (sec_cap->action !=
 				RTE_SECURITY_ACTION_TYPE_NONE) {
-			if (sec_cap->action == sa->type &&
+			if (sec_cap->action == ips->type &&
 			    sec_cap->protocol ==
 				RTE_SECURITY_PROTOCOL_IPSEC &&
 			    sec_cap->ipsec.mode ==
@@ -361,8 +363,8 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 			return -1;
 		}
 
-		sa->ol_flags = sec_cap->ol_flags;
-		sa->security_ctx = sec_ctx;
+		ips->security.ol_flags = sec_cap->ol_flags;
+		ips->security.ctx = sec_ctx;
 	}
 	sa->cdev_id_qp = 0;
 
@@ -409,6 +411,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 	struct ipsec_mbuf_metadata *priv;
 	struct rte_crypto_sym_op *sym_cop;
 	struct ipsec_sa *sa;
+	struct rte_ipsec_session *ips;
 
 	for (i = 0; i < nb_pkts; i++) {
 		if (unlikely(sas[i] == NULL)) {
@@ -422,16 +425,17 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 		priv = get_priv(pkts[i]);
 		sa = sas[i];
 		priv->sa = sa;
+		ips = ipsec_get_session(sa);
 
-		switch (sa->type) {
+		switch (ips->type) {
 		case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL:
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
 
-			if ((unlikely(sa->sec_session == NULL)) &&
-				create_lookaside_session(ipsec_ctx, sa)) {
+			if ((unlikely(ips->security.ses == NULL)) &&
+				create_lookaside_session(ipsec_ctx, sa, ips)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -440,7 +444,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			sym_cop->m_src = pkts[i];
 
 			rte_security_attach_session(&priv->cop,
-					sa->sec_session);
+				ips->security.ses);
 			break;
 		case RTE_SECURITY_ACTION_TYPE_NONE:
 
@@ -449,14 +453,14 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			rte_prefetch0(&priv->sym_cop);
 
-			if ((unlikely(sa->crypto_session == NULL)) &&
-				create_lookaside_session(ipsec_ctx, sa)) {
+			if ((unlikely(ips->crypto.ses == NULL)) &&
+				create_lookaside_session(ipsec_ctx, sa, ips)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
 
 			rte_crypto_op_attach_sym_session(&priv->cop,
-					sa->crypto_session);
+					ips->crypto.ses);
 
 			ret = xform_func(pkts[i], sa, &priv->cop);
 			if (unlikely(ret)) {
@@ -465,21 +469,22 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 			break;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
-			RTE_ASSERT(sa->sec_session != NULL);
+			RTE_ASSERT(ips->security.ses != NULL);
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
-			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
+			if (ips->security.ol_flags &
+				RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
-						sa->security_ctx,
-						sa->sec_session, pkts[i], NULL);
+					ips->security.ctx, ips->security.ses,
+					pkts[i], NULL);
 			continue;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
-			RTE_ASSERT(sa->sec_session != NULL);
+			RTE_ASSERT(ips->security.ses != NULL);
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
 			rte_security_attach_session(&priv->cop,
-					sa->sec_session);
+					ips->security.ses);
 
 			ret = xform_func(pkts[i], sa, &priv->cop);
 			if (unlikely(ret)) {
@@ -488,10 +493,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
-			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
+			if (ips->security.ol_flags &
+				RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
-						sa->security_ctx,
-						sa->sec_session, pkts[i], NULL);
+					ips->security.ctx, ips->security.ses,
+					pkts[i], NULL);
 			continue;
 		}
 
@@ -560,7 +566,8 @@ ipsec_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			RTE_ASSERT(sa != NULL);
 
-			if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {
+			if (ipsec_get_action_type(sa) ==
+				RTE_SECURITY_ACTION_TYPE_NONE) {
 				ret = xform_func(pkt, sa, cops[j]);
 				if (unlikely(ret)) {
 					rte_pktmbuf_free(pkt);
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index 1efa6e488..a4ad81b0e 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -87,10 +87,6 @@ struct ipsec_sa {
 	uint32_t cdev_id_qp;
 	uint64_t seq;
 	uint32_t salt;
-	union {
-		struct rte_cryptodev_sym_session *crypto_session;
-		struct rte_security_session *sec_session;
-	};
 	enum rte_crypto_cipher_algorithm cipher_algo;
 	enum rte_crypto_auth_algorithm auth_algo;
 	enum rte_crypto_aead_algorithm aead_algo;
@@ -114,11 +110,8 @@ struct ipsec_sa {
 		struct rte_crypto_sym_xform *xforms;
 		struct rte_security_ipsec_xform *sec_xform;
 	};
-	enum rte_security_session_action_type type;
 	enum rte_security_ipsec_sa_direction direction;
 	uint16_t portid;
-	struct rte_security_ctx *security_ctx;
-	uint32_t ol_flags;
 
 #define MAX_RTE_FLOW_PATTERN (4)
 #define MAX_RTE_FLOW_ACTIONS (3)
@@ -284,6 +277,20 @@ get_sym_cop(struct rte_crypto_op *cop)
 	return (cop + 1);
 }
 
+static inline struct rte_ipsec_session *
+ipsec_get_session(struct ipsec_sa *sa)
+{
+	return &sa->ips;
+}
+
+static inline enum rte_security_session_action_type
+ipsec_get_action_type(struct ipsec_sa *sa)
+{
+	struct rte_ipsec_session *ips;
+	ips = ipsec_get_session(sa);
+	return ips->type;
+}
+
 int
 inbound_sa_check(struct sa_ctx *sa_ctx, struct rte_mbuf *m, uint32_t sa_idx);
 
@@ -338,9 +345,11 @@ void
 enqueue_cop_burst(struct cdev_qp *cqp);
 
 int
-create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips);
 
 int
-create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa);
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips);
 
 #endif /* __IPSEC_H__ */
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index 868f1a28d..239d81ef6 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -94,22 +94,16 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
 
 	/* setup crypto section */
 	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		if (sa->crypto_session == NULL) {
-			rc = create_lookaside_session(ctx, sa);
-			if (rc != 0)
-				return rc;
-		}
-		ss->crypto.ses = sa->crypto_session;
+		RTE_ASSERT(ss->crypto.ses == NULL);
+		rc = create_lookaside_session(ctx, sa, ss);
+		if (rc != 0)
+			return rc;
 	/* setup session action type */
-	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
-		if (sa->sec_session == NULL) {
-			rc = create_lookaside_session(ctx, sa);
-			if (rc != 0)
-				return rc;
-		}
-		ss->security.ses = sa->sec_session;
-		ss->security.ctx = sa->security_ctx;
-		ss->security.ol_flags = sa->ol_flags;
+	} else if (ss->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
+		RTE_ASSERT(ss->security.ses == NULL);
+		rc = create_lookaside_session(ctx, sa, ss);
+		if (rc != 0)
+			return rc;
 	} else
 		RTE_ASSERT(0);
 
@@ -218,7 +212,7 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 		pg = grp + i;
 		sa = pg->id.ptr;
 
-		ips = &sa->ips;
+		ips = ipsec_get_session(sa);
 
 		/* no valid HW session for that SA, try to create one */
 		if (sa == NULL || (ips->crypto.ses == NULL &&
@@ -226,8 +220,8 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 			k = 0;
 
 		/* process packets inline */
-		else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
-				sa->type ==
+		else if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+				ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
 
 			satp = rte_ipsec_sa_type(ips->sa);
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index c3cf3bd1f..a3d8e4545 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -224,6 +224,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	struct parse_status *status)
 {
 	struct ipsec_sa *rule = NULL;
+	struct rte_ipsec_session *ips;
 	uint32_t ti; /*token index*/
 	uint32_t *ri /*rule index*/;
 	uint32_t cipher_algo_p = 0;
@@ -262,6 +263,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	if (atoi(tokens[1]) == INVALID_SPI)
 		return;
 	rule->spi = atoi(tokens[1]);
+	ips = ipsec_get_session(rule);
 
 	for (ti = 2; ti < n_tokens; ti++) {
 		if (strcmp(tokens[ti], "mode") == 0) {
@@ -558,18 +560,18 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 				return;
 
 			if (strcmp(tokens[ti], "inline-crypto-offload") == 0)
-				rule->type =
+				ips->type =
 					RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO;
 			else if (strcmp(tokens[ti],
 					"inline-protocol-offload") == 0)
-				rule->type =
+				ips->type =
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL;
 			else if (strcmp(tokens[ti],
 					"lookaside-protocol-offload") == 0)
-				rule->type =
+				ips->type =
 				RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL;
 			else if (strcmp(tokens[ti], "no-offload") == 0)
-				rule->type = RTE_SECURITY_ACTION_TYPE_NONE;
+				ips->type = RTE_SECURITY_ACTION_TYPE_NONE;
 			else {
 				APP_CHECK(0, status, "Invalid input \"%s\"",
 						tokens[ti]);
@@ -624,11 +626,11 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	if (status->status < 0)
 		return;
 
-	if ((rule->type != RTE_SECURITY_ACTION_TYPE_NONE) && (portid_p == 0))
+	if ((ips->type != RTE_SECURITY_ACTION_TYPE_NONE) && (portid_p == 0))
 		printf("Missing portid option, falling back to non-offload\n");
 
 	if (!type_p || !portid_p) {
-		rule->type = RTE_SECURITY_ACTION_TYPE_NONE;
+		ips->type = RTE_SECURITY_ACTION_TYPE_NONE;
 		rule->portid = -1;
 	}
 
@@ -640,6 +642,7 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 {
 	uint32_t i;
 	uint8_t a, b, c, d;
+	const struct rte_ipsec_session *ips;
 
 	printf("\tspi_%s(%3u):", inbound?"in":"out", sa->spi);
 
@@ -695,8 +698,10 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 		printf("Transport ");
 		break;
 	}
+
+	ips = &sa->ips;
 	printf(" type:");
-	switch (sa->type) {
+	switch (ips->type) {
 	case RTE_SECURITY_ACTION_TYPE_NONE:
 		printf("no-offload ");
 		break;
@@ -876,6 +881,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 	uint16_t iv_length, aad_length;
 	int inline_status;
 	int32_t rc;
+	struct rte_ipsec_session *ips;
 
 	/* for ESN upper 32 bits of SQN also need to be part of AAD */
 	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
@@ -890,9 +896,10 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 		}
 		*sa = entries[i];
 		sa->seq = 0;
+		ips = ipsec_get_session(sa);
 
-		if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
-			sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
+			ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
 			if (check_eth_dev_caps(sa->portid, inbound))
 				return -EINVAL;
 		}
@@ -907,7 +914,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 			sa->dst.ip.ip4 = rte_cpu_to_be_32(sa->dst.ip.ip4);
 			break;
 		case TRANSPORT:
-			if (sa->type ==
+			if (ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
 				inline_status =
 					sa_add_address_inline_crypto(sa);
@@ -918,6 +925,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 		}
 
 		if (sa->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) {
+			struct rte_ipsec_session *ips;
 			iv_length = 16;
 
 			sa_ctx->xf[idx].a.type = RTE_CRYPTO_SYM_XFORM_AEAD;
@@ -938,11 +946,12 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			sa->xforms = &sa_ctx->xf[idx].a;
 
-			if (sa->type ==
+			ips = ipsec_get_session(sa);
+			if (ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
-				sa->type ==
+				ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
-				rc = create_inline_session(skt_ctx, sa);
+				rc = create_inline_session(skt_ctx, sa, ips);
 				if (rc != 0) {
 					RTE_LOG(ERR, IPSEC_ESP,
 						"create_inline_session() failed\n");
@@ -1100,23 +1109,11 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm, const struct ipsec_sa *ss,
 }
 
 static int
-fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
-	const struct ipsec_sa *lsa)
+fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa)
 {
 	int32_t rc = 0;
 
 	ss->sa = sa;
-	ss->type = lsa->type;
-
-	/* setup crypto section */
-	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		ss->crypto.ses = lsa->crypto_session;
-	/* setup session action type */
-	} else {
-		ss->security.ses = lsa->sec_session;
-		ss->security.ctx = lsa->security_ctx;
-		ss->security.ol_flags = lsa->ol_flags;
-	}
 
 	if (ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
 		ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
@@ -1138,6 +1135,7 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 {
 	int rc;
 	struct rte_ipsec_sa_prm prm;
+	struct rte_ipsec_session *ips;
 	struct rte_ipv4_hdr v4  = {
 		.version_ihl = IPVERSION << 4 |
 			sizeof(v4) / RTE_IPV4_IHL_MULTIPLIER,
@@ -1162,7 +1160,10 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
+	/* init processing session */
+	ips = ipsec_get_session(lsa);
+	rc = fill_ipsec_session(ips, sa);
+
 	return rc;
 }
 
@@ -1386,6 +1387,7 @@ sa_check_offloads(uint16_t port_id, uint64_t *rx_offloads,
 {
 	struct ipsec_sa *rule;
 	uint32_t idx_sa;
+	enum rte_security_session_action_type rule_type;
 
 	*rx_offloads = 0;
 	*tx_offloads = 0;
@@ -1393,8 +1395,9 @@ sa_check_offloads(uint16_t port_id, uint64_t *rx_offloads,
 	/* Check for inbound rules that use offloads and use this port */
 	for (idx_sa = 0; idx_sa < nb_sa_in; idx_sa++) {
 		rule = &sa_in[idx_sa];
-		if ((rule->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
-				rule->type ==
+		rule_type = ipsec_get_action_type(rule);
+		if ((rule_type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+				rule_type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
 				&& rule->portid == port_id)
 			*rx_offloads |= DEV_RX_OFFLOAD_SECURITY;
@@ -1403,8 +1406,9 @@ sa_check_offloads(uint16_t port_id, uint64_t *rx_offloads,
 	/* Check for outbound rules that use offloads and use this port */
 	for (idx_sa = 0; idx_sa < nb_sa_out; idx_sa++) {
 		rule = &sa_out[idx_sa];
-		if ((rule->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
-				rule->type ==
+		rule_type = ipsec_get_action_type(rule);
+		if ((rule_type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+				rule_type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
 				&& rule->portid == port_id)
 			*tx_offloads |= DEV_TX_OFFLOAD_SECURITY;
-- 
2.21.0.windows.1

[dpdk-dev] [PATCH v2 2/3] examples/ipsec-secgw: add fallback session feature

[Marcin Smoczynski]

Inline processing is limited to a specified subset of traffic. It is
often unable to handle more complicated situations, such as fragmented
traffic. When using inline processing such traffic is dropped.

Introduce multiple sessions per SA allowing to configure a fallback
lookaside session for packets that normally would be dropped.
A fallback session type in the SA configuration by adding 'fallback'
with 'lookaside-none' or 'lookaside-protocol' parameter to determine
type of session.

Fallback session feature is available only when using librte_ipsec.

Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
---
 doc/guides/sample_app_ug/ipsec_secgw.rst |  17 +++-
 examples/ipsec-secgw/esp.c               |   4 +-
 examples/ipsec-secgw/ipsec-secgw.c       |  16 ++--
 examples/ipsec-secgw/ipsec.c             |  10 +--
 examples/ipsec-secgw/ipsec.h             |  40 +++++++--
 examples/ipsec-secgw/ipsec_process.c     |  85 ++++++++++++------
 examples/ipsec-secgw/sa.c                | 110 +++++++++++++++++++----
 7 files changed, 218 insertions(+), 64 deletions(-)

diff --git a/doc/guides/sample_app_ug/ipsec_secgw.rst b/doc/guides/sample_app_ug/ipsec_secgw.rst
index ad2d79e75..9ded5fb70 100644
--- a/doc/guides/sample_app_ug/ipsec_secgw.rst
+++ b/doc/guides/sample_app_ug/ipsec_secgw.rst
@@ -401,7 +401,7 @@ The SA rule syntax is shown as follows:
 .. code-block:: console
 
     sa <dir> <spi> <cipher_algo> <cipher_key> <auth_algo> <auth_key>
-    <mode> <src_ip> <dst_ip> <action_type> <port_id>
+    <mode> <src_ip> <dst_ip> <action_type> <port_id> <fallback>
 
 where each options means:
 
@@ -573,6 +573,21 @@ where each options means:
 
    * *port_id X* X is a valid device number in decimal
 
+ ``<fallback>``
+
+ * Action type for packets that inline processor failed to process.
+
+ * Optional: Yes, by default there is *no fallback* session type.
+
+ * Available options:
+
+   * *lookaside-none*: use automatically chosen cryptodev to process packets
+   * *lookaside-protocol*: lookaside protocol offload to HW accelerator
+
+ * Syntax:
+
+   * *fallback lookaside-none*
+   * *fallback lookaside-protocol*
 
 Example SA rules:
 
diff --git a/examples/ipsec-secgw/esp.c b/examples/ipsec-secgw/esp.c
index c1b49da1e..bfa7ff721 100644
--- a/examples/ipsec-secgw/esp.c
+++ b/examples/ipsec-secgw/esp.c
@@ -155,7 +155,7 @@ esp_inbound_post(struct rte_mbuf *m, struct ipsec_sa *sa,
 	RTE_ASSERT(sa != NULL);
 	RTE_ASSERT(cop != NULL);
 
-	ips = ipsec_get_session(sa);
+	ips = ipsec_get_primary_session(sa);
 
 	if ((ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
 			(ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
@@ -234,7 +234,7 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 	RTE_ASSERT(m != NULL);
 	RTE_ASSERT(sa != NULL);
 
-	ips = ipsec_get_session(sa);
+	ips = ipsec_get_primary_session(sa);
 	ip_hdr_len = 0;
 
 	ip4 = rte_pktmbuf_mtod(m, struct ip *);
diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index 0d1fd6af6..641ed3767 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -189,6 +189,7 @@ static uint32_t mtu_size = RTE_ETHER_MTU;
 
 /* application wide librte_ipsec/SA parameters */
 struct app_sa_prm app_sa_prm = {.enable = 0};
+static const char *cfgfile;
 
 struct lcore_rx_queue {
 	uint16_t port_id;
@@ -1462,12 +1463,7 @@ parse_args(int32_t argc, char **argv)
 				print_usage(prgname);
 				return -1;
 			}
-			if (parse_cfg_file(optarg) < 0) {
-				printf("parsing file \"%s\" failed\n",
-					optarg);
-				print_usage(prgname);
-				return -1;
-			}
+			cfgfile = optarg;
 			f_present = 1;
 			break;
 		case 'j':
@@ -2398,6 +2394,14 @@ main(int32_t argc, char **argv)
 	if (ret < 0)
 		rte_exit(EXIT_FAILURE, "Invalid parameters\n");
 
+	/* parse configuration file */
+	if (parse_cfg_file(cfgfile) < 0) {
+		printf("parsing file \"%s\" failed\n",
+			optarg);
+		print_usage(argv[0]);
+		return -1;
+	}
+
 	if ((unprotected_port_mask & enabled_port_mask) !=
 			unprotected_port_mask)
 		rte_exit(EXIT_FAILURE, "Invalid unprotected portmask 0x%x\n",
diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index 8c60bd84b..8b0441028 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -404,7 +404,7 @@ enqueue_cop(struct cdev_qp *cqp, struct rte_crypto_op *cop)
 
 static inline void
 ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
-		struct rte_mbuf *pkts[], struct ipsec_sa *sas[],
+		struct rte_mbuf *pkts[], void *sas[],
 		uint16_t nb_pkts)
 {
 	int32_t ret = 0, i;
@@ -423,9 +423,9 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 		rte_prefetch0(pkts[i]);
 
 		priv = get_priv(pkts[i]);
-		sa = sas[i];
+		sa = ipsec_mask_saptr(sas[i]);
 		priv->sa = sa;
-		ips = ipsec_get_session(sa);
+		ips = ipsec_get_primary_session(sa);
 
 		switch (ips->type) {
 		case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL:
@@ -586,7 +586,7 @@ uint16_t
 ipsec_inbound(struct ipsec_ctx *ctx, struct rte_mbuf *pkts[],
 		uint16_t nb_pkts, uint16_t len)
 {
-	struct ipsec_sa *sas[nb_pkts];
+	void *sas[nb_pkts];
 
 	inbound_sa_lookup(ctx->sa_ctx, pkts, sas, nb_pkts);
 
@@ -606,7 +606,7 @@ uint16_t
 ipsec_outbound(struct ipsec_ctx *ctx, struct rte_mbuf *pkts[],
 		uint32_t sa_idx[], uint16_t nb_pkts, uint16_t len)
 {
-	struct ipsec_sa *sas[nb_pkts];
+	void *sas[nb_pkts];
 
 	outbound_sa_lookup(ctx->sa_ctx, sa_idx, sas, nb_pkts);
 
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index a4ad81b0e..8e075216c 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -81,12 +81,32 @@ struct app_sa_prm {
 
 extern struct app_sa_prm app_sa_prm;
 
+enum {
+	IPSEC_SESSION_PRIMARY = 0,
+	IPSEC_SESSION_FALLBACK = 1,
+	IPSEC_SESSION_MAX
+};
+
+#define IPSEC_SA_OFFLOAD_FALLBACK_FLAG (1)
+
+static inline struct ipsec_sa *
+ipsec_mask_saptr(void *ptr)
+{
+	uintptr_t i = (uintptr_t)ptr;
+	static const uintptr_t mask = IPSEC_SA_OFFLOAD_FALLBACK_FLAG;
+
+	i &= ~mask;
+
+	return (struct ipsec_sa *)i;
+}
+
 struct ipsec_sa {
-	struct rte_ipsec_session ips; /* one session per sa for now */
+	struct rte_ipsec_session sessions[IPSEC_SESSION_MAX];
 	uint32_t spi;
 	uint32_t cdev_id_qp;
 	uint64_t seq;
 	uint32_t salt;
+	uint32_t fallback_sessions;
 	enum rte_crypto_cipher_algorithm cipher_algo;
 	enum rte_crypto_auth_algorithm auth_algo;
 	enum rte_crypto_aead_algorithm aead_algo;
@@ -210,7 +230,7 @@ struct cnt_blk {
 struct traffic_type {
 	const uint8_t *data[MAX_PKT_BURST * 2];
 	struct rte_mbuf *pkts[MAX_PKT_BURST * 2];
-	struct ipsec_sa *saptr[MAX_PKT_BURST * 2];
+	void *saptr[MAX_PKT_BURST * 2];
 	uint32_t res[MAX_PKT_BURST * 2];
 	uint32_t num;
 };
@@ -278,16 +298,22 @@ get_sym_cop(struct rte_crypto_op *cop)
 }
 
 static inline struct rte_ipsec_session *
-ipsec_get_session(struct ipsec_sa *sa)
+ipsec_get_primary_session(struct ipsec_sa *sa)
+{
+	return &sa->sessions[IPSEC_SESSION_PRIMARY];
+}
+
+static inline struct rte_ipsec_session *
+ipsec_get_fallback_session(struct ipsec_sa *sa)
 {
-	return &sa->ips;
+	return &sa->sessions[IPSEC_SESSION_FALLBACK];
 }
 
 static inline enum rte_security_session_action_type
 ipsec_get_action_type(struct ipsec_sa *sa)
 {
 	struct rte_ipsec_session *ips;
-	ips = ipsec_get_session(sa);
+	ips = ipsec_get_primary_session(sa);
 	return ips->type;
 }
 
@@ -296,11 +322,11 @@ inbound_sa_check(struct sa_ctx *sa_ctx, struct rte_mbuf *m, uint32_t sa_idx);
 
 void
 inbound_sa_lookup(struct sa_ctx *sa_ctx, struct rte_mbuf *pkts[],
-		struct ipsec_sa *sa[], uint16_t nb_pkts);
+		void *sa[], uint16_t nb_pkts);
 
 void
 outbound_sa_lookup(struct sa_ctx *sa_ctx, uint32_t sa_idx[],
-		struct ipsec_sa *sa[], uint16_t nb_pkts);
+		void *sa[], uint16_t nb_pkts);
 
 void
 sp4_init(struct socket_ctx *ctx, int32_t socket_id);
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index 239d81ef6..2eb5c8b34 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -18,7 +18,6 @@
 	(((t) & RTE_IPSEC_SATP_IPV_MASK) == RTE_IPSEC_SATP_IPV4)) || \
 	((t) & RTE_IPSEC_SATP_MODE_MASK) == RTE_IPSEC_SATP_MODE_TUNLV4)
 
-
 /* helper routine to free bulk of packets */
 static inline void
 free_pkts(struct rte_mbuf *mb[], uint32_t n)
@@ -118,7 +117,7 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
  * group input packets byt the SA they belong to.
  */
 static uint32_t
-sa_group(struct ipsec_sa *sa_ptr[], struct rte_mbuf *pkts[],
+sa_group(void *sa_ptr[], struct rte_mbuf *pkts[],
 	struct rte_ipsec_group grp[], uint32_t num)
 {
 	uint32_t i, n, spi;
@@ -185,6 +184,37 @@ copy_to_trf(struct ipsec_traffic *trf, uint64_t satp, struct rte_mbuf *mb[],
 	out->num += num;
 }
 
+static uint32_t
+ipsec_prepare_crypto_group(struct ipsec_ctx *ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips, struct rte_mbuf **m,
+		unsigned int cnt)
+{
+	struct cdev_qp *cqp;
+	struct rte_crypto_op *cop[cnt];
+	uint32_t j, k;
+	struct ipsec_mbuf_metadata *priv;
+
+	cqp = &ctx->tbl[sa->cdev_id_qp];
+
+	/* for that app each mbuf has it's own crypto op */
+	for (j = 0; j != cnt; j++) {
+		priv = get_priv(m[j]);
+		cop[j] = &priv->cop;
+		/*
+		 * this is just to satisfy inbound_sa_check()
+		 * should be removed in future.
+		 */
+		priv->sa = sa;
+	}
+
+	/* prepare and enqueue crypto ops */
+	k = rte_ipsec_pkt_crypto_prepare(ips, m, cop, cnt);
+	if (k != 0)
+		enqueue_cop_bulk(cqp, cop, k);
+
+	return k;
+}
+
 /*
  * Process ipsec packets.
  * If packet belong to SA that is subject of inline-crypto,
@@ -201,18 +231,15 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 	struct ipsec_mbuf_metadata *priv;
 	struct rte_ipsec_group *pg;
 	struct rte_ipsec_session *ips;
-	struct cdev_qp *cqp;
-	struct rte_crypto_op *cop[RTE_DIM(trf->ipsec.pkts)];
 	struct rte_ipsec_group grp[RTE_DIM(trf->ipsec.pkts)];
 
 	n = sa_group(trf->ipsec.saptr, trf->ipsec.pkts, grp, trf->ipsec.num);
 
 	for (i = 0; i != n; i++) {
-
 		pg = grp + i;
-		sa = pg->id.ptr;
+		sa = ipsec_mask_saptr(pg->id.ptr);
 
-		ips = ipsec_get_session(sa);
+		ips = ipsec_get_primary_session(sa);
 
 		/* no valid HW session for that SA, try to create one */
 		if (sa == NULL || (ips->crypto.ses == NULL &&
@@ -224,6 +251,7 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 				ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
 
+			/* get SA type */
 			satp = rte_ipsec_sa_type(ips->sa);
 
 			/*
@@ -236,30 +264,33 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 				priv->sa = sa;
 			}
 
-			k = rte_ipsec_pkt_process(ips, pg->m, pg->cnt);
-			copy_to_trf(trf, satp, pg->m, k);
+			/* fallback to cryptodev with RX packets which inline
+			 * processor was unable to process
+			 */
+			if (pg->id.val & IPSEC_SA_OFFLOAD_FALLBACK_FLAG) {
+				/* offload packets to cryptodev */
+				struct rte_ipsec_session *fallback;
+
+				fallback = ipsec_get_fallback_session(sa);
+				if (fallback->crypto.ses == NULL &&
+					fill_ipsec_session(fallback, ctx, sa)
+					!= 0)
+					k = 0;
+				else
+					k = ipsec_prepare_crypto_group(ctx, sa,
+						fallback, pg->m, pg->cnt);
+			} else {
+				/* finish processing of packets successfully
+				 * decrypted by an inline processor
+				 */
+				k = rte_ipsec_pkt_process(ips, pg->m, pg->cnt);
+				copy_to_trf(trf, satp, pg->m, k);
 
+			}
 		/* enqueue packets to crypto dev */
 		} else {
-
-			cqp = &ctx->tbl[sa->cdev_id_qp];
-
-			/* for that app each mbuf has it's own crypto op */
-			for (j = 0; j != pg->cnt; j++) {
-				priv = get_priv(pg->m[j]);
-				cop[j] = &priv->cop;
-				/*
-				 * this is just to satisfy inbound_sa_check()
-				 * should be removed in future.
-				 */
-				priv->sa = sa;
-			}
-
-			/* prepare and enqueue crypto ops */
-			k = rte_ipsec_pkt_crypto_prepare(ips, pg->m, cop,
+			k = ipsec_prepare_crypto_group(ctx, sa, ips, pg->m,
 				pg->cnt);
-			if (k != 0)
-				enqueue_cop_bulk(cqp, cop, k);
 		}
 
 		/* drop packets that cannot be enqueued/processed */
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index a3d8e4545..700d52490 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -31,6 +31,8 @@
 
 #define IP6_FULL_MASK (sizeof(((struct ip_addr *)NULL)->ip.ip6.ip6) * CHAR_BIT)
 
+#define MBUF_NO_SEC_OFFLOAD(m) ((m->ol_flags & PKT_RX_SEC_OFFLOAD) == 0)
+
 struct supported_cipher_algo {
 	const char *keyword;
 	enum rte_crypto_cipher_algorithm algo;
@@ -235,6 +237,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	uint32_t mode_p = 0;
 	uint32_t type_p = 0;
 	uint32_t portid_p = 0;
+	uint32_t fallback_p = 0;
 
 	if (strcmp(tokens[0], "in") == 0) {
 		ri = &nb_sa_in;
@@ -245,6 +248,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 			return;
 
 		rule = &sa_in[*ri];
+		rule->direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS;
 	} else {
 		ri = &nb_sa_out;
 
@@ -254,6 +258,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 			return;
 
 		rule = &sa_out[*ri];
+		rule->direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS;
 	}
 
 	/* spi number */
@@ -263,7 +268,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	if (atoi(tokens[1]) == INVALID_SPI)
 		return;
 	rule->spi = atoi(tokens[1]);
-	ips = ipsec_get_session(rule);
+	ips = ipsec_get_primary_session(rule);
 
 	for (ti = 2; ti < n_tokens; ti++) {
 		if (strcmp(tokens[ti], "mode") == 0) {
@@ -596,6 +601,43 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 			continue;
 		}
 
+		if (strcmp(tokens[ti], "fallback") == 0) {
+			struct rte_ipsec_session *fb;
+
+			APP_CHECK(app_sa_prm.enable, status, "Fallback session "
+				"not allowed for legacy mode.");
+			if (status->status < 0)
+				return;
+			APP_CHECK(rule->direction ==
+				RTE_SECURITY_IPSEC_SA_DIR_INGRESS, status,
+				"Fallback session not allowed for egress "
+				"rule");
+			if (status->status < 0)
+				return;
+			APP_CHECK_PRESENCE(fallback_p, tokens[ti], status);
+			if (status->status < 0)
+				return;
+			INCREMENT_TOKEN_INDEX(ti, n_tokens, status);
+			if (status->status < 0)
+				return;
+			fb = ipsec_get_fallback_session(rule);
+			if (strcmp(tokens[ti], "lookaside-none") == 0) {
+				fb->type = RTE_SECURITY_ACTION_TYPE_NONE;
+			} else if (strcmp(tokens[ti], "lookaside-protocol")
+				== 0) {
+				fb->type =
+				RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL;
+			} else {
+				APP_CHECK(0, status, "unrecognized fallback "
+					"type %s.", tokens[ti]);
+				return;
+			}
+
+			rule->fallback_sessions = 1;
+			fallback_p = 1;
+			continue;
+		}
+
 		/* unrecognizeable input */
 		APP_CHECK(0, status, "unrecognized input \"%s\"",
 			tokens[ti]);
@@ -643,6 +685,7 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 	uint32_t i;
 	uint8_t a, b, c, d;
 	const struct rte_ipsec_session *ips;
+	const struct rte_ipsec_session *fallback_ips;
 
 	printf("\tspi_%s(%3u):", inbound?"in":"out", sa->spi);
 
@@ -699,7 +742,7 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 		break;
 	}
 
-	ips = &sa->ips;
+	ips = &sa->sessions[IPSEC_SESSION_PRIMARY];
 	printf(" type:");
 	switch (ips->type) {
 	case RTE_SECURITY_ACTION_TYPE_NONE:
@@ -715,6 +758,22 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 		printf("lookaside-protocol-offload ");
 		break;
 	}
+
+	fallback_ips = &sa->sessions[IPSEC_SESSION_FALLBACK];
+	if (fallback_ips != NULL && sa->fallback_sessions > 0) {
+		printf("inline fallback:");
+		switch (fallback_ips->type) {
+		case RTE_SECURITY_ACTION_TYPE_NONE:
+			printf("lookaside-none ");
+			break;
+		case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL:
+			printf("lookaside-protocol ");
+			break;
+		default:
+			printf("invalid ");
+			break;
+		}
+	}
 	printf("\n");
 }
 
@@ -896,7 +955,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 		}
 		*sa = entries[i];
 		sa->seq = 0;
-		ips = ipsec_get_session(sa);
+		ips = ipsec_get_primary_session(sa);
 
 		if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
 			ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
@@ -904,9 +963,6 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 				return -EINVAL;
 		}
 
-		sa->direction = (inbound == 1) ?
-				RTE_SECURITY_IPSEC_SA_DIR_INGRESS :
-				RTE_SECURITY_IPSEC_SA_DIR_EGRESS;
 
 		switch (WITHOUT_TRANSPORT_VERSION(sa->flags)) {
 		case IP4_TUNNEL:
@@ -946,7 +1002,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			sa->xforms = &sa_ctx->xf[idx].a;
 
-			ips = ipsec_get_session(sa);
+			ips = ipsec_get_primary_session(sa);
 			if (ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
 				ips->type ==
@@ -1160,9 +1216,15 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	/* init processing session */
-	ips = ipsec_get_session(lsa);
+	/* init primary processing session */
+	ips = ipsec_get_primary_session(lsa);
 	rc = fill_ipsec_session(ips, sa);
+	if (rc != 0)
+		return rc;
+
+	/* init inline fallback processing session */
+	if (lsa->fallback_sessions == 1)
+		rc = fill_ipsec_session(ipsec_get_fallback_session(lsa), sa);
 
 	return rc;
 }
@@ -1318,13 +1380,14 @@ inbound_sa_check(struct sa_ctx *sa_ctx, struct rte_mbuf *m, uint32_t sa_idx)
 
 static inline void
 single_inbound_lookup(struct ipsec_sa *sadb, struct rte_mbuf *pkt,
-		struct ipsec_sa **sa_ret)
+		void **sa_ret)
 {
 	struct rte_esp_hdr *esp;
 	struct ip *ip;
 	uint32_t *src4_addr;
 	uint8_t *src6_addr;
 	struct ipsec_sa *sa;
+	void *result_sa;
 
 	*sa_ret = NULL;
 
@@ -1334,33 +1397,48 @@ single_inbound_lookup(struct ipsec_sa *sadb, struct rte_mbuf *pkt,
 	if (esp->spi == INVALID_SPI)
 		return;
 
-	sa = &sadb[SPI2IDX(rte_be_to_cpu_32(esp->spi))];
+	result_sa = sa = &sadb[SPI2IDX(rte_be_to_cpu_32(esp->spi))];
 	if (rte_be_to_cpu_32(esp->spi) != sa->spi)
 		return;
 
+	/*
+	 * Mark need for inline offload fallback on the LSB of SA pointer.
+	 * Thanks to packet grouping mechanism which ipsec_process is using
+	 * packets marked for fallback processing will form separate group.
+	 *
+	 * Because it is not safe to use SA pointer it is casted to generic
+	 * pointer to prevent from unintentional use. Use ipsec_mask_saptr
+	 * to get valid struct pointer.
+	 */
+	if (MBUF_NO_SEC_OFFLOAD(pkt) && sa->fallback_sessions > 0) {
+		uintptr_t intsa = (uintptr_t)sa;
+		intsa |= IPSEC_SA_OFFLOAD_FALLBACK_FLAG;
+		result_sa = (void *)intsa;
+	}
+
 	switch (WITHOUT_TRANSPORT_VERSION(sa->flags)) {
 	case IP4_TUNNEL:
 		src4_addr = RTE_PTR_ADD(ip, offsetof(struct ip, ip_src));
 		if ((ip->ip_v == IPVERSION) &&
 				(sa->src.ip.ip4 == *src4_addr) &&
 				(sa->dst.ip.ip4 == *(src4_addr + 1)))
-			*sa_ret = sa;
+			*sa_ret = result_sa;
 		break;
 	case IP6_TUNNEL:
 		src6_addr = RTE_PTR_ADD(ip, offsetof(struct ip6_hdr, ip6_src));
 		if ((ip->ip_v == IP6_VERSION) &&
 				!memcmp(&sa->src.ip.ip6.ip6, src6_addr, 16) &&
 				!memcmp(&sa->dst.ip.ip6.ip6, src6_addr + 16, 16))
-			*sa_ret = sa;
+			*sa_ret = result_sa;
 		break;
 	case TRANSPORT:
-		*sa_ret = sa;
+		*sa_ret = result_sa;
 	}
 }
 
 void
 inbound_sa_lookup(struct sa_ctx *sa_ctx, struct rte_mbuf *pkts[],
-		struct ipsec_sa *sa[], uint16_t nb_pkts)
+		void *sa[], uint16_t nb_pkts)
 {
 	uint32_t i;
 
@@ -1370,7 +1448,7 @@ inbound_sa_lookup(struct sa_ctx *sa_ctx, struct rte_mbuf *pkts[],
 
 void
 outbound_sa_lookup(struct sa_ctx *sa_ctx, uint32_t sa_idx[],
-		struct ipsec_sa *sa[], uint16_t nb_pkts)
+		void *sa[], uint16_t nb_pkts)
 {
 	uint32_t i;
 
-- 
2.21.0.windows.1

[dpdk-dev] [PATCH v2 3/3] examples/ipsec-secgw: add offload fallback tests

[Marcin Smoczynski]

Add tests for offload fallback feature; add inbound config modificator
SGW_CFG_XPRM_IN (offload fallback setting can be set only for inbound
SAs). Tests are using cryptodev for outbound SA.

To test fragmentation with QAT set:
MULTI_SEG_TEST="--reassemble=4096 --cryptodev_mask=0x5555"

Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
---
 examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh         | 4 ++--
 .../test/trs_aesgcm_inline_crypto_fallback_defs.sh          | 5 +++++
 examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh         | 6 ++++--
 .../test/tun_aesgcm_inline_crypto_fallback_defs.sh          | 5 +++++
 4 files changed, 16 insertions(+), 4 deletions(-)
 create mode 100644 examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
 create mode 100644 examples/ipsec-secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh

diff --git a/examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh b/examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh
index f6c5bf5a7..17f2f86d2 100644
--- a/examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh
+++ b/examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh
@@ -29,11 +29,11 @@ sp ipv6 out esp bypass pri 1 sport 0:65535 dport 0:65535
 #SA in rules
 sa in 7 aead_algo aes-128-gcm \
 aead_key de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef \
-mode transport ${SGW_CFG_XPRM}
+mode transport ${SGW_CFG_XPRM} ${SGW_CFG_XPRM_IN}
 
 sa in 9 aead_algo aes-128-gcm \
 aead_key de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef \
-mode transport ${SGW_CFG_XPRM}
+mode transport ${SGW_CFG_XPRM} ${SGW_CFG_XPRM_IN}
 
 #SA out rules
 sa out 7 aead_algo aes-128-gcm \
diff --git a/examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh b/examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
new file mode 100644
index 000000000..875a7457d
--- /dev/null
+++ b/examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
@@ -0,0 +1,5 @@
+#! /bin/bash
+
+. ${DIR}/trs_aesgcm_defs.sh
+
+SGW_CFG_XPRM_IN='port_id 0 type inline-crypto-offload fallback lookaside-none'
diff --git a/examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh b/examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh
index 278377967..7490baded 100644
--- a/examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh
+++ b/examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh
@@ -29,11 +29,13 @@ sp ipv6 out esp bypass pri 1 sport 0:65535 dport 0:65535
 #SA in rules
 sa in 7 aead_algo aes-128-gcm \
 aead_key de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef \
-mode ipv4-tunnel src ${REMOTE_IPV4} dst ${LOCAL_IPV4} ${SGW_CFG_XPRM}
+mode ipv4-tunnel src ${REMOTE_IPV4} dst ${LOCAL_IPV4} ${SGW_CFG_XPRM} \
+${SGW_CFG_XPRM_IN}
 
 sa in 9 aead_algo aes-128-gcm \
 aead_key de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef \
-mode ipv6-tunnel src ${REMOTE_IPV6} dst ${LOCAL_IPV6} ${SGW_CFG_XPRM}
+mode ipv6-tunnel src ${REMOTE_IPV6} dst ${LOCAL_IPV6} ${SGW_CFG_XPRM} \
+${SGW_CFG_XPRM_IN}
 
 #SA out rules
 sa out 7 aead_algo aes-128-gcm \
diff --git a/examples/ipsec-secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh b/examples/ipsec-secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh
new file mode 100644
index 000000000..696848432
--- /dev/null
+++ b/examples/ipsec-secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh
@@ -0,0 +1,5 @@
+#! /bin/bash
+
+. ${DIR}/tun_aesgcm_defs.sh
+
+SGW_CFG_XPRM_IN='port_id 0 type inline-crypto-offload fallback lookaside-none'
-- 
2.21.0.windows.1

Re: [dpdk-dev] [PATCH v2 0/3] examples/ipsec-secgw: add fallback session

[Anoob Joseph]

Hi Marcin,

Sorry for the late response. But how do you plan to handle "inline protocol" processed packets? 

Thanks,
Anoob

> -----Original Message-----
> From: dev <dev-bounces@dpdk.org> On Behalf Of Marcin Smoczynski
> Sent: Wednesday, September 4, 2019 7:47 PM
> To: konstantin.ananyev@intel.com; akhil.goyal@nxp.com
> Cc: dev@dpdk.org; Marcin Smoczynski <marcinx.smoczynski@intel.com>
> Subject: [dpdk-dev] [PATCH v2 0/3] examples/ipsec-secgw: add fallback
> session
> 
> Inline processing is limited to a specified subset of traffic. It is often unable to
> handle more complicated situations, such as fragmented traffic. When using
> inline processing such traffic is dropped.
> 
> Introduce multiple sessions per SA allowing to configure a fallback lookaside
> session for packets that normally would be dropped.
> A fallback session type in the SA configuration by adding 'fallback'
> with 'lookaside-none' or 'lookaside-protocol' parameter to determine type of
> session.
> 
> Fallback session feature is available only when using librte_ipsec.
> 
> v1 to v2 changes:
>  - disable fallback offload for outbound SAs
>  - add test scripts
> 
> Marcin Smoczynski (3):
>   examples/ipsec-secgw: ipsec_sa structure cleanup
>   examples/ipsec-secgw: add fallback session feature
>   examples/ipsec-secgw: add offload fallback tests
> 
>  doc/guides/sample_app_ug/ipsec_secgw.rst      |  17 +-
>  examples/ipsec-secgw/esp.c                    |  35 ++--
>  examples/ipsec-secgw/ipsec-secgw.c            |  16 +-
>  examples/ipsec-secgw/ipsec.c                  |  99 ++++++-----
>  examples/ipsec-secgw/ipsec.h                  |  61 +++++--
>  examples/ipsec-secgw/ipsec_process.c          | 113 +++++++-----
>  examples/ipsec-secgw/sa.c                     | 164 +++++++++++++-----
>  .../test/trs_aesgcm_common_defs.sh            |   4 +-
>  .../trs_aesgcm_inline_crypto_fallback_defs.sh |   5 +
>  .../test/tun_aesgcm_common_defs.sh            |   6 +-
>  .../tun_aesgcm_inline_crypto_fallback_defs.sh |   5 +
>  11 files changed, 358 insertions(+), 167 deletions(-)  create mode 100644
> examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
>  create mode 100644 examples/ipsec-
> secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh
> 
> --
> 2.21.0.windows.1

Re: [dpdk-dev] [PATCH v2 0/3] examples/ipsec-secgw: add fallback session

[Ananyev, Konstantin]

Hi Anoob,

> 
> Hi Marcin,
> 
> Sorry for the late response. But how do you plan to handle "inline protocol" processed packets?

Right now that feature is supported for "inline crypto" only.
For the case when SA doesn't enable replay window and/or ESN
current patch should also work for "inline proto" too,
but this is just my understanding (not tested, etc.). 
Konstantin

> 
> Thanks,
> Anoob
> 
> > -----Original Message-----
> > From: dev <dev-bounces@dpdk.org> On Behalf Of Marcin Smoczynski
> > Sent: Wednesday, September 4, 2019 7:47 PM
> > To: konstantin.ananyev@intel.com; akhil.goyal@nxp.com
> > Cc: dev@dpdk.org; Marcin Smoczynski <marcinx.smoczynski@intel.com>
> > Subject: [dpdk-dev] [PATCH v2 0/3] examples/ipsec-secgw: add fallback
> > session
> >
> > Inline processing is limited to a specified subset of traffic. It is often unable to
> > handle more complicated situations, such as fragmented traffic. When using
> > inline processing such traffic is dropped.
> >
> > Introduce multiple sessions per SA allowing to configure a fallback lookaside
> > session for packets that normally would be dropped.
> > A fallback session type in the SA configuration by adding 'fallback'
> > with 'lookaside-none' or 'lookaside-protocol' parameter to determine type of
> > session.
> >
> > Fallback session feature is available only when using librte_ipsec.
> >
> > v1 to v2 changes:
> >  - disable fallback offload for outbound SAs
> >  - add test scripts
> >
> > Marcin Smoczynski (3):
> >   examples/ipsec-secgw: ipsec_sa structure cleanup
> >   examples/ipsec-secgw: add fallback session feature
> >   examples/ipsec-secgw: add offload fallback tests
> >
> >  doc/guides/sample_app_ug/ipsec_secgw.rst      |  17 +-
> >  examples/ipsec-secgw/esp.c                    |  35 ++--
> >  examples/ipsec-secgw/ipsec-secgw.c            |  16 +-
> >  examples/ipsec-secgw/ipsec.c                  |  99 ++++++-----
> >  examples/ipsec-secgw/ipsec.h                  |  61 +++++--
> >  examples/ipsec-secgw/ipsec_process.c          | 113 +++++++-----
> >  examples/ipsec-secgw/sa.c                     | 164 +++++++++++++-----
> >  .../test/trs_aesgcm_common_defs.sh            |   4 +-
> >  .../trs_aesgcm_inline_crypto_fallback_defs.sh |   5 +
> >  .../test/tun_aesgcm_common_defs.sh            |   6 +-
> >  .../tun_aesgcm_inline_crypto_fallback_defs.sh |   5 +
> >  11 files changed, 358 insertions(+), 167 deletions(-)  create mode 100644
> > examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
> >  create mode 100644 examples/ipsec-
> > secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh
> >
> > --
> > 2.21.0.windows.1

Re: [dpdk-dev] [PATCH v2 0/3] examples/ipsec-secgw: add fallback session

[Anoob Joseph]

Hi Konstantin,

Please see inline.

Thanks,
Anoob

> -----Original Message-----
> From: Ananyev, Konstantin <konstantin.ananyev@intel.com>
> Sent: Wednesday, September 18, 2019 2:16 PM
> To: Anoob Joseph <anoobj@marvell.com>; Smoczynski, MarcinX
> <marcinx.smoczynski@intel.com>; akhil.goyal@nxp.com
> Cc: dev@dpdk.org; Narayana Prasad Raju Athreya <pathreya@marvell.com>;
> Jerin Jacob Kollanukkaran <jerinj@marvell.com>; Archana Muniganti
> <marchana@marvell.com>
> Subject: RE: [dpdk-dev] [PATCH v2 0/3] examples/ipsec-secgw: add fallback
> session
> 
> 
> Hi Anoob,
> 
> >
> > Hi Marcin,
> >
> > Sorry for the late response. But how do you plan to handle "inline protocol"
> processed packets?
> 
> Right now that feature is supported for "inline crypto" only.

[Anoob] The description says "inline processed" packets. Hence the confusion.
 
> For the case when SA doesn't enable replay window and/or ESN current
> patch should also work for "inline proto" too, but this is just my
> understanding (not tested, etc.).

[Anoob] In case of inline ipsec processing, the ipsec state (which would track sequence number etc) will be internal to the PMDs. So anti-replay/ESN would have to be done either in the h/w or PMD. This would mean application will not have state information regarding ipsec processing. Hence fallback handling with the above scheme will not work in that case.

To address this properly for inline protocol, we will have to come up with some logic to share session private data b/w "eligible" PMDs. This would involve library changes to rte_security, etc. Once that is proposed, there will be one kind of handling for inline protocol processing and another kind for inline crypto processing. Would you be fine with that?
 
> Konstantin
> 
> >
> > Thanks,
> > Anoob
> >
> > > -----Original Message-----
> > > From: dev <dev-bounces@dpdk.org> On Behalf Of Marcin Smoczynski
> > > Sent: Wednesday, September 4, 2019 7:47 PM
> > > To: konstantin.ananyev@intel.com; akhil.goyal@nxp.com
> > > Cc: dev@dpdk.org; Marcin Smoczynski <marcinx.smoczynski@intel.com>
> > > Subject: [dpdk-dev] [PATCH v2 0/3] examples/ipsec-secgw: add
> > > fallback session
> > >
> > > Inline processing is limited to a specified subset of traffic. It is
> > > often unable to handle more complicated situations, such as
> > > fragmented traffic. When using inline processing such traffic is dropped.
> > >
> > > Introduce multiple sessions per SA allowing to configure a fallback
> > > lookaside session for packets that normally would be dropped.
> > > A fallback session type in the SA configuration by adding 'fallback'
> > > with 'lookaside-none' or 'lookaside-protocol' parameter to determine
> > > type of session.
> > >
> > > Fallback session feature is available only when using librte_ipsec.
> > >
> > > v1 to v2 changes:
> > >  - disable fallback offload for outbound SAs
> > >  - add test scripts
> > >
> > > Marcin Smoczynski (3):
> > >   examples/ipsec-secgw: ipsec_sa structure cleanup
> > >   examples/ipsec-secgw: add fallback session feature
> > >   examples/ipsec-secgw: add offload fallback tests
> > >
> > >  doc/guides/sample_app_ug/ipsec_secgw.rst      |  17 +-
> > >  examples/ipsec-secgw/esp.c                    |  35 ++--
> > >  examples/ipsec-secgw/ipsec-secgw.c            |  16 +-
> > >  examples/ipsec-secgw/ipsec.c                  |  99 ++++++-----
> > >  examples/ipsec-secgw/ipsec.h                  |  61 +++++--
> > >  examples/ipsec-secgw/ipsec_process.c          | 113 +++++++-----
> > >  examples/ipsec-secgw/sa.c                     | 164 +++++++++++++-----
> > >  .../test/trs_aesgcm_common_defs.sh            |   4 +-
> > >  .../trs_aesgcm_inline_crypto_fallback_defs.sh |   5 +
> > >  .../test/tun_aesgcm_common_defs.sh            |   6 +-
> > >  .../tun_aesgcm_inline_crypto_fallback_defs.sh |   5 +
> > >  11 files changed, 358 insertions(+), 167 deletions(-)  create mode
> > > 100644
> > > examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
> > >  create mode 100644 examples/ipsec-
> > > secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh
> > >
> > > --
> > > 2.21.0.windows.1

Re: [dpdk-dev] [PATCH v2 0/3] examples/ipsec-secgw: add fallback session

[Ananyev, Konstantin]

Hi Anoob,

> > > Sorry for the late response. But how do you plan to handle "inline protocol"
> > processed packets?
> >
> > Right now that feature is supported for "inline crypto" only.
> 
> [Anoob] The description says "inline processed" packets. Hence the confusion.
> 
> > For the case when SA doesn't enable replay window and/or ESN current
> > patch should also work for "inline proto" too, but this is just my
> > understanding (not tested, etc.).
> 
> [Anoob] In case of inline ipsec processing, the ipsec state (which would track sequence number etc) will be internal to the PMDs. So anti-
> replay/ESN would have to be done either in the h/w or PMD. This would mean application will not have state information regarding ipsec
> processing. Hence fallback handling with the above scheme will not work in that case.

Agree, that's why I wrote above that current wok might work for inline-proto
*only* if replay window and ESN is disabled. 

> 
> To address this properly for inline protocol, we will have to come up with some logic to share session private data b/w "eligible" PMDs. This
> would involve library changes to rte_security, etc.

Again, totally agree.
As I remember we already discussed it about a year ago, but didn't come up with any concrete proposal.

> Once that is proposed, there will be one kind of handling for inline protocol processing
> and another kind for inline crypto processing. Would you be fine with that?

For sure something needs to be changed for inline-proto to sync replay-window/ESN related data
between HW/PMD and SW. 
What it should be - new function, or something else - hard to tell right now. 
Konstantin

> 
> > Konstantin
> >
> > >
> > > Thanks,
> > > Anoob
> > >
> > > > -----Original Message-----
> > > > From: dev <dev-bounces@dpdk.org> On Behalf Of Marcin Smoczynski
> > > > Sent: Wednesday, September 4, 2019 7:47 PM
> > > > To: konstantin.ananyev@intel.com; akhil.goyal@nxp.com
> > > > Cc: dev@dpdk.org; Marcin Smoczynski <marcinx.smoczynski@intel.com>
> > > > Subject: [dpdk-dev] [PATCH v2 0/3] examples/ipsec-secgw: add
> > > > fallback session
> > > >
> > > > Inline processing is limited to a specified subset of traffic. It is
> > > > often unable to handle more complicated situations, such as
> > > > fragmented traffic. When using inline processing such traffic is dropped.
> > > >
> > > > Introduce multiple sessions per SA allowing to configure a fallback
> > > > lookaside session for packets that normally would be dropped.
> > > > A fallback session type in the SA configuration by adding 'fallback'
> > > > with 'lookaside-none' or 'lookaside-protocol' parameter to determine
> > > > type of session.
> > > >
> > > > Fallback session feature is available only when using librte_ipsec.
> > > >
> > > > v1 to v2 changes:
> > > >  - disable fallback offload for outbound SAs
> > > >  - add test scripts
> > > >
> > > > Marcin Smoczynski (3):
> > > >   examples/ipsec-secgw: ipsec_sa structure cleanup
> > > >   examples/ipsec-secgw: add fallback session feature
> > > >   examples/ipsec-secgw: add offload fallback tests
> > > >
> > > >  doc/guides/sample_app_ug/ipsec_secgw.rst      |  17 +-
> > > >  examples/ipsec-secgw/esp.c                    |  35 ++--
> > > >  examples/ipsec-secgw/ipsec-secgw.c            |  16 +-
> > > >  examples/ipsec-secgw/ipsec.c                  |  99 ++++++-----
> > > >  examples/ipsec-secgw/ipsec.h                  |  61 +++++--
> > > >  examples/ipsec-secgw/ipsec_process.c          | 113 +++++++-----
> > > >  examples/ipsec-secgw/sa.c                     | 164 +++++++++++++-----
> > > >  .../test/trs_aesgcm_common_defs.sh            |   4 +-
> > > >  .../trs_aesgcm_inline_crypto_fallback_defs.sh |   5 +
> > > >  .../test/tun_aesgcm_common_defs.sh            |   6 +-
> > > >  .../tun_aesgcm_inline_crypto_fallback_defs.sh |   5 +
> > > >  11 files changed, 358 insertions(+), 167 deletions(-)  create mode
> > > > 100644
> > > > examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
> > > >  create mode 100644 examples/ipsec-
> > > > secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh
> > > >
> > > > --
> > > > 2.21.0.windows.1

Re: [dpdk-dev] [PATCH v2 0/3] examples/ipsec-secgw: add fallback session

[Anoob Joseph]

Hi Konstantin, 

Please see inline.

Thanks,
Anoob

> -----Original Message-----
> From: Ananyev, Konstantin <konstantin.ananyev@intel.com>
> Sent: Thursday, September 19, 2019 3:49 AM
> To: Anoob Joseph <anoobj@marvell.com>; Smoczynski, MarcinX
> <marcinx.smoczynski@intel.com>; akhil.goyal@nxp.com
> Cc: dev@dpdk.org; Narayana Prasad Raju Athreya <pathreya@marvell.com>;
> Jerin Jacob Kollanukkaran <jerinj@marvell.com>; Archana Muniganti
> <marchana@marvell.com>
> Subject: RE: [dpdk-dev] [PATCH v2 0/3] examples/ipsec-secgw: add fallback
> session
> 
> 
> 
> Hi Anoob,
> 
> > > > Sorry for the late response. But how do you plan to handle "inline
> protocol"
> > > processed packets?
> > >
> > > Right now that feature is supported for "inline crypto" only.
> >
> > [Anoob] The description says "inline processed" packets. Hence the confusion.
> >
> > > For the case when SA doesn't enable replay window and/or ESN current
> > > patch should also work for "inline proto" too, but this is just my
> > > understanding (not tested, etc.).
> >
> > [Anoob] In case of inline ipsec processing, the ipsec state (which
> > would track sequence number etc) will be internal to the PMDs. So
> > anti- replay/ESN would have to be done either in the h/w or PMD. This would
> mean application will not have state information regarding ipsec processing.
> Hence fallback handling with the above scheme will not work in that case.
> 
> Agree, that's why I wrote above that current wok might work for inline-proto
> *only* if replay window and ESN is disabled.

[Anoob] Any feature that makes use of protocol "state" would fail with this scheme. In case of inline ipsec, that is anti-replay & ESN. I see that you are not planning for fallback session for outbound. If at all that is planned, this scheme will fail to co-ordinate sequence number between original and fallback sessions.

> 
> >
> > To address this properly for inline protocol, we will have to come up
> > with some logic to share session private data b/w "eligible" PMDs. This would
> involve library changes to rte_security, etc.
> 
> Again, totally agree.
> As I remember we already discussed it about a year ago, but didn't come up with
> any concrete proposal.
> 
> > Once that is proposed, there will be one kind of handling for inline
> > protocol processing and another kind for inline crypto processing. Would you
> be fine with that?
> 
> For sure something needs to be changed for inline-proto to sync replay-
> window/ESN related data between HW/PMD and SW.
> What it should be - new function, or something else - hard to tell right now.

[Anoob] No disagreement. My only concern is the incompleteness of this solution. We will have to propose a totally new scheme for inline protocol. You do agree that this approach will not help inline protocol offloading, right? If you are okay with having different solutions for inline crypto & inline protocol, I don't have any issue with this series.

Also, how do you plan to pass "state" info to lookaside protocol session? That will be required to handle ESN/anti-replay in lookaside protocol capable PMD as well.
 
> Konstantin
> 
> >
> > > Konstantin
> > >
> > > >
> > > > Thanks,
> > > > Anoob
> > > >
> > > > > -----Original Message-----
> > > > > From: dev <dev-bounces@dpdk.org> On Behalf Of Marcin Smoczynski
> > > > > Sent: Wednesday, September 4, 2019 7:47 PM
> > > > > To: konstantin.ananyev@intel.com; akhil.goyal@nxp.com
> > > > > Cc: dev@dpdk.org; Marcin Smoczynski
> > > > > <marcinx.smoczynski@intel.com>
> > > > > Subject: [dpdk-dev] [PATCH v2 0/3] examples/ipsec-secgw: add
> > > > > fallback session
> > > > >
> > > > > Inline processing is limited to a specified subset of traffic.
> > > > > It is often unable to handle more complicated situations, such
> > > > > as fragmented traffic. When using inline processing such traffic is
> dropped.
> > > > >
> > > > > Introduce multiple sessions per SA allowing to configure a
> > > > > fallback lookaside session for packets that normally would be dropped.
> > > > > A fallback session type in the SA configuration by adding 'fallback'
> > > > > with 'lookaside-none' or 'lookaside-protocol' parameter to
> > > > > determine type of session.
> > > > >
> > > > > Fallback session feature is available only when using librte_ipsec.
> > > > >
> > > > > v1 to v2 changes:
> > > > >  - disable fallback offload for outbound SAs
> > > > >  - add test scripts
> > > > >
> > > > > Marcin Smoczynski (3):
> > > > >   examples/ipsec-secgw: ipsec_sa structure cleanup
> > > > >   examples/ipsec-secgw: add fallback session feature
> > > > >   examples/ipsec-secgw: add offload fallback tests
> > > > >
> > > > >  doc/guides/sample_app_ug/ipsec_secgw.rst      |  17 +-
> > > > >  examples/ipsec-secgw/esp.c                    |  35 ++--
> > > > >  examples/ipsec-secgw/ipsec-secgw.c            |  16 +-
> > > > >  examples/ipsec-secgw/ipsec.c                  |  99 ++++++-----
> > > > >  examples/ipsec-secgw/ipsec.h                  |  61 +++++--
> > > > >  examples/ipsec-secgw/ipsec_process.c          | 113 +++++++-----
> > > > >  examples/ipsec-secgw/sa.c                     | 164 +++++++++++++-----
> > > > >  .../test/trs_aesgcm_common_defs.sh            |   4 +-
> > > > >  .../trs_aesgcm_inline_crypto_fallback_defs.sh |   5 +
> > > > >  .../test/tun_aesgcm_common_defs.sh            |   6 +-
> > > > >  .../tun_aesgcm_inline_crypto_fallback_defs.sh |   5 +
> > > > >  11 files changed, 358 insertions(+), 167 deletions(-)  create
> > > > > mode
> > > > > 100644
> > > > > examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs
> > > > > .sh  create mode 100644 examples/ipsec-
> > > > > secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh
> > > > >
> > > > > --
> > > > > 2.21.0.windows.1

Re: [dpdk-dev] [PATCH v2 0/3] examples/ipsec-secgw: add fallback session

[Ananyev, Konstantin]

Hi Anoob,

> > > > > Sorry for the late response. But how do you plan to handle "inline
> > protocol"
> > > > processed packets?
> > > >
> > > > Right now that feature is supported for "inline crypto" only.
> > >
> > > [Anoob] The description says "inline processed" packets. Hence the confusion.
> > >
> > > > For the case when SA doesn't enable replay window and/or ESN current
> > > > patch should also work for "inline proto" too, but this is just my
> > > > understanding (not tested, etc.).
> > >
> > > [Anoob] In case of inline ipsec processing, the ipsec state (which
> > > would track sequence number etc) will be internal to the PMDs. So
> > > anti- replay/ESN would have to be done either in the h/w or PMD. This would
> > mean application will not have state information regarding ipsec processing.
> > Hence fallback handling with the above scheme will not work in that case.
> >
> > Agree, that's why I wrote above that current wok might work for inline-proto
> > *only* if replay window and ESN is disabled.
> 
> [Anoob] Any feature that makes use of protocol "state" would fail with this scheme. In case of inline ipsec, that is anti-replay & ESN. I see
> that you are not planning for fallback session for outbound. If at all that is planned, this scheme will fail to co-ordinate sequence number
> between original and fallback sessions.

Right now we don't do outbound fallback sessions.
Again, similar scheme would work for TX with lookaside-none and inline-crypto, but not for inline/lookaside-proto.
Of course few extra changes would be needed - move fragmentation before ipsec processing part. 

> 
> >
> > >
> > > To address this properly for inline protocol, we will have to come up
> > > with some logic to share session private data b/w "eligible" PMDs. This would
> > involve library changes to rte_security, etc.
> >
> > Again, totally agree.
> > As I remember we already discussed it about a year ago, but didn't come up with
> > any concrete proposal.
> >
> > > Once that is proposed, there will be one kind of handling for inline
> > > protocol processing and another kind for inline crypto processing. Would you
> > be fine with that?
> >
> > For sure something needs to be changed for inline-proto to sync replay-
> > window/ESN related data between HW/PMD and SW.
> > What it should be - new function, or something else - hard to tell right now.
> 
> [Anoob] No disagreement. My only concern is the incompleteness of this solution.
> We will have to propose a totally new scheme for inline
> protocol.

Sure, looking forward for it.
In fact, I asked these questions a year ago but as I said before there was no progress on that.

> You do agree that this approach will not help inline protocol offloading, right?

Yes, see above.

> If you are okay with having different solutions for
> inline crypto & inline protocol, I don't have any issue with this series.
> 
> Also, how do you plan to pass "state" info to lookaside protocol session? That will be required to handle ESN/anti-replay in lookaside
> protocol capable PMD as well.

Obviously there is no such API for any *-proto devices, see above.
Not sure why do you keep repeating it, I already agreed with you, see above :)
Just to summarize:
1. Yes I think for lookaiside/iniline-proto devices there is a need for some extra API
    to sync session state between HW and SW.
   Right now, I don't have a clear idea how exactly it should look like.
2. Intel team doesn't plan to work on this API right now.
3. I am happy to review with what you guys will come-up with and provide my feedback.  
Hope that makes things clear.
Konstantin

> 
> > Konstantin
> >
> > >
> > > > Konstantin
> > > >
> > > > >
> > > > > Thanks,
> > > > > Anoob
> > > > >
> > > > > > -----Original Message-----
> > > > > > From: dev <dev-bounces@dpdk.org> On Behalf Of Marcin Smoczynski
> > > > > > Sent: Wednesday, September 4, 2019 7:47 PM
> > > > > > To: konstantin.ananyev@intel.com; akhil.goyal@nxp.com
> > > > > > Cc: dev@dpdk.org; Marcin Smoczynski
> > > > > > <marcinx.smoczynski@intel.com>
> > > > > > Subject: [dpdk-dev] [PATCH v2 0/3] examples/ipsec-secgw: add
> > > > > > fallback session
> > > > > >
> > > > > > Inline processing is limited to a specified subset of traffic.
> > > > > > It is often unable to handle more complicated situations, such
> > > > > > as fragmented traffic. When using inline processing such traffic is
> > dropped.
> > > > > >
> > > > > > Introduce multiple sessions per SA allowing to configure a
> > > > > > fallback lookaside session for packets that normally would be dropped.
> > > > > > A fallback session type in the SA configuration by adding 'fallback'
> > > > > > with 'lookaside-none' or 'lookaside-protocol' parameter to
> > > > > > determine type of session.
> > > > > >
> > > > > > Fallback session feature is available only when using librte_ipsec.
> > > > > >
> > > > > > v1 to v2 changes:
> > > > > >  - disable fallback offload for outbound SAs
> > > > > >  - add test scripts
> > > > > >
> > > > > > Marcin Smoczynski (3):
> > > > > >   examples/ipsec-secgw: ipsec_sa structure cleanup
> > > > > >   examples/ipsec-secgw: add fallback session feature
> > > > > >   examples/ipsec-secgw: add offload fallback tests
> > > > > >
> > > > > >  doc/guides/sample_app_ug/ipsec_secgw.rst      |  17 +-
> > > > > >  examples/ipsec-secgw/esp.c                    |  35 ++--
> > > > > >  examples/ipsec-secgw/ipsec-secgw.c            |  16 +-
> > > > > >  examples/ipsec-secgw/ipsec.c                  |  99 ++++++-----
> > > > > >  examples/ipsec-secgw/ipsec.h                  |  61 +++++--
> > > > > >  examples/ipsec-secgw/ipsec_process.c          | 113 +++++++-----
> > > > > >  examples/ipsec-secgw/sa.c                     | 164 +++++++++++++-----
> > > > > >  .../test/trs_aesgcm_common_defs.sh            |   4 +-
> > > > > >  .../trs_aesgcm_inline_crypto_fallback_defs.sh |   5 +
> > > > > >  .../test/tun_aesgcm_common_defs.sh            |   6 +-
> > > > > >  .../tun_aesgcm_inline_crypto_fallback_defs.sh |   5 +
> > > > > >  11 files changed, 358 insertions(+), 167 deletions(-)  create
> > > > > > mode
> > > > > > 100644
> > > > > > examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs
> > > > > > .sh  create mode 100644 examples/ipsec-
> > > > > > secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh
> > > > > >
> > > > > > --
> > > > > > 2.21.0.windows.1

Re: [dpdk-dev] [PATCH v2 0/3] examples/ipsec-secgw: add fallback session

[Anoob Joseph]

Hi Konstantin,

Please see inline.

Thanks,
Anoob

> -----Original Message-----
> From: Ananyev, Konstantin <konstantin.ananyev@intel.com>
> Sent: Thursday, September 19, 2019 1:04 PM
> To: Anoob Joseph <anoobj@marvell.com>; Smoczynski, MarcinX
> <marcinx.smoczynski@intel.com>; akhil.goyal@nxp.com
> Cc: dev@dpdk.org; Narayana Prasad Raju Athreya <pathreya@marvell.com>;
> Jerin Jacob Kollanukkaran <jerinj@marvell.com>; Archana Muniganti
> <marchana@marvell.com>
> Subject: RE: [dpdk-dev] [PATCH v2 0/3] examples/ipsec-secgw: add fallback
> session
> 
> 
> Hi Anoob,
> 
> > > > > > Sorry for the late response. But how do you plan to handle
> > > > > > "inline
> > > protocol"
> > > > > processed packets?
> > > > >
> > > > > Right now that feature is supported for "inline crypto" only.
> > > >
> > > > [Anoob] The description says "inline processed" packets. Hence the
> confusion.
> > > >
> > > > > For the case when SA doesn't enable replay window and/or ESN
> > > > > current patch should also work for "inline proto" too, but this
> > > > > is just my understanding (not tested, etc.).
> > > >
> > > > [Anoob] In case of inline ipsec processing, the ipsec state (which
> > > > would track sequence number etc) will be internal to the PMDs. So
> > > > anti- replay/ESN would have to be done either in the h/w or PMD.
> > > > This would
> > > mean application will not have state information regarding ipsec processing.
> > > Hence fallback handling with the above scheme will not work in that case.
> > >
> > > Agree, that's why I wrote above that current wok might work for
> > > inline-proto
> > > *only* if replay window and ESN is disabled.
> >
> > [Anoob] Any feature that makes use of protocol "state" would fail with
> > this scheme. In case of inline ipsec, that is anti-replay & ESN. I see
> > that you are not planning for fallback session for outbound. If at all that is
> planned, this scheme will fail to co-ordinate sequence number between original
> and fallback sessions.
> 
> Right now we don't do outbound fallback sessions.
> Again, similar scheme would work for TX with lookaside-none and inline-crypto,
> but not for inline/lookaside-proto.
> Of course few extra changes would be needed - move fragmentation before
> ipsec processing part.

[Anoob] Understood.
 
> 
> >
> > >
> > > >
> > > > To address this properly for inline protocol, we will have to come
> > > > up with some logic to share session private data b/w "eligible"
> > > > PMDs. This would
> > > involve library changes to rte_security, etc.
> > >
> > > Again, totally agree.
> > > As I remember we already discussed it about a year ago, but didn't
> > > come up with any concrete proposal.
> > >
> > > > Once that is proposed, there will be one kind of handling for
> > > > inline protocol processing and another kind for inline crypto
> > > > processing. Would you
> > > be fine with that?
> > >
> > > For sure something needs to be changed for inline-proto to sync
> > > replay- window/ESN related data between HW/PMD and SW.
> > > What it should be - new function, or something else - hard to tell right now.
> >
> > [Anoob] No disagreement. My only concern is the incompleteness of this
> solution.
> > We will have to propose a totally new scheme for inline protocol.
> 
> Sure, looking forward for it.
> In fact, I asked these questions a year ago but as I said before there was no
> progress on that.
> 
> > You do agree that this approach will not help inline protocol offloading, right?
> 
> Yes, see above.
> 
> > If you are okay with having different solutions for inline crypto &
> > inline protocol, I don't have any issue with this series.
> >
> > Also, how do you plan to pass "state" info to lookaside protocol
> > session? That will be required to handle ESN/anti-replay in lookaside protocol
> capable PMD as well.
> 
> Obviously there is no such API for any *-proto devices, see above.
> Not sure why do you keep repeating it, I already agreed with you, see above :)
> Just to summarize:
> 1. Yes I think for lookaiside/iniline-proto devices there is a need for some extra
> API
>     to sync session state between HW and SW.
>    Right now, I don't have a clear idea how exactly it should look like.
> 2. Intel team doesn't plan to work on this API right now.
> 3. I am happy to review with what you guys will come-up with and provide my
> feedback.
> Hope that makes things clear.

[Anoob] My apologies if the queries seemed repetitive. I guess all my concerns are answered. Please do update the logs (and description) to state limitations regarding protocol offloads. I would suggest using "inline crypto processing" instead of "inline processing" wherever applicable. Also, if lookaside protocol is mentioned, better state that the "protocol state" will have to be maintained by application (or lib ipsec in case of lookaside ipsec offload).
 
> Konstantin
> 
> >
> > > Konstantin
> > >
> > > >
> > > > > Konstantin
> > > > >
> > > > > >
> > > > > > Thanks,
> > > > > > Anoob
> > > > > >
> > > > > > > -----Original Message-----
> > > > > > > From: dev <dev-bounces@dpdk.org> On Behalf Of Marcin
> > > > > > > Smoczynski
> > > > > > > Sent: Wednesday, September 4, 2019 7:47 PM
> > > > > > > To: konstantin.ananyev@intel.com; akhil.goyal@nxp.com
> > > > > > > Cc: dev@dpdk.org; Marcin Smoczynski
> > > > > > > <marcinx.smoczynski@intel.com>
> > > > > > > Subject: [dpdk-dev] [PATCH v2 0/3] examples/ipsec-secgw: add
> > > > > > > fallback session
> > > > > > >
> > > > > > > Inline processing is limited to a specified subset of traffic.
> > > > > > > It is often unable to handle more complicated situations,
> > > > > > > such as fragmented traffic. When using inline processing
> > > > > > > such traffic is
> > > dropped.
> > > > > > >
> > > > > > > Introduce multiple sessions per SA allowing to configure a
> > > > > > > fallback lookaside session for packets that normally would be
> dropped.
> > > > > > > A fallback session type in the SA configuration by adding 'fallback'
> > > > > > > with 'lookaside-none' or 'lookaside-protocol' parameter to
> > > > > > > determine type of session.
> > > > > > >
> > > > > > > Fallback session feature is available only when using librte_ipsec.
> > > > > > >
> > > > > > > v1 to v2 changes:
> > > > > > >  - disable fallback offload for outbound SAs
> > > > > > >  - add test scripts
> > > > > > >
> > > > > > > Marcin Smoczynski (3):
> > > > > > >   examples/ipsec-secgw: ipsec_sa structure cleanup
> > > > > > >   examples/ipsec-secgw: add fallback session feature
> > > > > > >   examples/ipsec-secgw: add offload fallback tests
> > > > > > >
> > > > > > >  doc/guides/sample_app_ug/ipsec_secgw.rst      |  17 +-
> > > > > > >  examples/ipsec-secgw/esp.c                    |  35 ++--
> > > > > > >  examples/ipsec-secgw/ipsec-secgw.c            |  16 +-
> > > > > > >  examples/ipsec-secgw/ipsec.c                  |  99 ++++++-----
> > > > > > >  examples/ipsec-secgw/ipsec.h                  |  61 +++++--
> > > > > > >  examples/ipsec-secgw/ipsec_process.c          | 113 +++++++-----
> > > > > > >  examples/ipsec-secgw/sa.c                     | 164 +++++++++++++-----
> > > > > > >  .../test/trs_aesgcm_common_defs.sh            |   4 +-
> > > > > > >  .../trs_aesgcm_inline_crypto_fallback_defs.sh |   5 +
> > > > > > >  .../test/tun_aesgcm_common_defs.sh            |   6 +-
> > > > > > >  .../tun_aesgcm_inline_crypto_fallback_defs.sh |   5 +
> > > > > > >  11 files changed, 358 insertions(+), 167 deletions(-)
> > > > > > > create mode
> > > > > > > 100644
> > > > > > > examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_
> > > > > > > defs .sh  create mode 100644 examples/ipsec-
> > > > > > > secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh
> > > > > > >
> > > > > > > --
> > > > > > > 2.21.0.windows.1

[dpdk-dev] [PATCH v3 0/3] add fallback session

[Marcin Smoczynski]

Add fallback session feature allowing to process packets that inline
processor is unable to handle (e.g. fragmented traffic). Processing
takes place in a secondary session defined for SA in a configuration
file.

This feature is limited to ingress IPsec traffic only. IPsec
anti-replay window and ESN are supported in conjunction with fallback
session when following conditions are met:
 * primary session is 'inline-crypto-offload,
 * fallback sessions is 'lookaside-none'.

v2 to v3 changes:
 - doc and commit log update - explicitly state feature limitations

v1 to v2 changes:
 - disable fallback offload for outbound SAs
 - add test scripts

Marcin Smoczynski (3):
  examples/ipsec-secgw: ipsec_sa structure cleanup
  examples/ipsec-secgw: add fallback session feature
  examples/ipsec-secgw: add offload fallback tests

 doc/guides/sample_app_ug/ipsec_secgw.rst      |  20 ++-
 examples/ipsec-secgw/esp.c                    |  35 ++--
 examples/ipsec-secgw/ipsec-secgw.c            |  16 +-
 examples/ipsec-secgw/ipsec.c                  |  99 ++++++-----
 examples/ipsec-secgw/ipsec.h                  |  61 +++++--
 examples/ipsec-secgw/ipsec_process.c          | 113 +++++++-----
 examples/ipsec-secgw/sa.c                     | 164 +++++++++++++-----
 .../test/trs_aesgcm_common_defs.sh            |   4 +-
 .../trs_aesgcm_inline_crypto_fallback_defs.sh |   5 +
 .../test/tun_aesgcm_common_defs.sh            |   6 +-
 .../tun_aesgcm_inline_crypto_fallback_defs.sh |   5 +
 11 files changed, 361 insertions(+), 167 deletions(-)
 create mode 100644 examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
 create mode 100644 examples/ipsec-secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh

--
2.17.1

[dpdk-dev] [PATCH v3 1/3] examples/ipsec-secgw: ipsec_sa structure cleanup

[Marcin Smoczynski]

Cleanup ipsec_sa structure by removing every field that is already in
the rte_ipsec_session structure:
 * cryptodev/security session union
 * action type
 * offload flags
 * security context
References to abovementioned fields are changed to direct references
to matching fields of rte_ipsec_session structure.

Such refactoring is needed to introduce many sessions per SA feature,
e.g. fallback session for inline offload processing.

Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
---
 examples/ipsec-secgw/esp.c           | 35 +++++++----
 examples/ipsec-secgw/ipsec.c         | 91 +++++++++++++++-------------
 examples/ipsec-secgw/ipsec.h         | 27 ++++++---
 examples/ipsec-secgw/ipsec_process.c | 30 ++++-----
 examples/ipsec-secgw/sa.c            | 66 ++++++++++----------
 5 files changed, 137 insertions(+), 112 deletions(-)

diff --git a/examples/ipsec-secgw/esp.c b/examples/ipsec-secgw/esp.c
index d6d7b1256..c1b49da1e 100644
--- a/examples/ipsec-secgw/esp.c
+++ b/examples/ipsec-secgw/esp.c
@@ -30,7 +30,8 @@ esp_inbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 	int32_t payload_len, ip_hdr_len;
 
 	RTE_ASSERT(sa != NULL);
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
+	if (ipsec_get_action_type(sa) ==
+			RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
 		return 0;
 
 	RTE_ASSERT(m != NULL);
@@ -148,13 +149,16 @@ esp_inbound_post(struct rte_mbuf *m, struct ipsec_sa *sa,
 	uint8_t *nexthdr, *pad_len;
 	uint8_t *padding;
 	uint16_t i;
+	struct rte_ipsec_session *ips;
 
 	RTE_ASSERT(m != NULL);
 	RTE_ASSERT(sa != NULL);
 	RTE_ASSERT(cop != NULL);
 
-	if ((sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
-			(sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
+	ips = ipsec_get_session(sa);
+
+	if ((ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
+			(ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
 		if (m->ol_flags & PKT_RX_SEC_OFFLOAD) {
 			if (m->ol_flags & PKT_RX_SEC_OFFLOAD_FAILED)
 				cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
@@ -169,8 +173,8 @@ esp_inbound_post(struct rte_mbuf *m, struct ipsec_sa *sa,
 		return -1;
 	}
 
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO &&
-	    sa->ol_flags & RTE_SECURITY_RX_HW_TRAILER_OFFLOAD) {
+	if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO &&
+		ips->security.ol_flags & RTE_SECURITY_RX_HW_TRAILER_OFFLOAD) {
 		nexthdr = &m->inner_esp_next_proto;
 	} else {
 		nexthdr = rte_pktmbuf_mtod_offset(m, uint8_t*,
@@ -225,10 +229,12 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 	struct rte_crypto_sym_op *sym_cop;
 	int32_t i;
 	uint16_t pad_payload_len, pad_len, ip_hdr_len;
+	struct rte_ipsec_session *ips;
 
 	RTE_ASSERT(m != NULL);
 	RTE_ASSERT(sa != NULL);
 
+	ips = ipsec_get_session(sa);
 	ip_hdr_len = 0;
 
 	ip4 = rte_pktmbuf_mtod(m, struct ip *);
@@ -277,9 +283,10 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 	}
 
 	/* Add trailer padding if it is not constructed by HW */
-	if (sa->type != RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
-	    (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO &&
-	     !(sa->ol_flags & RTE_SECURITY_TX_HW_TRAILER_OFFLOAD))) {
+	if (ips->type != RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+		(ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO &&
+		 !(ips->security.ol_flags &
+			 RTE_SECURITY_TX_HW_TRAILER_OFFLOAD))) {
 		padding = (uint8_t *)rte_pktmbuf_append(m, pad_len +
 							sa->digest_len);
 		if (unlikely(padding == NULL)) {
@@ -344,8 +351,9 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 		}
 	}
 
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
-		if (sa->ol_flags & RTE_SECURITY_TX_HW_TRAILER_OFFLOAD) {
+	if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		if (ips->security.ol_flags &
+				RTE_SECURITY_TX_HW_TRAILER_OFFLOAD) {
 			/* Set the inner esp next protocol for HW trailer */
 			m->inner_esp_next_proto = nlp;
 			m->packet_type |= RTE_PTYPE_TUNNEL_ESP;
@@ -448,11 +456,14 @@ esp_outbound_post(struct rte_mbuf *m,
 		  struct ipsec_sa *sa,
 		  struct rte_crypto_op *cop)
 {
+	enum rte_security_session_action_type type;
 	RTE_ASSERT(m != NULL);
 	RTE_ASSERT(sa != NULL);
 
-	if ((sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
-			(sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
+	type = ipsec_get_action_type(sa);
+
+	if ((type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
+			(type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
 		m->ol_flags |= PKT_TX_SEC_OFFLOAD;
 	} else {
 		RTE_ASSERT(cop != NULL);
diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index dc85adfe5..8c60bd84b 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -40,7 +40,8 @@ set_ipsec_conf(struct ipsec_sa *sa, struct rte_security_ipsec_xform *ipsec)
 }
 
 int
-create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips)
 {
 	struct rte_cryptodev_info cdev_info;
 	unsigned long cdev_id_qp = 0;
@@ -71,9 +72,9 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			ipsec_ctx->tbl[cdev_id_qp].id,
 			ipsec_ctx->tbl[cdev_id_qp].qp);
 
-	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
+	if (ips->type != RTE_SECURITY_ACTION_TYPE_NONE) {
 		struct rte_security_session_conf sess_conf = {
-			.action_type = sa->type,
+			.action_type = ips->type,
 			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 			{.ipsec = {
 				.spi = sa->spi,
@@ -90,7 +91,7 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 
 		};
 
-		if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
+		if (ips->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
 			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
 							rte_cryptodev_get_sec_ctx(
 							ipsec_ctx->tbl[cdev_id_qp].id);
@@ -98,9 +99,9 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			/* Set IPsec parameters in conf */
 			set_ipsec_conf(sa, &(sess_conf.ipsec));
 
-			sa->sec_session = rte_security_session_create(ctx,
+			ips->security.ses = rte_security_session_create(ctx,
 					&sess_conf, ipsec_ctx->session_priv_pool);
-			if (sa->sec_session == NULL) {
+			if (ips->security.ses == NULL) {
 				RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 				return -1;
@@ -110,10 +111,10 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			return -1;
 		}
 	} else {
-		sa->crypto_session = rte_cryptodev_sym_session_create(
+		ips->crypto.ses = rte_cryptodev_sym_session_create(
 				ipsec_ctx->session_pool);
 		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
-				sa->crypto_session, sa->xforms,
+				ips->crypto.ses, sa->xforms,
 				ipsec_ctx->session_priv_pool);
 
 		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
@@ -126,12 +127,13 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 }
 
 int
-create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips)
 {
 	int32_t ret = 0;
 	struct rte_security_ctx *sec_ctx;
 	struct rte_security_session_conf sess_conf = {
-		.action_type = sa->type,
+		.action_type = ips->type,
 		.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 		{.ipsec = {
 			.spi = sa->spi,
@@ -151,7 +153,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on port %u\n",
 		sa->spi, sa->portid);
 
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+	if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
 		struct rte_flow_error err;
 		const struct rte_security_capability *sec_cap;
 		int ret = 0;
@@ -165,9 +167,9 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 			return -1;
 		}
 
-		sa->sec_session = rte_security_session_create(sec_ctx,
+		ips->security.ses = rte_security_session_create(sec_ctx,
 				&sess_conf, skt_ctx->session_pool);
-		if (sa->sec_session == NULL) {
+		if (ips->security.ses == NULL) {
 			RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 			return -1;
@@ -177,7 +179,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 
 		/* iterate until ESP tunnel*/
 		while (sec_cap->action != RTE_SECURITY_ACTION_TYPE_NONE) {
-			if (sec_cap->action == sa->type &&
+			if (sec_cap->action == ips->type &&
 			    sec_cap->protocol ==
 				RTE_SECURITY_PROTOCOL_IPSEC &&
 			    sec_cap->ipsec.mode ==
@@ -193,8 +195,8 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 			return -1;
 		}
 
-		sa->ol_flags = sec_cap->ol_flags;
-		sa->security_ctx = sec_ctx;
+		ips->security.ol_flags = sec_cap->ol_flags;
+		ips->security.ctx = sec_ctx;
 		sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
 
 		if (IS_IP6(sa->flags)) {
@@ -223,7 +225,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 		sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
 
 		sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
-		sa->action[0].conf = sa->sec_session;
+		sa->action[0].conf = ips->security.ses;
 
 		sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
 
@@ -282,8 +284,8 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 			if (ret)
 				goto flow_create_failure;
 		} else if (sa->attr.egress &&
-			   (sa->ol_flags &
-				    RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
+				(ips->security.ol_flags &
+					RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
 			sa->action[1].type =
 					RTE_FLOW_ACTION_TYPE_PASSTHRU;
 			sa->action[2].type =
@@ -299,7 +301,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 				err.message);
 			return -1;
 		}
-	} else if (sa->type ==	RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
+	} else if (ips->type ==	RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
 		const struct rte_security_capability *sec_cap;
 
 		sec_ctx = (struct rte_security_ctx *)
@@ -327,9 +329,9 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 
 		sess_conf.userdata = (void *) sa;
 
-		sa->sec_session = rte_security_session_create(sec_ctx,
+		ips->security.ses = rte_security_session_create(sec_ctx,
 					&sess_conf, skt_ctx->session_pool);
-		if (sa->sec_session == NULL) {
+		if (ips->security.ses == NULL) {
 			RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 			return -1;
@@ -345,7 +347,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 		/* iterate until ESP tunnel*/
 		while (sec_cap->action !=
 				RTE_SECURITY_ACTION_TYPE_NONE) {
-			if (sec_cap->action == sa->type &&
+			if (sec_cap->action == ips->type &&
 			    sec_cap->protocol ==
 				RTE_SECURITY_PROTOCOL_IPSEC &&
 			    sec_cap->ipsec.mode ==
@@ -361,8 +363,8 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 			return -1;
 		}
 
-		sa->ol_flags = sec_cap->ol_flags;
-		sa->security_ctx = sec_ctx;
+		ips->security.ol_flags = sec_cap->ol_flags;
+		ips->security.ctx = sec_ctx;
 	}
 	sa->cdev_id_qp = 0;
 
@@ -409,6 +411,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 	struct ipsec_mbuf_metadata *priv;
 	struct rte_crypto_sym_op *sym_cop;
 	struct ipsec_sa *sa;
+	struct rte_ipsec_session *ips;
 
 	for (i = 0; i < nb_pkts; i++) {
 		if (unlikely(sas[i] == NULL)) {
@@ -422,16 +425,17 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 		priv = get_priv(pkts[i]);
 		sa = sas[i];
 		priv->sa = sa;
+		ips = ipsec_get_session(sa);
 
-		switch (sa->type) {
+		switch (ips->type) {
 		case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL:
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
 
-			if ((unlikely(sa->sec_session == NULL)) &&
-				create_lookaside_session(ipsec_ctx, sa)) {
+			if ((unlikely(ips->security.ses == NULL)) &&
+				create_lookaside_session(ipsec_ctx, sa, ips)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -440,7 +444,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			sym_cop->m_src = pkts[i];
 
 			rte_security_attach_session(&priv->cop,
-					sa->sec_session);
+				ips->security.ses);
 			break;
 		case RTE_SECURITY_ACTION_TYPE_NONE:
 
@@ -449,14 +453,14 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			rte_prefetch0(&priv->sym_cop);
 
-			if ((unlikely(sa->crypto_session == NULL)) &&
-				create_lookaside_session(ipsec_ctx, sa)) {
+			if ((unlikely(ips->crypto.ses == NULL)) &&
+				create_lookaside_session(ipsec_ctx, sa, ips)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
 
 			rte_crypto_op_attach_sym_session(&priv->cop,
-					sa->crypto_session);
+					ips->crypto.ses);
 
 			ret = xform_func(pkts[i], sa, &priv->cop);
 			if (unlikely(ret)) {
@@ -465,21 +469,22 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 			break;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
-			RTE_ASSERT(sa->sec_session != NULL);
+			RTE_ASSERT(ips->security.ses != NULL);
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
-			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
+			if (ips->security.ol_flags &
+				RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
-						sa->security_ctx,
-						sa->sec_session, pkts[i], NULL);
+					ips->security.ctx, ips->security.ses,
+					pkts[i], NULL);
 			continue;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
-			RTE_ASSERT(sa->sec_session != NULL);
+			RTE_ASSERT(ips->security.ses != NULL);
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
 			rte_security_attach_session(&priv->cop,
-					sa->sec_session);
+					ips->security.ses);
 
 			ret = xform_func(pkts[i], sa, &priv->cop);
 			if (unlikely(ret)) {
@@ -488,10 +493,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
-			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
+			if (ips->security.ol_flags &
+				RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
-						sa->security_ctx,
-						sa->sec_session, pkts[i], NULL);
+					ips->security.ctx, ips->security.ses,
+					pkts[i], NULL);
 			continue;
 		}
 
@@ -560,7 +566,8 @@ ipsec_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			RTE_ASSERT(sa != NULL);
 
-			if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {
+			if (ipsec_get_action_type(sa) ==
+				RTE_SECURITY_ACTION_TYPE_NONE) {
 				ret = xform_func(pkt, sa, cops[j]);
 				if (unlikely(ret)) {
 					rte_pktmbuf_free(pkt);
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index 1efa6e488..a4ad81b0e 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -87,10 +87,6 @@ struct ipsec_sa {
 	uint32_t cdev_id_qp;
 	uint64_t seq;
 	uint32_t salt;
-	union {
-		struct rte_cryptodev_sym_session *crypto_session;
-		struct rte_security_session *sec_session;
-	};
 	enum rte_crypto_cipher_algorithm cipher_algo;
 	enum rte_crypto_auth_algorithm auth_algo;
 	enum rte_crypto_aead_algorithm aead_algo;
@@ -114,11 +110,8 @@ struct ipsec_sa {
 		struct rte_crypto_sym_xform *xforms;
 		struct rte_security_ipsec_xform *sec_xform;
 	};
-	enum rte_security_session_action_type type;
 	enum rte_security_ipsec_sa_direction direction;
 	uint16_t portid;
-	struct rte_security_ctx *security_ctx;
-	uint32_t ol_flags;
 
 #define MAX_RTE_FLOW_PATTERN (4)
 #define MAX_RTE_FLOW_ACTIONS (3)
@@ -284,6 +277,20 @@ get_sym_cop(struct rte_crypto_op *cop)
 	return (cop + 1);
 }
 
+static inline struct rte_ipsec_session *
+ipsec_get_session(struct ipsec_sa *sa)
+{
+	return &sa->ips;
+}
+
+static inline enum rte_security_session_action_type
+ipsec_get_action_type(struct ipsec_sa *sa)
+{
+	struct rte_ipsec_session *ips;
+	ips = ipsec_get_session(sa);
+	return ips->type;
+}
+
 int
 inbound_sa_check(struct sa_ctx *sa_ctx, struct rte_mbuf *m, uint32_t sa_idx);
 
@@ -338,9 +345,11 @@ void
 enqueue_cop_burst(struct cdev_qp *cqp);
 
 int
-create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips);
 
 int
-create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa);
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips);
 
 #endif /* __IPSEC_H__ */
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index 868f1a28d..239d81ef6 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -94,22 +94,16 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
 
 	/* setup crypto section */
 	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		if (sa->crypto_session == NULL) {
-			rc = create_lookaside_session(ctx, sa);
-			if (rc != 0)
-				return rc;
-		}
-		ss->crypto.ses = sa->crypto_session;
+		RTE_ASSERT(ss->crypto.ses == NULL);
+		rc = create_lookaside_session(ctx, sa, ss);
+		if (rc != 0)
+			return rc;
 	/* setup session action type */
-	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
-		if (sa->sec_session == NULL) {
-			rc = create_lookaside_session(ctx, sa);
-			if (rc != 0)
-				return rc;
-		}
-		ss->security.ses = sa->sec_session;
-		ss->security.ctx = sa->security_ctx;
-		ss->security.ol_flags = sa->ol_flags;
+	} else if (ss->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
+		RTE_ASSERT(ss->security.ses == NULL);
+		rc = create_lookaside_session(ctx, sa, ss);
+		if (rc != 0)
+			return rc;
 	} else
 		RTE_ASSERT(0);
 
@@ -218,7 +212,7 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 		pg = grp + i;
 		sa = pg->id.ptr;
 
-		ips = &sa->ips;
+		ips = ipsec_get_session(sa);
 
 		/* no valid HW session for that SA, try to create one */
 		if (sa == NULL || (ips->crypto.ses == NULL &&
@@ -226,8 +220,8 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 			k = 0;
 
 		/* process packets inline */
-		else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
-				sa->type ==
+		else if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+				ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
 
 			satp = rte_ipsec_sa_type(ips->sa);
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index c3cf3bd1f..a3d8e4545 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -224,6 +224,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	struct parse_status *status)
 {
 	struct ipsec_sa *rule = NULL;
+	struct rte_ipsec_session *ips;
 	uint32_t ti; /*token index*/
 	uint32_t *ri /*rule index*/;
 	uint32_t cipher_algo_p = 0;
@@ -262,6 +263,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	if (atoi(tokens[1]) == INVALID_SPI)
 		return;
 	rule->spi = atoi(tokens[1]);
+	ips = ipsec_get_session(rule);
 
 	for (ti = 2; ti < n_tokens; ti++) {
 		if (strcmp(tokens[ti], "mode") == 0) {
@@ -558,18 +560,18 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 				return;
 
 			if (strcmp(tokens[ti], "inline-crypto-offload") == 0)
-				rule->type =
+				ips->type =
 					RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO;
 			else if (strcmp(tokens[ti],
 					"inline-protocol-offload") == 0)
-				rule->type =
+				ips->type =
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL;
 			else if (strcmp(tokens[ti],
 					"lookaside-protocol-offload") == 0)
-				rule->type =
+				ips->type =
 				RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL;
 			else if (strcmp(tokens[ti], "no-offload") == 0)
-				rule->type = RTE_SECURITY_ACTION_TYPE_NONE;
+				ips->type = RTE_SECURITY_ACTION_TYPE_NONE;
 			else {
 				APP_CHECK(0, status, "Invalid input \"%s\"",
 						tokens[ti]);
@@ -624,11 +626,11 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	if (status->status < 0)
 		return;
 
-	if ((rule->type != RTE_SECURITY_ACTION_TYPE_NONE) && (portid_p == 0))
+	if ((ips->type != RTE_SECURITY_ACTION_TYPE_NONE) && (portid_p == 0))
 		printf("Missing portid option, falling back to non-offload\n");
 
 	if (!type_p || !portid_p) {
-		rule->type = RTE_SECURITY_ACTION_TYPE_NONE;
+		ips->type = RTE_SECURITY_ACTION_TYPE_NONE;
 		rule->portid = -1;
 	}
 
@@ -640,6 +642,7 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 {
 	uint32_t i;
 	uint8_t a, b, c, d;
+	const struct rte_ipsec_session *ips;
 
 	printf("\tspi_%s(%3u):", inbound?"in":"out", sa->spi);
 
@@ -695,8 +698,10 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 		printf("Transport ");
 		break;
 	}
+
+	ips = &sa->ips;
 	printf(" type:");
-	switch (sa->type) {
+	switch (ips->type) {
 	case RTE_SECURITY_ACTION_TYPE_NONE:
 		printf("no-offload ");
 		break;
@@ -876,6 +881,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 	uint16_t iv_length, aad_length;
 	int inline_status;
 	int32_t rc;
+	struct rte_ipsec_session *ips;
 
 	/* for ESN upper 32 bits of SQN also need to be part of AAD */
 	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
@@ -890,9 +896,10 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 		}
 		*sa = entries[i];
 		sa->seq = 0;
+		ips = ipsec_get_session(sa);
 
-		if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
-			sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
+			ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
 			if (check_eth_dev_caps(sa->portid, inbound))
 				return -EINVAL;
 		}
@@ -907,7 +914,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 			sa->dst.ip.ip4 = rte_cpu_to_be_32(sa->dst.ip.ip4);
 			break;
 		case TRANSPORT:
-			if (sa->type ==
+			if (ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
 				inline_status =
 					sa_add_address_inline_crypto(sa);
@@ -918,6 +925,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 		}
 
 		if (sa->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) {
+			struct rte_ipsec_session *ips;
 			iv_length = 16;
 
 			sa_ctx->xf[idx].a.type = RTE_CRYPTO_SYM_XFORM_AEAD;
@@ -938,11 +946,12 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			sa->xforms = &sa_ctx->xf[idx].a;
 
-			if (sa->type ==
+			ips = ipsec_get_session(sa);
+			if (ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
-				sa->type ==
+				ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
-				rc = create_inline_session(skt_ctx, sa);
+				rc = create_inline_session(skt_ctx, sa, ips);
 				if (rc != 0) {
 					RTE_LOG(ERR, IPSEC_ESP,
 						"create_inline_session() failed\n");
@@ -1100,23 +1109,11 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm, const struct ipsec_sa *ss,
 }
 
 static int
-fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
-	const struct ipsec_sa *lsa)
+fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa)
 {
 	int32_t rc = 0;
 
 	ss->sa = sa;
-	ss->type = lsa->type;
-
-	/* setup crypto section */
-	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		ss->crypto.ses = lsa->crypto_session;
-	/* setup session action type */
-	} else {
-		ss->security.ses = lsa->sec_session;
-		ss->security.ctx = lsa->security_ctx;
-		ss->security.ol_flags = lsa->ol_flags;
-	}
 
 	if (ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
 		ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
@@ -1138,6 +1135,7 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 {
 	int rc;
 	struct rte_ipsec_sa_prm prm;
+	struct rte_ipsec_session *ips;
 	struct rte_ipv4_hdr v4  = {
 		.version_ihl = IPVERSION << 4 |
 			sizeof(v4) / RTE_IPV4_IHL_MULTIPLIER,
@@ -1162,7 +1160,10 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
+	/* init processing session */
+	ips = ipsec_get_session(lsa);
+	rc = fill_ipsec_session(ips, sa);
+
 	return rc;
 }
 
@@ -1386,6 +1387,7 @@ sa_check_offloads(uint16_t port_id, uint64_t *rx_offloads,
 {
 	struct ipsec_sa *rule;
 	uint32_t idx_sa;
+	enum rte_security_session_action_type rule_type;
 
 	*rx_offloads = 0;
 	*tx_offloads = 0;
@@ -1393,8 +1395,9 @@ sa_check_offloads(uint16_t port_id, uint64_t *rx_offloads,
 	/* Check for inbound rules that use offloads and use this port */
 	for (idx_sa = 0; idx_sa < nb_sa_in; idx_sa++) {
 		rule = &sa_in[idx_sa];
-		if ((rule->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
-				rule->type ==
+		rule_type = ipsec_get_action_type(rule);
+		if ((rule_type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+				rule_type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
 				&& rule->portid == port_id)
 			*rx_offloads |= DEV_RX_OFFLOAD_SECURITY;
@@ -1403,8 +1406,9 @@ sa_check_offloads(uint16_t port_id, uint64_t *rx_offloads,
 	/* Check for outbound rules that use offloads and use this port */
 	for (idx_sa = 0; idx_sa < nb_sa_out; idx_sa++) {
 		rule = &sa_out[idx_sa];
-		if ((rule->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
-				rule->type ==
+		rule_type = ipsec_get_action_type(rule);
+		if ((rule_type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+				rule_type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
 				&& rule->portid == port_id)
 			*tx_offloads |= DEV_TX_OFFLOAD_SECURITY;
-- 
2.17.1

Re: [dpdk-dev] [PATCH v3 1/3] examples/ipsec-secgw: ipsec_sa structure cleanup

[Ananyev, Konstantin]

> -----Original Message-----
> From: Smoczynski, MarcinX
> Sent: Monday, September 23, 2019 12:44 PM
> To: anoobj@marvell.com; akhil.goyal@nxp.com; Ananyev, Konstantin <konstantin.ananyev@intel.com>
> Cc: dev@dpdk.org; Smoczynski, MarcinX <marcinx.smoczynski@intel.com>
> Subject: [PATCH v3 1/3] examples/ipsec-secgw: ipsec_sa structure cleanup
> 
> Cleanup ipsec_sa structure by removing every field that is already in
> the rte_ipsec_session structure:
>  * cryptodev/security session union
>  * action type
>  * offload flags
>  * security context
> References to abovementioned fields are changed to direct references
> to matching fields of rte_ipsec_session structure.
> 
> Such refactoring is needed to introduce many sessions per SA feature,
> e.g. fallback session for inline offload processing.
> 
> Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
> ---

Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>

> 2.17.1

[dpdk-dev] [PATCH v3 2/3] examples/ipsec-secgw: add fallback session feature

[Marcin Smoczynski]

Inline processing is limited to a specified subset of traffic. It is
often unable to handle more complicated situations, such as fragmented
traffic. When using inline processing such traffic is dropped.

Introduce fallback session for inline processing allowing processing
packets that normally would be dropped. A fallback session is
configured by adding 'fallback' keyword with 'lookaside-none' or
'lookaside-protocol' parameter to an SA configuration.

Using IPsec anti-replay window or ESN feature with fallback session is
not yet supported when primary session is of type
'inline-protocol-offload' or fallback session is 'lookaside-protocol'
because SA sequence number is not synchronized between software and
hardware sessions. Fallback sessions are also limited to ingress IPsec
traffic.

Fallback session feature is not available in the legacy mode.

Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
---
 doc/guides/sample_app_ug/ipsec_secgw.rst |  20 ++++-
 examples/ipsec-secgw/esp.c               |   4 +-
 examples/ipsec-secgw/ipsec-secgw.c       |  16 ++--
 examples/ipsec-secgw/ipsec.c             |  10 +--
 examples/ipsec-secgw/ipsec.h             |  40 +++++++--
 examples/ipsec-secgw/ipsec_process.c     |  85 ++++++++++++------
 examples/ipsec-secgw/sa.c                | 110 +++++++++++++++++++----
 7 files changed, 221 insertions(+), 64 deletions(-)

diff --git a/doc/guides/sample_app_ug/ipsec_secgw.rst b/doc/guides/sample_app_ug/ipsec_secgw.rst
index ad2d79e75..6a39c56ce 100644
--- a/doc/guides/sample_app_ug/ipsec_secgw.rst
+++ b/doc/guides/sample_app_ug/ipsec_secgw.rst
@@ -401,7 +401,7 @@ The SA rule syntax is shown as follows:
 .. code-block:: console
 
     sa <dir> <spi> <cipher_algo> <cipher_key> <auth_algo> <auth_key>
-    <mode> <src_ip> <dst_ip> <action_type> <port_id>
+    <mode> <src_ip> <dst_ip> <action_type> <port_id> <fallback>
 
 where each options means:
 
@@ -573,6 +573,24 @@ where each options means:
 
    * *port_id X* X is a valid device number in decimal
 
+ ``<fallback>``
+
+ * Action type for ingress IPsec packets that inline processor failed to
+   process. IPsec anti-replay window and ESN is supported with fallback
+   processing session only when primary session is *lookaside-crypto-offload*
+   and fallback session is *lookaside-none*.
+
+ * Optional: Yes.
+
+ * Available options:
+
+   * *lookaside-none*: use automatically chosen cryptodev to process packets
+   * *lookaside-protocol*: lookaside protocol hardware offload
+
+ * Syntax:
+
+   * *fallback lookaside-none*
+   * *fallback lookaside-protocol*
 
 Example SA rules:
 
diff --git a/examples/ipsec-secgw/esp.c b/examples/ipsec-secgw/esp.c
index c1b49da1e..bfa7ff721 100644
--- a/examples/ipsec-secgw/esp.c
+++ b/examples/ipsec-secgw/esp.c
@@ -155,7 +155,7 @@ esp_inbound_post(struct rte_mbuf *m, struct ipsec_sa *sa,
 	RTE_ASSERT(sa != NULL);
 	RTE_ASSERT(cop != NULL);
 
-	ips = ipsec_get_session(sa);
+	ips = ipsec_get_primary_session(sa);
 
 	if ((ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
 			(ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
@@ -234,7 +234,7 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 	RTE_ASSERT(m != NULL);
 	RTE_ASSERT(sa != NULL);
 
-	ips = ipsec_get_session(sa);
+	ips = ipsec_get_primary_session(sa);
 	ip_hdr_len = 0;
 
 	ip4 = rte_pktmbuf_mtod(m, struct ip *);
diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index 0d1fd6af6..641ed3767 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -189,6 +189,7 @@ static uint32_t mtu_size = RTE_ETHER_MTU;
 
 /* application wide librte_ipsec/SA parameters */
 struct app_sa_prm app_sa_prm = {.enable = 0};
+static const char *cfgfile;
 
 struct lcore_rx_queue {
 	uint16_t port_id;
@@ -1462,12 +1463,7 @@ parse_args(int32_t argc, char **argv)
 				print_usage(prgname);
 				return -1;
 			}
-			if (parse_cfg_file(optarg) < 0) {
-				printf("parsing file \"%s\" failed\n",
-					optarg);
-				print_usage(prgname);
-				return -1;
-			}
+			cfgfile = optarg;
 			f_present = 1;
 			break;
 		case 'j':
@@ -2398,6 +2394,14 @@ main(int32_t argc, char **argv)
 	if (ret < 0)
 		rte_exit(EXIT_FAILURE, "Invalid parameters\n");
 
+	/* parse configuration file */
+	if (parse_cfg_file(cfgfile) < 0) {
+		printf("parsing file \"%s\" failed\n",
+			optarg);
+		print_usage(argv[0]);
+		return -1;
+	}
+
 	if ((unprotected_port_mask & enabled_port_mask) !=
 			unprotected_port_mask)
 		rte_exit(EXIT_FAILURE, "Invalid unprotected portmask 0x%x\n",
diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index 8c60bd84b..8b0441028 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -404,7 +404,7 @@ enqueue_cop(struct cdev_qp *cqp, struct rte_crypto_op *cop)
 
 static inline void
 ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
-		struct rte_mbuf *pkts[], struct ipsec_sa *sas[],
+		struct rte_mbuf *pkts[], void *sas[],
 		uint16_t nb_pkts)
 {
 	int32_t ret = 0, i;
@@ -423,9 +423,9 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 		rte_prefetch0(pkts[i]);
 
 		priv = get_priv(pkts[i]);
-		sa = sas[i];
+		sa = ipsec_mask_saptr(sas[i]);
 		priv->sa = sa;
-		ips = ipsec_get_session(sa);
+		ips = ipsec_get_primary_session(sa);
 
 		switch (ips->type) {
 		case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL:
@@ -586,7 +586,7 @@ uint16_t
 ipsec_inbound(struct ipsec_ctx *ctx, struct rte_mbuf *pkts[],
 		uint16_t nb_pkts, uint16_t len)
 {
-	struct ipsec_sa *sas[nb_pkts];
+	void *sas[nb_pkts];
 
 	inbound_sa_lookup(ctx->sa_ctx, pkts, sas, nb_pkts);
 
@@ -606,7 +606,7 @@ uint16_t
 ipsec_outbound(struct ipsec_ctx *ctx, struct rte_mbuf *pkts[],
 		uint32_t sa_idx[], uint16_t nb_pkts, uint16_t len)
 {
-	struct ipsec_sa *sas[nb_pkts];
+	void *sas[nb_pkts];
 
 	outbound_sa_lookup(ctx->sa_ctx, sa_idx, sas, nb_pkts);
 
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index a4ad81b0e..8e075216c 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -81,12 +81,32 @@ struct app_sa_prm {
 
 extern struct app_sa_prm app_sa_prm;
 
+enum {
+	IPSEC_SESSION_PRIMARY = 0,
+	IPSEC_SESSION_FALLBACK = 1,
+	IPSEC_SESSION_MAX
+};
+
+#define IPSEC_SA_OFFLOAD_FALLBACK_FLAG (1)
+
+static inline struct ipsec_sa *
+ipsec_mask_saptr(void *ptr)
+{
+	uintptr_t i = (uintptr_t)ptr;
+	static const uintptr_t mask = IPSEC_SA_OFFLOAD_FALLBACK_FLAG;
+
+	i &= ~mask;
+
+	return (struct ipsec_sa *)i;
+}
+
 struct ipsec_sa {
-	struct rte_ipsec_session ips; /* one session per sa for now */
+	struct rte_ipsec_session sessions[IPSEC_SESSION_MAX];
 	uint32_t spi;
 	uint32_t cdev_id_qp;
 	uint64_t seq;
 	uint32_t salt;
+	uint32_t fallback_sessions;
 	enum rte_crypto_cipher_algorithm cipher_algo;
 	enum rte_crypto_auth_algorithm auth_algo;
 	enum rte_crypto_aead_algorithm aead_algo;
@@ -210,7 +230,7 @@ struct cnt_blk {
 struct traffic_type {
 	const uint8_t *data[MAX_PKT_BURST * 2];
 	struct rte_mbuf *pkts[MAX_PKT_BURST * 2];
-	struct ipsec_sa *saptr[MAX_PKT_BURST * 2];
+	void *saptr[MAX_PKT_BURST * 2];
 	uint32_t res[MAX_PKT_BURST * 2];
 	uint32_t num;
 };
@@ -278,16 +298,22 @@ get_sym_cop(struct rte_crypto_op *cop)
 }
 
 static inline struct rte_ipsec_session *
-ipsec_get_session(struct ipsec_sa *sa)
+ipsec_get_primary_session(struct ipsec_sa *sa)
+{
+	return &sa->sessions[IPSEC_SESSION_PRIMARY];
+}
+
+static inline struct rte_ipsec_session *
+ipsec_get_fallback_session(struct ipsec_sa *sa)
 {
-	return &sa->ips;
+	return &sa->sessions[IPSEC_SESSION_FALLBACK];
 }
 
 static inline enum rte_security_session_action_type
 ipsec_get_action_type(struct ipsec_sa *sa)
 {
 	struct rte_ipsec_session *ips;
-	ips = ipsec_get_session(sa);
+	ips = ipsec_get_primary_session(sa);
 	return ips->type;
 }
 
@@ -296,11 +322,11 @@ inbound_sa_check(struct sa_ctx *sa_ctx, struct rte_mbuf *m, uint32_t sa_idx);
 
 void
 inbound_sa_lookup(struct sa_ctx *sa_ctx, struct rte_mbuf *pkts[],
-		struct ipsec_sa *sa[], uint16_t nb_pkts);
+		void *sa[], uint16_t nb_pkts);
 
 void
 outbound_sa_lookup(struct sa_ctx *sa_ctx, uint32_t sa_idx[],
-		struct ipsec_sa *sa[], uint16_t nb_pkts);
+		void *sa[], uint16_t nb_pkts);
 
 void
 sp4_init(struct socket_ctx *ctx, int32_t socket_id);
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index 239d81ef6..2eb5c8b34 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -18,7 +18,6 @@
 	(((t) & RTE_IPSEC_SATP_IPV_MASK) == RTE_IPSEC_SATP_IPV4)) || \
 	((t) & RTE_IPSEC_SATP_MODE_MASK) == RTE_IPSEC_SATP_MODE_TUNLV4)
 
-
 /* helper routine to free bulk of packets */
 static inline void
 free_pkts(struct rte_mbuf *mb[], uint32_t n)
@@ -118,7 +117,7 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
  * group input packets byt the SA they belong to.
  */
 static uint32_t
-sa_group(struct ipsec_sa *sa_ptr[], struct rte_mbuf *pkts[],
+sa_group(void *sa_ptr[], struct rte_mbuf *pkts[],
 	struct rte_ipsec_group grp[], uint32_t num)
 {
 	uint32_t i, n, spi;
@@ -185,6 +184,37 @@ copy_to_trf(struct ipsec_traffic *trf, uint64_t satp, struct rte_mbuf *mb[],
 	out->num += num;
 }
 
+static uint32_t
+ipsec_prepare_crypto_group(struct ipsec_ctx *ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips, struct rte_mbuf **m,
+		unsigned int cnt)
+{
+	struct cdev_qp *cqp;
+	struct rte_crypto_op *cop[cnt];
+	uint32_t j, k;
+	struct ipsec_mbuf_metadata *priv;
+
+	cqp = &ctx->tbl[sa->cdev_id_qp];
+
+	/* for that app each mbuf has it's own crypto op */
+	for (j = 0; j != cnt; j++) {
+		priv = get_priv(m[j]);
+		cop[j] = &priv->cop;
+		/*
+		 * this is just to satisfy inbound_sa_check()
+		 * should be removed in future.
+		 */
+		priv->sa = sa;
+	}
+
+	/* prepare and enqueue crypto ops */
+	k = rte_ipsec_pkt_crypto_prepare(ips, m, cop, cnt);
+	if (k != 0)
+		enqueue_cop_bulk(cqp, cop, k);
+
+	return k;
+}
+
 /*
  * Process ipsec packets.
  * If packet belong to SA that is subject of inline-crypto,
@@ -201,18 +231,15 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 	struct ipsec_mbuf_metadata *priv;
 	struct rte_ipsec_group *pg;
 	struct rte_ipsec_session *ips;
-	struct cdev_qp *cqp;
-	struct rte_crypto_op *cop[RTE_DIM(trf->ipsec.pkts)];
 	struct rte_ipsec_group grp[RTE_DIM(trf->ipsec.pkts)];
 
 	n = sa_group(trf->ipsec.saptr, trf->ipsec.pkts, grp, trf->ipsec.num);
 
 	for (i = 0; i != n; i++) {
-
 		pg = grp + i;
-		sa = pg->id.ptr;
+		sa = ipsec_mask_saptr(pg->id.ptr);
 
-		ips = ipsec_get_session(sa);
+		ips = ipsec_get_primary_session(sa);
 
 		/* no valid HW session for that SA, try to create one */
 		if (sa == NULL || (ips->crypto.ses == NULL &&
@@ -224,6 +251,7 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 				ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
 
+			/* get SA type */
 			satp = rte_ipsec_sa_type(ips->sa);
 
 			/*
@@ -236,30 +264,33 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 				priv->sa = sa;
 			}
 
-			k = rte_ipsec_pkt_process(ips, pg->m, pg->cnt);
-			copy_to_trf(trf, satp, pg->m, k);
+			/* fallback to cryptodev with RX packets which inline
+			 * processor was unable to process
+			 */
+			if (pg->id.val & IPSEC_SA_OFFLOAD_FALLBACK_FLAG) {
+				/* offload packets to cryptodev */
+				struct rte_ipsec_session *fallback;
+
+				fallback = ipsec_get_fallback_session(sa);
+				if (fallback->crypto.ses == NULL &&
+					fill_ipsec_session(fallback, ctx, sa)
+					!= 0)
+					k = 0;
+				else
+					k = ipsec_prepare_crypto_group(ctx, sa,
+						fallback, pg->m, pg->cnt);
+			} else {
+				/* finish processing of packets successfully
+				 * decrypted by an inline processor
+				 */
+				k = rte_ipsec_pkt_process(ips, pg->m, pg->cnt);
+				copy_to_trf(trf, satp, pg->m, k);
 
+			}
 		/* enqueue packets to crypto dev */
 		} else {
-
-			cqp = &ctx->tbl[sa->cdev_id_qp];
-
-			/* for that app each mbuf has it's own crypto op */
-			for (j = 0; j != pg->cnt; j++) {
-				priv = get_priv(pg->m[j]);
-				cop[j] = &priv->cop;
-				/*
-				 * this is just to satisfy inbound_sa_check()
-				 * should be removed in future.
-				 */
-				priv->sa = sa;
-			}
-
-			/* prepare and enqueue crypto ops */
-			k = rte_ipsec_pkt_crypto_prepare(ips, pg->m, cop,
+			k = ipsec_prepare_crypto_group(ctx, sa, ips, pg->m,
 				pg->cnt);
-			if (k != 0)
-				enqueue_cop_bulk(cqp, cop, k);
 		}
 
 		/* drop packets that cannot be enqueued/processed */
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index a3d8e4545..700d52490 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -31,6 +31,8 @@
 
 #define IP6_FULL_MASK (sizeof(((struct ip_addr *)NULL)->ip.ip6.ip6) * CHAR_BIT)
 
+#define MBUF_NO_SEC_OFFLOAD(m) ((m->ol_flags & PKT_RX_SEC_OFFLOAD) == 0)
+
 struct supported_cipher_algo {
 	const char *keyword;
 	enum rte_crypto_cipher_algorithm algo;
@@ -235,6 +237,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	uint32_t mode_p = 0;
 	uint32_t type_p = 0;
 	uint32_t portid_p = 0;
+	uint32_t fallback_p = 0;
 
 	if (strcmp(tokens[0], "in") == 0) {
 		ri = &nb_sa_in;
@@ -245,6 +248,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 			return;
 
 		rule = &sa_in[*ri];
+		rule->direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS;
 	} else {
 		ri = &nb_sa_out;
 
@@ -254,6 +258,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 			return;
 
 		rule = &sa_out[*ri];
+		rule->direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS;
 	}
 
 	/* spi number */
@@ -263,7 +268,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	if (atoi(tokens[1]) == INVALID_SPI)
 		return;
 	rule->spi = atoi(tokens[1]);
-	ips = ipsec_get_session(rule);
+	ips = ipsec_get_primary_session(rule);
 
 	for (ti = 2; ti < n_tokens; ti++) {
 		if (strcmp(tokens[ti], "mode") == 0) {
@@ -596,6 +601,43 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 			continue;
 		}
 
+		if (strcmp(tokens[ti], "fallback") == 0) {
+			struct rte_ipsec_session *fb;
+
+			APP_CHECK(app_sa_prm.enable, status, "Fallback session "
+				"not allowed for legacy mode.");
+			if (status->status < 0)
+				return;
+			APP_CHECK(rule->direction ==
+				RTE_SECURITY_IPSEC_SA_DIR_INGRESS, status,
+				"Fallback session not allowed for egress "
+				"rule");
+			if (status->status < 0)
+				return;
+			APP_CHECK_PRESENCE(fallback_p, tokens[ti], status);
+			if (status->status < 0)
+				return;
+			INCREMENT_TOKEN_INDEX(ti, n_tokens, status);
+			if (status->status < 0)
+				return;
+			fb = ipsec_get_fallback_session(rule);
+			if (strcmp(tokens[ti], "lookaside-none") == 0) {
+				fb->type = RTE_SECURITY_ACTION_TYPE_NONE;
+			} else if (strcmp(tokens[ti], "lookaside-protocol")
+				== 0) {
+				fb->type =
+				RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL;
+			} else {
+				APP_CHECK(0, status, "unrecognized fallback "
+					"type %s.", tokens[ti]);
+				return;
+			}
+
+			rule->fallback_sessions = 1;
+			fallback_p = 1;
+			continue;
+		}
+
 		/* unrecognizeable input */
 		APP_CHECK(0, status, "unrecognized input \"%s\"",
 			tokens[ti]);
@@ -643,6 +685,7 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 	uint32_t i;
 	uint8_t a, b, c, d;
 	const struct rte_ipsec_session *ips;
+	const struct rte_ipsec_session *fallback_ips;
 
 	printf("\tspi_%s(%3u):", inbound?"in":"out", sa->spi);
 
@@ -699,7 +742,7 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 		break;
 	}
 
-	ips = &sa->ips;
+	ips = &sa->sessions[IPSEC_SESSION_PRIMARY];
 	printf(" type:");
 	switch (ips->type) {
 	case RTE_SECURITY_ACTION_TYPE_NONE:
@@ -715,6 +758,22 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 		printf("lookaside-protocol-offload ");
 		break;
 	}
+
+	fallback_ips = &sa->sessions[IPSEC_SESSION_FALLBACK];
+	if (fallback_ips != NULL && sa->fallback_sessions > 0) {
+		printf("inline fallback:");
+		switch (fallback_ips->type) {
+		case RTE_SECURITY_ACTION_TYPE_NONE:
+			printf("lookaside-none ");
+			break;
+		case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL:
+			printf("lookaside-protocol ");
+			break;
+		default:
+			printf("invalid ");
+			break;
+		}
+	}
 	printf("\n");
 }
 
@@ -896,7 +955,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 		}
 		*sa = entries[i];
 		sa->seq = 0;
-		ips = ipsec_get_session(sa);
+		ips = ipsec_get_primary_session(sa);
 
 		if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
 			ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
@@ -904,9 +963,6 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 				return -EINVAL;
 		}
 
-		sa->direction = (inbound == 1) ?
-				RTE_SECURITY_IPSEC_SA_DIR_INGRESS :
-				RTE_SECURITY_IPSEC_SA_DIR_EGRESS;
 
 		switch (WITHOUT_TRANSPORT_VERSION(sa->flags)) {
 		case IP4_TUNNEL:
@@ -946,7 +1002,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			sa->xforms = &sa_ctx->xf[idx].a;
 
-			ips = ipsec_get_session(sa);
+			ips = ipsec_get_primary_session(sa);
 			if (ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
 				ips->type ==
@@ -1160,9 +1216,15 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	/* init processing session */
-	ips = ipsec_get_session(lsa);
+	/* init primary processing session */
+	ips = ipsec_get_primary_session(lsa);
 	rc = fill_ipsec_session(ips, sa);
+	if (rc != 0)
+		return rc;
+
+	/* init inline fallback processing session */
+	if (lsa->fallback_sessions == 1)
+		rc = fill_ipsec_session(ipsec_get_fallback_session(lsa), sa);
 
 	return rc;
 }
@@ -1318,13 +1380,14 @@ inbound_sa_check(struct sa_ctx *sa_ctx, struct rte_mbuf *m, uint32_t sa_idx)
 
 static inline void
 single_inbound_lookup(struct ipsec_sa *sadb, struct rte_mbuf *pkt,
-		struct ipsec_sa **sa_ret)
+		void **sa_ret)
 {
 	struct rte_esp_hdr *esp;
 	struct ip *ip;
 	uint32_t *src4_addr;
 	uint8_t *src6_addr;
 	struct ipsec_sa *sa;
+	void *result_sa;
 
 	*sa_ret = NULL;
 
@@ -1334,33 +1397,48 @@ single_inbound_lookup(struct ipsec_sa *sadb, struct rte_mbuf *pkt,
 	if (esp->spi == INVALID_SPI)
 		return;
 
-	sa = &sadb[SPI2IDX(rte_be_to_cpu_32(esp->spi))];
+	result_sa = sa = &sadb[SPI2IDX(rte_be_to_cpu_32(esp->spi))];
 	if (rte_be_to_cpu_32(esp->spi) != sa->spi)
 		return;
 
+	/*
+	 * Mark need for inline offload fallback on the LSB of SA pointer.
+	 * Thanks to packet grouping mechanism which ipsec_process is using
+	 * packets marked for fallback processing will form separate group.
+	 *
+	 * Because it is not safe to use SA pointer it is casted to generic
+	 * pointer to prevent from unintentional use. Use ipsec_mask_saptr
+	 * to get valid struct pointer.
+	 */
+	if (MBUF_NO_SEC_OFFLOAD(pkt) && sa->fallback_sessions > 0) {
+		uintptr_t intsa = (uintptr_t)sa;
+		intsa |= IPSEC_SA_OFFLOAD_FALLBACK_FLAG;
+		result_sa = (void *)intsa;
+	}
+
 	switch (WITHOUT_TRANSPORT_VERSION(sa->flags)) {
 	case IP4_TUNNEL:
 		src4_addr = RTE_PTR_ADD(ip, offsetof(struct ip, ip_src));
 		if ((ip->ip_v == IPVERSION) &&
 				(sa->src.ip.ip4 == *src4_addr) &&
 				(sa->dst.ip.ip4 == *(src4_addr + 1)))
-			*sa_ret = sa;
+			*sa_ret = result_sa;
 		break;
 	case IP6_TUNNEL:
 		src6_addr = RTE_PTR_ADD(ip, offsetof(struct ip6_hdr, ip6_src));
 		if ((ip->ip_v == IP6_VERSION) &&
 				!memcmp(&sa->src.ip.ip6.ip6, src6_addr, 16) &&
 				!memcmp(&sa->dst.ip.ip6.ip6, src6_addr + 16, 16))
-			*sa_ret = sa;
+			*sa_ret = result_sa;
 		break;
 	case TRANSPORT:
-		*sa_ret = sa;
+		*sa_ret = result_sa;
 	}
 }
 
 void
 inbound_sa_lookup(struct sa_ctx *sa_ctx, struct rte_mbuf *pkts[],
-		struct ipsec_sa *sa[], uint16_t nb_pkts)
+		void *sa[], uint16_t nb_pkts)
 {
 	uint32_t i;
 
@@ -1370,7 +1448,7 @@ inbound_sa_lookup(struct sa_ctx *sa_ctx, struct rte_mbuf *pkts[],
 
 void
 outbound_sa_lookup(struct sa_ctx *sa_ctx, uint32_t sa_idx[],
-		struct ipsec_sa *sa[], uint16_t nb_pkts)
+		void *sa[], uint16_t nb_pkts)
 {
 	uint32_t i;
 
-- 
2.17.1

Re: [dpdk-dev] [PATCH v3 2/3] examples/ipsec-secgw: add fallback session feature

[Ananyev, Konstantin]

> 
> Inline processing is limited to a specified subset of traffic. It is
> often unable to handle more complicated situations, such as fragmented
> traffic. When using inline processing such traffic is dropped.
> 
> Introduce fallback session for inline processing allowing processing
> packets that normally would be dropped. A fallback session is
> configured by adding 'fallback' keyword with 'lookaside-none' or
> 'lookaside-protocol' parameter to an SA configuration.
> 
> Using IPsec anti-replay window or ESN feature with fallback session is
> not yet supported when primary session is of type
> 'inline-protocol-offload' or fallback session is 'lookaside-protocol'
> because SA sequence number is not synchronized between software and
> hardware sessions. Fallback sessions are also limited to ingress IPsec
> traffic.
> 
> Fallback session feature is not available in the legacy mode.
> 
> Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
> ---

Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>

> 2.17.1

[dpdk-dev] [PATCH v3 3/3] examples/ipsec-secgw: add offload fallback tests

[Marcin Smoczynski]

Add tests for offload fallback feature; add inbound config modificator
SGW_CFG_XPRM_IN (offload fallback setting can be set only for inbound
SAs). Tests are using cryptodev for outbound SA.

To test fragmentation with QAT set:
MULTI_SEG_TEST="--reassemble=4096 --cryptodev_mask=0x5555"

Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
---
 examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh         | 4 ++--
 .../test/trs_aesgcm_inline_crypto_fallback_defs.sh          | 5 +++++
 examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh         | 6 ++++--
 .../test/tun_aesgcm_inline_crypto_fallback_defs.sh          | 5 +++++
 4 files changed, 16 insertions(+), 4 deletions(-)
 create mode 100644 examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
 create mode 100644 examples/ipsec-secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh

diff --git a/examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh b/examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh
index f6c5bf5a7..17f2f86d2 100644
--- a/examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh
+++ b/examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh
@@ -29,11 +29,11 @@ sp ipv6 out esp bypass pri 1 sport 0:65535 dport 0:65535
 #SA in rules
 sa in 7 aead_algo aes-128-gcm \
 aead_key de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef \
-mode transport ${SGW_CFG_XPRM}
+mode transport ${SGW_CFG_XPRM} ${SGW_CFG_XPRM_IN}
 
 sa in 9 aead_algo aes-128-gcm \
 aead_key de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef \
-mode transport ${SGW_CFG_XPRM}
+mode transport ${SGW_CFG_XPRM} ${SGW_CFG_XPRM_IN}
 
 #SA out rules
 sa out 7 aead_algo aes-128-gcm \
diff --git a/examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh b/examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
new file mode 100644
index 000000000..875a7457d
--- /dev/null
+++ b/examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
@@ -0,0 +1,5 @@
+#! /bin/bash
+
+. ${DIR}/trs_aesgcm_defs.sh
+
+SGW_CFG_XPRM_IN='port_id 0 type inline-crypto-offload fallback lookaside-none'
diff --git a/examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh b/examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh
index 278377967..7490baded 100644
--- a/examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh
+++ b/examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh
@@ -29,11 +29,13 @@ sp ipv6 out esp bypass pri 1 sport 0:65535 dport 0:65535
 #SA in rules
 sa in 7 aead_algo aes-128-gcm \
 aead_key de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef \
-mode ipv4-tunnel src ${REMOTE_IPV4} dst ${LOCAL_IPV4} ${SGW_CFG_XPRM}
+mode ipv4-tunnel src ${REMOTE_IPV4} dst ${LOCAL_IPV4} ${SGW_CFG_XPRM} \
+${SGW_CFG_XPRM_IN}
 
 sa in 9 aead_algo aes-128-gcm \
 aead_key de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef \
-mode ipv6-tunnel src ${REMOTE_IPV6} dst ${LOCAL_IPV6} ${SGW_CFG_XPRM}
+mode ipv6-tunnel src ${REMOTE_IPV6} dst ${LOCAL_IPV6} ${SGW_CFG_XPRM} \
+${SGW_CFG_XPRM_IN}
 
 #SA out rules
 sa out 7 aead_algo aes-128-gcm \
diff --git a/examples/ipsec-secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh b/examples/ipsec-secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh
new file mode 100644
index 000000000..696848432
--- /dev/null
+++ b/examples/ipsec-secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh
@@ -0,0 +1,5 @@
+#! /bin/bash
+
+. ${DIR}/tun_aesgcm_defs.sh
+
+SGW_CFG_XPRM_IN='port_id 0 type inline-crypto-offload fallback lookaside-none'
-- 
2.17.1

Re: [dpdk-dev] [PATCH v3 3/3] examples/ipsec-secgw: add offload fallback tests

[Ananyev, Konstantin]

> Add tests for offload fallback feature; add inbound config modificator
> SGW_CFG_XPRM_IN (offload fallback setting can be set only for inbound
> SAs). Tests are using cryptodev for outbound SA.
> 
> To test fragmentation with QAT set:
> MULTI_SEG_TEST="--reassemble=4096 --cryptodev_mask=0x5555"
> 
> Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
> ---

Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>

> 2.17.1

Re: [dpdk-dev] [PATCH v3 0/3] add fallback session

[Smoczynski, MarcinX]

Joseph, is this patchset ok with you after changes? I've changed second
patch description and sample application documentation regarding this
feature.

> -----Original Message-----
> From: Smoczynski, MarcinX
> Sent: Monday, September 23, 2019 1:44 PM
> To: anoobj@marvell.com; akhil.goyal@nxp.com; Ananyev, Konstantin
> <konstantin.ananyev@intel.com>
> Cc: dev@dpdk.org; Smoczynski, MarcinX <marcinx.smoczynski@intel.com>
> Subject: [PATCH v3 0/3] add fallback session
> 
> Add fallback session feature allowing to process packets that inline processor
> is unable to handle (e.g. fragmented traffic). Processing takes place in a
> secondary session defined for SA in a configuration file.
> 
> This feature is limited to ingress IPsec traffic only. IPsec anti-replay window
> and ESN are supported in conjunction with fallback session when following
> conditions are met:
>  * primary session is 'inline-crypto-offload,
>  * fallback sessions is 'lookaside-none'.
> 
> v2 to v3 changes:
>  - doc and commit log update - explicitly state feature limitations
> 
> v1 to v2 changes:
>  - disable fallback offload for outbound SAs
>  - add test scripts
> 
> Marcin Smoczynski (3):
>   examples/ipsec-secgw: ipsec_sa structure cleanup
>   examples/ipsec-secgw: add fallback session feature
>   examples/ipsec-secgw: add offload fallback tests
> 
>  doc/guides/sample_app_ug/ipsec_secgw.rst      |  20 ++-
>  examples/ipsec-secgw/esp.c                    |  35 ++--
>  examples/ipsec-secgw/ipsec-secgw.c            |  16 +-
>  examples/ipsec-secgw/ipsec.c                  |  99 ++++++-----
>  examples/ipsec-secgw/ipsec.h                  |  61 +++++--
>  examples/ipsec-secgw/ipsec_process.c          | 113 +++++++-----
>  examples/ipsec-secgw/sa.c                     | 164 +++++++++++++-----
>  .../test/trs_aesgcm_common_defs.sh            |   4 +-
>  .../trs_aesgcm_inline_crypto_fallback_defs.sh |   5 +
>  .../test/tun_aesgcm_common_defs.sh            |   6 +-
>  .../tun_aesgcm_inline_crypto_fallback_defs.sh |   5 +
>  11 files changed, 361 insertions(+), 167 deletions(-)  create mode 100644
> examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
>  create mode 100644 examples/ipsec-
> secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh
> 
> --
> 2.17.1

Re: [dpdk-dev] [PATCH v3 0/3] add fallback session

[Anoob Joseph]

Hi Marcin, Konstantin,

I've few more observations regarding the proposed feature.

1. From what I understood, if an ESP packet ends up on an unprotected interface and doesn't have 'PKT_RX_SEC_OFFLOAD' bit set, then the packet would be looked up to see the associated SA and then fallback session is figured out and then further processing is done.

Can you confirm if I understood the sequence correctly? If yes, then aren't we doing an extra lookup in the s/w? The packet may be looked by the h/w using rte_flow and that information could be used to determine the SA. Also, if the ESP packet is expected to be forwarded, then the above logic will add an unnecessary lookup even after your h/w has detected that the packet need not be security processed.

2. The solution proposed here seems like adding the handling in ipsec-secgw instead of ipsec library. In other words, this feature is not getting added in ipsec library, which was supposed to simplify the whole ipsec usage in DPDK, but fails to handle the case of fragmentation. Also, since the fallback feature is entirely done in the application, it begs the question why the same feature is omitted for legacy use case.

3. It seems like ordering won't be maintained once this processing is done. Again, this is the sequence I understood. Please correct me if I missed something,
       a. Application receives a bunch of packets (let's say 6 packets), in which few are fragmented (P3 & P4) and the rest can be inline processed.
       b. Application receives P1->P2->P3->P4->P5->P6 (in this, P1, P2, P5, P6 are inline processed successfully) and P4 & P5 are the fragments
       c. Application groups packets. P1->P2->P5->P6 becomes one group and P3->P4 becomes another and goes for fallback processing.
Now how is ordering maintained? I couldn't figure out how that is done in this case.

Thanks,
Anoob

> -----Original Message-----
> From: dev <dev-bounces@dpdk.org> On Behalf Of Marcin Smoczynski
> Sent: Monday, September 23, 2019 5:14 PM
> To: Anoob Joseph <anoobj@marvell.com>; akhil.goyal@nxp.com;
> konstantin.ananyev@intel.com
> Cc: dev@dpdk.org; Marcin Smoczynski <marcinx.smoczynski@intel.com>
> Subject: [dpdk-dev] [PATCH v3 0/3] add fallback session
> 
> Add fallback session feature allowing to process packets that inline processor
> is unable to handle (e.g. fragmented traffic). Processing takes place in a
> secondary session defined for SA in a configuration file.
> 
> This feature is limited to ingress IPsec traffic only. IPsec anti-replay window
> and ESN are supported in conjunction with fallback session when following
> conditions are met:
>  * primary session is 'inline-crypto-offload,
>  * fallback sessions is 'lookaside-none'.
> 
> v2 to v3 changes:
>  - doc and commit log update - explicitly state feature limitations
> 
> v1 to v2 changes:
>  - disable fallback offload for outbound SAs
>  - add test scripts
> 
> Marcin Smoczynski (3):
>   examples/ipsec-secgw: ipsec_sa structure cleanup
>   examples/ipsec-secgw: add fallback session feature
>   examples/ipsec-secgw: add offload fallback tests
> 
>  doc/guides/sample_app_ug/ipsec_secgw.rst      |  20 ++-
>  examples/ipsec-secgw/esp.c                    |  35 ++--
>  examples/ipsec-secgw/ipsec-secgw.c            |  16 +-
>  examples/ipsec-secgw/ipsec.c                  |  99 ++++++-----
>  examples/ipsec-secgw/ipsec.h                  |  61 +++++--
>  examples/ipsec-secgw/ipsec_process.c          | 113 +++++++-----
>  examples/ipsec-secgw/sa.c                     | 164 +++++++++++++-----
>  .../test/trs_aesgcm_common_defs.sh            |   4 +-
>  .../trs_aesgcm_inline_crypto_fallback_defs.sh |   5 +
>  .../test/tun_aesgcm_common_defs.sh            |   6 +-
>  .../tun_aesgcm_inline_crypto_fallback_defs.sh |   5 +
>  11 files changed, 361 insertions(+), 167 deletions(-)  create mode 100644
> examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
>  create mode 100644 examples/ipsec-
> secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh
> 
> --
> 2.17.1

Re: [dpdk-dev] [PATCH v3 0/3] add fallback session

[Ananyev, Konstantin]

Hi Anoob,
Answers/comments inline.
Marcin, please correct me, if I missed something.
Konstantin

> 
> Hi Marcin, Konstantin,
> 
> I've few more observations regarding the proposed feature.
> 
> 1. From what I understood, if an ESP packet ends up on an unprotected interface and doesn't have 'PKT_RX_SEC_OFFLOAD' bit set, then the
> packet would be looked up to see the associated SA and then fallback session is figured out and then further processing is done.
> 
> Can you confirm if I understood the sequence correctly? If yes, then aren't we doing an extra lookup in the s/w? The packet may be looked
> by the h/w using rte_flow and that information could be used to determine the SA. Also, if the ESP packet is expected to be forwarded, then
> the above logic will add an unnecessary lookup even after your h/w has detected that the packet need not be security processed.

Not sure I understood your whole statement above.
AFAIK, right now (with dpdk master) for unprotected iface it works like that:

1.  slit incoming traffic into 3 groups: ESP packets, IPv4 packets, IPv6 packets.
For ESP packets: 
2. perform SAD lookup
    a. if it fails (non SA found for that SPI), then drop the packet.
    b. otherwise (SA found) process the packet using found SA

What fall-back patch adds:
Before step 2.b check:
does that SA has its primary session  with type INLINE-CRYPTO and   
does HW fail to do IPsec realted processing for it (by some reason)?
If yes, then mark this packet to be processed by fall-back session.
if (MBUF_NO_SEC_OFFLOAD(pkt) && sa->fallback_sessions > 0) {
                uintptr_t intsa = (uintptr_t)sa;
                intsa |= IPSEC_SA_OFFLOAD_FALLBACK_FLAG;
                result_sa = (void *)intsa;
 }

So from my perspective, no additional lookup where introduced.
Also AFAIK, right now there is no possibility to configure ipsec-secgw
to BYPASS some ESP traffic.
Should we do it (to conform to ipsec RFC) - that's probably subject of
another discussion.
 
> 
> 2. The solution proposed here seems like adding the handling in ipsec-secgw instead of ipsec library. In other words, this feature is not
> getting added in ipsec library, which was supposed to simplify the whole ipsec usage in DPDK, but fails to handle the case of fragmentation.

What we have right now with ipsec library is SA (low) level API.
It can handle multi-segment packets properly, but expects someone else to do other steps (fragmentation/reassembly).
ipsec-secgw demonstrates how librte_ip_frag and librte_ipsec can be used together to deal with fragmented IPsec traffic
in a proper manner.
Probably in future we'll consider adding some high-level API that will be able to do whole processing under hood
(SPD/SAD lookup, fragment/reassembly, actual IPsec packet processing, matching inbound selectors, etc.),
but right now it is not the case.  

> Also, since the fallback feature is entirely done in the application, it begs the question why the same feature is omitted for legacy use case.

Because legacy mode doesn't support multi-seg packets at first place.
Also it is an extra overhead to support 2 code-paths (legacy and library)
for the same app, so we'd like in future to deprecate and later remove legacy code-path.
As a first step we propose to make library code-path a default one:
http://patches.dpdk.org/cover/58247/

> 
> 3. It seems like ordering won't be maintained once this processing is done. Again, this is the sequence I understood. Please correct me if I
> missed something,
>        a. Application receives a bunch of packets (let's say 6 packets), in which few are fragmented (P3 & P4) and the rest can be inline
> processed.
>        b. Application receives P1->P2->P3->P4->P5->P6 (in this, P1, P2, P5, P6 are inline processed successfully) and P4 & P5 are the fragments
>        c. Application groups packets. P1->P2->P5->P6 becomes one group and P3->P4 becomes another and goes for fallback processing.
> Now how is ordering maintained? I couldn't figure out how that is done in this case.

You right, fallback session can introduce packet reordering.
At least till we'll have ability to process packets in sync mode too.
See our presentation at dpdk userspace (slides 17, 18):
https://static.sched.com/hosted_files/dpdkbordeaux2019/8f/DPDK-IPSECu9.pdf
Right now the only way to deal with it - have replay window big enough to sustain
reordering and async processing latency.
That's actually another reason why we add this feature into ipsec-secgw sample app:
so people can evaluate it on their platforms, determine what replay window size
would be needed, what issues/slowdowns it might cause, etc.

> 
> Thanks,
> Anoob
> 
> > -----Original Message-----
> > From: dev <dev-bounces@dpdk.org> On Behalf Of Marcin Smoczynski
> > Sent: Monday, September 23, 2019 5:14 PM
> > To: Anoob Joseph <anoobj@marvell.com>; akhil.goyal@nxp.com;
> > konstantin.ananyev@intel.com
> > Cc: dev@dpdk.org; Marcin Smoczynski <marcinx.smoczynski@intel.com>
> > Subject: [dpdk-dev] [PATCH v3 0/3] add fallback session
> >
> > Add fallback session feature allowing to process packets that inline processor
> > is unable to handle (e.g. fragmented traffic). Processing takes place in a
> > secondary session defined for SA in a configuration file.
> >
> > This feature is limited to ingress IPsec traffic only. IPsec anti-replay window
> > and ESN are supported in conjunction with fallback session when following
> > conditions are met:
> >  * primary session is 'inline-crypto-offload,
> >  * fallback sessions is 'lookaside-none'.
> >
> > v2 to v3 changes:
> >  - doc and commit log update - explicitly state feature limitations
> >
> > v1 to v2 changes:
> >  - disable fallback offload for outbound SAs
> >  - add test scripts
> >
> > Marcin Smoczynski (3):
> >   examples/ipsec-secgw: ipsec_sa structure cleanup
> >   examples/ipsec-secgw: add fallback session feature
> >   examples/ipsec-secgw: add offload fallback tests
> >
> >  doc/guides/sample_app_ug/ipsec_secgw.rst      |  20 ++-
> >  examples/ipsec-secgw/esp.c                    |  35 ++--
> >  examples/ipsec-secgw/ipsec-secgw.c            |  16 +-
> >  examples/ipsec-secgw/ipsec.c                  |  99 ++++++-----
> >  examples/ipsec-secgw/ipsec.h                  |  61 +++++--
> >  examples/ipsec-secgw/ipsec_process.c          | 113 +++++++-----
> >  examples/ipsec-secgw/sa.c                     | 164 +++++++++++++-----
> >  .../test/trs_aesgcm_common_defs.sh            |   4 +-
> >  .../trs_aesgcm_inline_crypto_fallback_defs.sh |   5 +
> >  .../test/tun_aesgcm_common_defs.sh            |   6 +-
> >  .../tun_aesgcm_inline_crypto_fallback_defs.sh |   5 +
> >  11 files changed, 361 insertions(+), 167 deletions(-)  create mode 100644
> > examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
> >  create mode 100644 examples/ipsec-
> > secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh
> >
> > --
> > 2.17.1

Re: [dpdk-dev] [PATCH v3 0/3] add fallback session

[Anoob Joseph]

Hi Konstantin,

Please see inline.

Thanks,
Anoob

> -----Original Message-----
> From: Ananyev, Konstantin <konstantin.ananyev@intel.com>
> Sent: Thursday, September 26, 2019 6:08 PM
> To: Anoob Joseph <anoobj@marvell.com>; Smoczynski, MarcinX
> <marcinx.smoczynski@intel.com>; akhil.goyal@nxp.com
> Cc: dev@dpdk.org; Jerin Jacob Kollanukkaran <jerinj@marvell.com>; Narayana
> Prasad Raju Athreya <pathreya@marvell.com>; Archana Muniganti
> <marchana@marvell.com>
> Subject: RE: [dpdk-dev] [PATCH v3 0/3] add fallback session
> 
> 
> Hi Anoob,
> Answers/comments inline.
> Marcin, please correct me, if I missed something.
> Konstantin
> 
> >
> > Hi Marcin, Konstantin,
> >
> > I've few more observations regarding the proposed feature.
> >
> > 1. From what I understood, if an ESP packet ends up on an unprotected
> > interface and doesn't have 'PKT_RX_SEC_OFFLOAD' bit set, then the packet
> would be looked up to see the associated SA and then fallback session is figured
> out and then further processing is done.
> >
> > Can you confirm if I understood the sequence correctly? If yes, then
> > aren't we doing an extra lookup in the s/w? The packet may be looked
> > by the h/w using rte_flow and that information could be used to determine the
> SA. Also, if the ESP packet is expected to be forwarded, then the above logic will
> add an unnecessary lookup even after your h/w has detected that the packet
> need not be security processed.
> 
> Not sure I understood your whole statement above.
> AFAIK, right now (with dpdk master) for unprotected iface it works like that:
> 
> 1.  slit incoming traffic into 3 groups: ESP packets, IPv4 packets, IPv6 packets.
> For ESP packets:
> 2. perform SAD lookup
>     a. if it fails (non SA found for that SPI), then drop the packet.
>     b. otherwise (SA found) process the packet using found SA
> 
> What fall-back patch adds:
> Before step 2.b check:
> does that SA has its primary session  with type INLINE-CRYPTO and
> does HW fail to do IPsec realted processing for it (by some reason)?
> If yes, then mark this packet to be processed by fall-back session.
> if (MBUF_NO_SEC_OFFLOAD(pkt) && sa->fallback_sessions > 0) {
>                 uintptr_t intsa = (uintptr_t)sa;
>                 intsa |= IPSEC_SA_OFFLOAD_FALLBACK_FLAG;
>                 result_sa = (void *)intsa;  }
> 
> So from my perspective, no additional lookup where introduced.

[Anoob] For inline processing, h/w does a lookup and figures out the security processing to be done on the packet. "rte_security_get_userdata()" allows s/w to further segregate that info. In this approach, I believe we don't consider how such info from h/w can be used.
 
> Also AFAIK, right now there is no possibility to configure ipsec-secgw to BYPASS
> some ESP traffic.
> Should we do it (to conform to ipsec RFC) - that's probably subject of another
> discussion.

[Anoob] The approach (choice of flags) forces a software-based SA lookup for packets that need to be bypassed instead of leveraging possible hardware SA lookup. I don't think what ipsec-secgw does matters here.

For example, ESN was not supported by ipsec-secgw (before library mode was introduced), but every single library change and application change was added keeping in mind that ESN is a valid feature for ipsec. So it is one thing to say ipsec-secgw doesn't support ESP bypass and saying the solution doesn't consider a possibility of ESP bypass. 
 
> 
> >
> > 2. The solution proposed here seems like adding the handling in
> > ipsec-secgw instead of ipsec library. In other words, this feature is not getting
> added in ipsec library, which was supposed to simplify the whole ipsec usage in
> DPDK, but fails to handle the case of fragmentation.
> 
> What we have right now with ipsec library is SA (low) level API.
> It can handle multi-segment packets properly, but expects someone else to do
> other steps (fragmentation/reassembly).
> ipsec-secgw demonstrates how librte_ip_frag and librte_ipsec can be used
> together to deal with fragmented IPsec traffic in a proper manner.
> Probably in future we'll consider adding some high-level API that will be able to
> do whole processing under hood (SPD/SAD lookup, fragment/reassembly, actual
> IPsec packet processing, matching inbound selectors, etc.), but right now it is
> not the case.
> 
> > Also, since the fallback feature is entirely done in the application, it begs the
> question why the same feature is omitted for legacy use case.
> 
> Because legacy mode doesn't support multi-seg packets at first place.
> Also it is an extra overhead to support 2 code-paths (legacy and library) for the
> same app, so we'd like in future to deprecate and later remove legacy code-
> path.
> As a first step we propose to make library code-path a default one:
> http://patches.dpdk.org/cover/58247/

[Anoob] Even if we need the application to do the reassembly, it would look simple if application uses the same "rte_ipsec_session" to process the fallback packet. In that case, the processing required to handle the other packet will be hidden from the application. Here application has to make sure it chooses the correct session, which I believe should be moved to ipsec library.

> 
> >
> > 3. It seems like ordering won't be maintained once this processing is
> > done. Again, this is the sequence I understood. Please correct me if I missed
> something,
> >        a. Application receives a bunch of packets (let's say 6
> > packets), in which few are fragmented (P3 & P4) and the rest can be inline
> processed.
> >        b. Application receives P1->P2->P3->P4->P5->P6 (in this, P1, P2, P5, P6 are
> inline processed successfully) and P4 & P5 are the fragments
> >        c. Application groups packets. P1->P2->P5->P6 becomes one group and P3-
> >P4 becomes another and goes for fallback processing.
> > Now how is ordering maintained? I couldn't figure out how that is done in this
> case.
> 
> You right, fallback session can introduce packet reordering.
> At least till we'll have ability to process packets in sync mode too.
> See our presentation at dpdk userspace (slides 17, 18):
> https://static.sched.com/hosted_files/dpdkbordeaux2019/8f/DPDK-IPSECu9.pdf
> Right now the only way to deal with it - have replay window big enough to
> sustain reordering and async processing latency.
> That's actually another reason why we add this feature into ipsec-secgw sample
> app:
> so people can evaluate it on their platforms, determine what replay window size
> would be needed, what issues/slowdowns it might cause, etc.

[Anoob] This is something I had noticed while going through the code flow. The ordering info is lost because of the approach. As we dig deeper, the feature looks hardly complete. The concerning part is the approach doesn't seem conducive to fixing any of these issues in the future.

Also, if the new ipsec related features are introduced via ipsec-secgw than via rte_ipsec, then it raises doubts over the utility of rte_ipsec library.
 
> 
> >
> > Thanks,
> > Anoob
> >
> > > -----Original Message-----
> > > From: dev <dev-bounces@dpdk.org> On Behalf Of Marcin Smoczynski
> > > Sent: Monday, September 23, 2019 5:14 PM
> > > To: Anoob Joseph <anoobj@marvell.com>; akhil.goyal@nxp.com;
> > > konstantin.ananyev@intel.com
> > > Cc: dev@dpdk.org; Marcin Smoczynski <marcinx.smoczynski@intel.com>
> > > Subject: [dpdk-dev] [PATCH v3 0/3] add fallback session
> > >
> > > Add fallback session feature allowing to process packets that inline
> > > processor is unable to handle (e.g. fragmented traffic). Processing
> > > takes place in a secondary session defined for SA in a configuration file.
> > >
> > > This feature is limited to ingress IPsec traffic only. IPsec
> > > anti-replay window and ESN are supported in conjunction with
> > > fallback session when following conditions are met:
> > >  * primary session is 'inline-crypto-offload,
> > >  * fallback sessions is 'lookaside-none'.
> > >
> > > v2 to v3 changes:
> > >  - doc and commit log update - explicitly state feature limitations
> > >
> > > v1 to v2 changes:
> > >  - disable fallback offload for outbound SAs
> > >  - add test scripts
> > >
> > > Marcin Smoczynski (3):
> > >   examples/ipsec-secgw: ipsec_sa structure cleanup
> > >   examples/ipsec-secgw: add fallback session feature
> > >   examples/ipsec-secgw: add offload fallback tests
> > >
> > >  doc/guides/sample_app_ug/ipsec_secgw.rst      |  20 ++-
> > >  examples/ipsec-secgw/esp.c                    |  35 ++--
> > >  examples/ipsec-secgw/ipsec-secgw.c            |  16 +-
> > >  examples/ipsec-secgw/ipsec.c                  |  99 ++++++-----
> > >  examples/ipsec-secgw/ipsec.h                  |  61 +++++--
> > >  examples/ipsec-secgw/ipsec_process.c          | 113 +++++++-----
> > >  examples/ipsec-secgw/sa.c                     | 164 +++++++++++++-----
> > >  .../test/trs_aesgcm_common_defs.sh            |   4 +-
> > >  .../trs_aesgcm_inline_crypto_fallback_defs.sh |   5 +
> > >  .../test/tun_aesgcm_common_defs.sh            |   6 +-
> > >  .../tun_aesgcm_inline_crypto_fallback_defs.sh |   5 +
> > >  11 files changed, 361 insertions(+), 167 deletions(-)  create mode
> > > 100644
> > > examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
> > >  create mode 100644 examples/ipsec-
> > > secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh
> > >
> > > --
> > > 2.17.1

Re: [dpdk-dev] [PATCH v3 0/3] add fallback session

[Ananyev, Konstantin]

Hi Anoob,

> > > I've few more observations regarding the proposed feature.
> > >
> > > 1. From what I understood, if an ESP packet ends up on an unprotected
> > > interface and doesn't have 'PKT_RX_SEC_OFFLOAD' bit set, then the packet
> > would be looked up to see the associated SA and then fallback session is figured
> > out and then further processing is done.
> > >
> > > Can you confirm if I understood the sequence correctly? If yes, then
> > > aren't we doing an extra lookup in the s/w? The packet may be looked
> > > by the h/w using rte_flow and that information could be used to determine the
> > SA. Also, if the ESP packet is expected to be forwarded, then the above logic will
> > add an unnecessary lookup even after your h/w has detected that the packet
> > need not be security processed.
> >
> > Not sure I understood your whole statement above.
> > AFAIK, right now (with dpdk master) for unprotected iface it works like that:
> >
> > 1.  slit incoming traffic into 3 groups: ESP packets, IPv4 packets, IPv6 packets.
> > For ESP packets:
> > 2. perform SAD lookup
> >     a. if it fails (non SA found for that SPI), then drop the packet.
> >     b. otherwise (SA found) process the packet using found SA
> >
> > What fall-back patch adds:
> > Before step 2.b check:
> > does that SA has its primary session  with type INLINE-CRYPTO and
> > does HW fail to do IPsec realted processing for it (by some reason)?
> > If yes, then mark this packet to be processed by fall-back session.
> > if (MBUF_NO_SEC_OFFLOAD(pkt) && sa->fallback_sessions > 0) {
> >                 uintptr_t intsa = (uintptr_t)sa;
> >                 intsa |= IPSEC_SA_OFFLOAD_FALLBACK_FLAG;
> >                 result_sa = (void *)intsa;  }
> >
> > So from my perspective, no additional lookup where introduced.
> 
> [Anoob] For inline processing, h/w does a lookup and figures out the security processing to be done on the packet.
> "rte_security_get_userdata()" allows s/w to further segregate that info. In this approach, I believe we don't consider how such info from
> h/w can be used.

Right now it is not the case with ipsec-secgw:
for each inbound ESP packet it *always* does a lookup in SW based SADB,
and if lookup fails - drops the packet (see 2.a above).
So, I still not see any additional lookups introduced with the patch.

> 
> > Also AFAIK, right now there is no possibility to configure ipsec-secgw to BYPASS
> > some ESP traffic.
> > Should we do it (to conform to ipsec RFC) - that's probably subject of another
> > discussion.
> 
> [Anoob] The approach (choice of flags) forces a software-based SA lookup for packets that need to be bypassed instead of leveraging
> possible hardware SA lookup. I don't think what ipsec-secgw does matters here.

I do not agree with that statement.
First of all - current patch doesn't introduce any additional SW lookups, see above.
Second, if someone would like to add BYPASS for ESP packets into ipsec-secgw, I think he would have to
insert new code that do de-mux *before* any ESP related processing starts.
Most obvious variant: at prepare_one_packet() when we split our incoming traffic to IPsec and non-IPsec ones.
Second variant - at early stage of single_inbound_lookup(), straight after actual SAD lookup failure.
In both cases, I don't see how session selection (primary or faillback) would affect us here,
actual decision do we want to PROCESS or BYPASS particular ESP packet needs to take place before that.
So I still believe we are ok here.

> 
> For example, ESN was not supported by ipsec-secgw (before library mode was introduced), but every single library change and application
> change was added keeping in mind that ESN is a valid feature for ipsec. So it is one thing to say ipsec-secgw doesn't support ESP bypass and
> saying the solution doesn't consider a possibility of ESP bypass.
> 
> >
> > >
> > > 2. The solution proposed here seems like adding the handling in
> > > ipsec-secgw instead of ipsec library. In other words, this feature is not getting
> > added in ipsec library, which was supposed to simplify the whole ipsec usage in
> > DPDK, but fails to handle the case of fragmentation.
> >
> > What we have right now with ipsec library is SA (low) level API.
> > It can handle multi-segment packets properly, but expects someone else to do
> > other steps (fragmentation/reassembly).
> > ipsec-secgw demonstrates how librte_ip_frag and librte_ipsec can be used
> > together to deal with fragmented IPsec traffic in a proper manner.
> > Probably in future we'll consider adding some high-level API that will be able to
> > do whole processing under hood (SPD/SAD lookup, fragment/reassembly, actual
> > IPsec packet processing, matching inbound selectors, etc.), but right now it is
> > not the case.
> >
> > > Also, since the fallback feature is entirely done in the application, it begs the
> > question why the same feature is omitted for legacy use case.
> >
> > Because legacy mode doesn't support multi-seg packets at first place.
> > Also it is an extra overhead to support 2 code-paths (legacy and library) for the
> > same app, so we'd like in future to deprecate and later remove legacy code-
> > path.
> > As a first step we propose to make library code-path a default one:
> > http://patches.dpdk.org/cover/58247/
> 
> [Anoob] Even if we need the application to do the reassembly, it would look simple if application uses the same "rte_ipsec_session" to
> process the fallback packet.

I think there is some sort of misunderstanding here.
With current librte_ipsec design:
rte_ipsec_sa - opaque SW representation of the SA (HW neutral)
 rte_ipsec_session associates SA (rte_ipsec_sa)with particular HW device (session).
Same SA can be referred by multiple sessions.

From rte_ipsec.h:
/**
 * rte_ipsec_session is an aggregate structure that defines particular
 * IPsec Security Association IPsec (SA) on given security/crypto device:
 * - pointer to the SA object
 * - security session action type
 * - pointer to security/crypto session, plus other related data
 * - session/device specific functions to prepare/process IPsec packets.
 */

> In that case, the processing required to handle the other packet will be hidden from the application. Here
> application has to make sure it chooses the correct session, which I believe should be moved to ipsec library.

From my side it is a good thing to let application define its own usage model.
I.E. librte_ipsec supports multiple-sessions per SA, upper-layer (app) decides how it
will use that feature.
Though if you strongly feel that some higher-level API is needed here,
and have some good ideas about it  - please go ahead and submit RFC for it. 

> 
> >
> > >
> > > 3. It seems like ordering won't be maintained once this processing is
> > > done. Again, this is the sequence I understood. Please correct me if I missed
> > something,
> > >        a. Application receives a bunch of packets (let's say 6
> > > packets), in which few are fragmented (P3 & P4) and the rest can be inline
> > processed.
> > >        b. Application receives P1->P2->P3->P4->P5->P6 (in this, P1, P2, P5, P6 are
> > inline processed successfully) and P4 & P5 are the fragments
> > >        c. Application groups packets. P1->P2->P5->P6 becomes one group and P3-
> > >P4 becomes another and goes for fallback processing.
> > > Now how is ordering maintained? I couldn't figure out how that is done in this
> > case.
> >
> > You right, fallback session can introduce packet reordering.
> > At least till we'll have ability to process packets in sync mode too.
> > See our presentation at dpdk userspace (slides 17, 18):
> > https://static.sched.com/hosted_files/dpdkbordeaux2019/8f/DPDK-IPSECu9.pdf
> > Right now the only way to deal with it - have replay window big enough to
> > sustain reordering and async processing latency.
> > That's actually another reason why we add this feature into ipsec-secgw sample
> > app:
> > so people can evaluate it on their platforms, determine what replay window size
> > would be needed, what issues/slowdowns it might cause, etc.
> 
> [Anoob] This is something I had noticed while going through the code flow. The ordering info is lost because of the approach. 

Once again, it is a known limitation and we are not trying to hide it from you :)
It was outlined here:
https://static.sched.com/hosted_files/dpdkbordeaux2019/8f/DPDK-IPSECu9.pdf
And in the latest patch version Marcin clearly stated it inside the AG session:
http://patches.dpdk.org/patch/60039/
If you think even further clarification within the doc is needed, please let us know.

About the actual packet re-oridering:
 AFAIK, for some use-cases it might be acceptable, for others not.
Though it is an optional feature, that wouldn't be enabled by default, so user can always
choose is it does he needs/wants this feature or not.
If/when we'll have CPU_CRYPTO processing mode:
http://patches.dpdk.org/cover/58862/
we'll add ability for the user to avoid this reordering problem 

> As we dig deeper, the feature looks hardly complete. The concerning part is the approach doesn't seem conducive to fixing any of these issues in the
> future.

Again, don't agree with that statement.
From my perspective: while current implementation has its limitations (packet reordering, inline-proto/lksd-proto),
it is totally functional and provides users with a reference how to deal with such kind of problems. 

> 
> Also, if the new ipsec related features are introduced via ipsec-secgw than via rte_ipsec, then it raises doubts over the utility of rte_ipsec
> library.

Re: [dpdk-dev] [PATCH v3 0/3] add fallback session

[Anoob Joseph]

Hi Konstantin,

Please see inline.

Thanks,
Anoob

> -----Original Message-----
> From: Ananyev, Konstantin <konstantin.ananyev@intel.com>
> Sent: Monday, September 30, 2019 7:01 PM
> To: Anoob Joseph <anoobj@marvell.com>; Smoczynski, MarcinX
> <marcinx.smoczynski@intel.com>; akhil.goyal@nxp.com
> Cc: dev@dpdk.org; Jerin Jacob Kollanukkaran <jerinj@marvell.com>; Narayana
> Prasad Raju Athreya <pathreya@marvell.com>; Archana Muniganti
> <marchana@marvell.com>
> Subject: RE: [dpdk-dev] [PATCH v3 0/3] add fallback session
> 
> 
> Hi Anoob,
> 
> > > > I've few more observations regarding the proposed feature.
> > > >
> > > > 1. From what I understood, if an ESP packet ends up on an
> > > > unprotected interface and doesn't have 'PKT_RX_SEC_OFFLOAD' bit
> > > > set, then the packet
> > > would be looked up to see the associated SA and then fallback
> > > session is figured out and then further processing is done.
> > > >
> > > > Can you confirm if I understood the sequence correctly? If yes,
> > > > then aren't we doing an extra lookup in the s/w? The packet may be
> > > > looked by the h/w using rte_flow and that information could be
> > > > used to determine the
> > > SA. Also, if the ESP packet is expected to be forwarded, then the
> > > above logic will add an unnecessary lookup even after your h/w has
> > > detected that the packet need not be security processed.
> > >
> > > Not sure I understood your whole statement above.
> > > AFAIK, right now (with dpdk master) for unprotected iface it works like that:
> > >
> > > 1.  slit incoming traffic into 3 groups: ESP packets, IPv4 packets, IPv6
> packets.
> > > For ESP packets:
> > > 2. perform SAD lookup
> > >     a. if it fails (non SA found for that SPI), then drop the packet.
> > >     b. otherwise (SA found) process the packet using found SA
> > >
> > > What fall-back patch adds:
> > > Before step 2.b check:
> > > does that SA has its primary session  with type INLINE-CRYPTO and
> > > does HW fail to do IPsec realted processing for it (by some reason)?
> > > If yes, then mark this packet to be processed by fall-back session.
> > > if (MBUF_NO_SEC_OFFLOAD(pkt) && sa->fallback_sessions > 0) {
> > >                 uintptr_t intsa = (uintptr_t)sa;
> > >                 intsa |= IPSEC_SA_OFFLOAD_FALLBACK_FLAG;
> > >                 result_sa = (void *)intsa;  }
> > >
> > > So from my perspective, no additional lookup where introduced.
> >
> > [Anoob] For inline processing, h/w does a lookup and figures out the security
> processing to be done on the packet.
> > "rte_security_get_userdata()" allows s/w to further segregate that
> > info. In this approach, I believe we don't consider how such info from h/w can
> be used.
> 
> Right now it is not the case with ipsec-secgw:

[Anoob] Are you saying there is no h/w lookup involved when doing inline crypto processing? Then I'm confused. My understanding is that rte_flow with ACTION type as SECURITY is being used to setup h/w lookups.

> for each inbound ESP packet it *always* does a lookup in SW based SADB, and if
> lookup fails - drops the packet (see 2.a above).
> So, I still not see any additional lookups introduced with the patch.

[Anoob] In case of inline crypto, you could do that. No disagreement. But that doesn't mean that is the only way. If PMDs can retrieve info about h/w lookups and pass it on to the upper layers, isn't that the better approach? 
 
> 
> >
> > > Also AFAIK, right now there is no possibility to configure
> > > ipsec-secgw to BYPASS some ESP traffic.
> > > Should we do it (to conform to ipsec RFC) - that's probably subject
> > > of another discussion.
> >
> > [Anoob] The approach (choice of flags) forces a software-based SA
> > lookup for packets that need to be bypassed instead of leveraging possible
> hardware SA lookup. I don't think what ipsec-secgw does matters here.
> 
> I do not agree with that statement.
> First of all - current patch doesn't introduce any additional SW lookups, see
> above.
> Second, if someone would like to add BYPASS for ESP packets into ipsec-secgw, I
> think he would have to insert new code that do de-mux *before* any ESP
> related processing starts.
> Most obvious variant: at prepare_one_packet() when we split our incoming
> traffic to IPsec and non-IPsec ones.
> Second variant - at early stage of single_inbound_lookup(), straight after actual
> SAD lookup failure.
> In both cases, I don't see how session selection (primary or faillback) would
> affect us here, actual decision do we want to PROCESS or BYPASS particular ESP
> packet needs to take place before that.
> So I still believe we are ok here.

[Anoob] I don't think you understood what I'm trying to highlight here. You could have a situation when h/w can detect that the packet has to be "PROTECT"ed using an "inline crypto" session. But then it might detect that it cannot proceed with inline processing because of some issue (fragmentation, h/w queue overflow etc). Now the h/w has already figured out the action to be done to the packet. If PMDs allow this to be communicated to the application, the application won't be required to do the lookup.

In inline crypto, application can ignore the h/w lookup data and do a s/w lookup with the protocol headers as they are not stripped off. It was done this way, because API and the associated means to get this info from PMD was not introduced when inline crypto and corresponding support in Intel's PMD was added. But inline protocol cannot do the lookup in the application and so the concept of userdata etc was introduced. The same can be applied to inline crypto also. Advantage? It could remove the extra lookup done in s/w.

To summarize, we cannot assume that every inline crypto packet need to looked up in the application to figure out the processing done on the packet. There can be applications which relies on h/w lookup data to figure out the processing done. 

Example: Using rte_flow, I can create a rule for ESP packets to be MARKed. There is no security session created for the flow and the application intends to forward this flow packets to a different port. With your approach, even these packets would be looked up. The packet will have info from the h/w lookup which doesn't get used, because the approach fails to introduce the required concepts. 

Does the above sound like BYPASS? Yes.
Does ipsec-secgw do it this way? No.
Does this approach prevent an application from introducing such a BYPASS? Yes.

> 
> >
> > For example, ESN was not supported by ipsec-secgw (before library mode
> > was introduced), but every single library change and application
> > change was added keeping in mind that ESN is a valid feature for ipsec. So it is
> one thing to say ipsec-secgw doesn't support ESP bypass and saying the solution
> doesn't consider a possibility of ESP bypass.
> >
> > >
> > > >
> > > > 2. The solution proposed here seems like adding the handling in
> > > > ipsec-secgw instead of ipsec library. In other words, this feature
> > > > is not getting
> > > added in ipsec library, which was supposed to simplify the whole
> > > ipsec usage in DPDK, but fails to handle the case of fragmentation.
> > >
> > > What we have right now with ipsec library is SA (low) level API.
> > > It can handle multi-segment packets properly, but expects someone
> > > else to do other steps (fragmentation/reassembly).
> > > ipsec-secgw demonstrates how librte_ip_frag and librte_ipsec can be
> > > used together to deal with fragmented IPsec traffic in a proper manner.
> > > Probably in future we'll consider adding some high-level API that
> > > will be able to do whole processing under hood (SPD/SAD lookup,
> > > fragment/reassembly, actual IPsec packet processing, matching
> > > inbound selectors, etc.), but right now it is not the case.
> > >
> > > > Also, since the fallback feature is entirely done in the
> > > > application, it begs the
> > > question why the same feature is omitted for legacy use case.
> > >
> > > Because legacy mode doesn't support multi-seg packets at first place.
> > > Also it is an extra overhead to support 2 code-paths (legacy and
> > > library) for the same app, so we'd like in future to deprecate and
> > > later remove legacy code- path.
> > > As a first step we propose to make library code-path a default one:
> > > https://urldefense.proofpoint.com/v2/url?u=http-3A__patches.dpdk.org
> > >
> _cover_58247_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=jPfB8rwwviRSxyL
> Ws2
> > > n6B-WYLn1v9SyTMrT5EQqh2TU&m=b3E429fuo8P-K5XfH8wG-
> 7hwr1d8oM4uJGErAkbf
> > > DvA&s=vz7ioUzJOuzoJdmV7QO0QLPKYn1ytFsj_0eYatbSCrg&e=
> >
> > [Anoob] Even if we need the application to do the reassembly, it would
> > look simple if application uses the same "rte_ipsec_session" to process the
> fallback packet.
> 
> I think there is some sort of misunderstanding here.
> With current librte_ipsec design:
> rte_ipsec_sa - opaque SW representation of the SA (HW neutral)
> rte_ipsec_session associates SA (rte_ipsec_sa)with particular HW device
> (session).
> Same SA can be referred by multiple sessions.
> 
> From rte_ipsec.h:
> /**
>  * rte_ipsec_session is an aggregate structure that defines particular
>  * IPsec Security Association IPsec (SA) on given security/crypto device:
>  * - pointer to the SA object
>  * - security session action type
>  * - pointer to security/crypto session, plus other related data
>  * - session/device specific functions to prepare/process IPsec packets.
>  */
> 
> > In that case, the processing required to handle the other packet will
> > be hidden from the application. Here application has to make sure it chooses
> the correct session, which I believe should be moved to ipsec library.
> 
> From my side it is a good thing to let application define its own usage model.
> I.E. librte_ipsec supports multiple-sessions per SA, upper-layer (app) decides how
> it will use that feature.
> Though if you strongly feel that some higher-level API is needed here, and have
> some good ideas about it  - please go ahead and submit RFC for it.

[Anoob] If you are okay with application defining how it implements ipsec, then probably there is no use case for lib ipsec?

> 
> >
> > >
> > > >
> > > > 3. It seems like ordering won't be maintained once this processing
> > > > is done. Again, this is the sequence I understood. Please correct
> > > > me if I missed
> > > something,
> > > >        a. Application receives a bunch of packets (let's say 6
> > > > packets), in which few are fragmented (P3 & P4) and the rest can
> > > > be inline
> > > processed.
> > > >        b. Application receives P1->P2->P3->P4->P5->P6 (in this,
> > > > P1, P2, P5, P6 are
> > > inline processed successfully) and P4 & P5 are the fragments
> > > >        c. Application groups packets. P1->P2->P5->P6 becomes one
> > > >group and P3-
> > > >P4 becomes another and goes for fallback processing.
> > > > Now how is ordering maintained? I couldn't figure out how that is
> > > >done in this
> > > case.
> > >
> > > You right, fallback session can introduce packet reordering.
> > > At least till we'll have ability to process packets in sync mode too.
> > > See our presentation at dpdk userspace (slides 17, 18):
> > > https://urldefense.proofpoint.com/v2/url?u=https-3A__static.sched.co
> > > m_hosted-5Ffiles_dpdkbordeaux2019_8f_DPDK-
> 2DIPSECu9.pdf&d=DwIFAg&c=n
> > > KjWec2b6R0mOyPaz7xtfQ&r=jPfB8rwwviRSxyLWs2n6B-
> WYLn1v9SyTMrT5EQqh2TU&
> > > m=b3E429fuo8P-K5XfH8wG-
> 7hwr1d8oM4uJGErAkbfDvA&s=MLRnnYcykjnsqXrHGUuR
> > > YHc5uDxaAod0Yl7f06EQr1M&e= Right now the only way to deal with it -
> > > have replay window big enough to sustain reordering and async
> > > processing latency.
> > > That's actually another reason why we add this feature into
> > > ipsec-secgw sample
> > > app:
> > > so people can evaluate it on their platforms, determine what replay
> > > window size would be needed, what issues/slowdowns it might cause, etc.
> >
> > [Anoob] This is something I had noticed while going through the code flow.
> The ordering info is lost because of the approach.
> 
> Once again, it is a known limitation and we are not trying to hide it from you :)

[Anoob] Never said it was hidden. Initially I had okayed the approach even though it wasn't a solution suitable for inline protocol. But as more cases were considered, few shortcomings were observed, and I was skeptical about how any of that would be solved in the long run.

My suggestion, if these limitations and the plans to address it were mentioned in the cover letter, we could've saved few cycles here. But my reply would still be the same 😊

> It
> was outlined here:
> https://urldefense.proofpoint.com/v2/url?u=https-
> 3A__static.sched.com_hosted-5Ffiles_dpdkbordeaux2019_8f_DPDK-
> 2DIPSECu9.pdf&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=jPfB8rwwviRSxyL
> Ws2n6B-WYLn1v9SyTMrT5EQqh2TU&m=b3E429fuo8P-K5XfH8wG-
> 7hwr1d8oM4uJGErAkbfDvA&s=MLRnnYcykjnsqXrHGUuRYHc5uDxaAod0Yl7f06E
> Qr1M&e=
> And in the latest patch version Marcin clearly stated it inside the AG session:
> https://urldefense.proofpoint.com/v2/url?u=http-
> 3A__patches.dpdk.org_patch_60039_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtf
> Q&r=jPfB8rwwviRSxyLWs2n6B-WYLn1v9SyTMrT5EQqh2TU&m=b3E429fuo8P-
> K5XfH8wG-
> 7hwr1d8oM4uJGErAkbfDvA&s=zrn91Vjf_ZElAlDf7IeZGmGevcA6RMwpPTLUCGlgZ
> cY&e=
> If you think even further clarification within the doc is needed, please let us
> know.
> 
> About the actual packet re-oridering:
>  AFAIK, for some use-cases it might be acceptable, for others not.
> Though it is an optional feature, that wouldn't be enabled by default, so user can
> always choose is it does he needs/wants this feature or not.
> If/when we'll have CPU_CRYPTO processing mode:

[Anoob] Shouldn't we get that accepted first then?

Also, the ordering is lost because application is not considering that. Why can't we use re-ordering library? Or an event-dev?
 
> https://urldefense.proofpoint.com/v2/url?u=http-
> 3A__patches.dpdk.org_cover_58862_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtf
> Q&r=jPfB8rwwviRSxyLWs2n6B-WYLn1v9SyTMrT5EQqh2TU&m=b3E429fuo8P-
> K5XfH8wG-
> 7hwr1d8oM4uJGErAkbfDvA&s=k1zef_1uqELXblKN3_qjX04WXNFGAUEFIIJrvKPr6
> eA&e=
> we'll add ability for the user to avoid this reordering problem
> 
> > As we dig deeper, the feature looks hardly complete. The concerning
> > part is the approach doesn't seem conducive to fixing any of these issues in the
> future.
> 
> Again, don't agree with that statement.
> From my perspective: while current implementation has its limitations (packet
> reordering, inline-proto/lksd-proto), it is totally functional and provides users
> with a reference how to deal with such kind of problems.

[Anoob] Well, that is exactly my problem. It gives a skewed reference on how to deal with the problem, without considering other possibilities. If every application starts adopting this method and starts doing lookup for every ESP packet, h/w based lookups (and the associated h/w) would be rendered useless. I really think this series, which happens to be the first step in solving a complex problem, will be very useful if all these issues are addressed properly.

If you can introduce the inline+lookaside via librte_ipsec, its perhaps ok for the patch to have some limitations(like lack of ordering, assumptions on ESP etc). However, if the patch is introduced directly into ipsec-secgw, it needs to be more comprehensive and robust. I also expect performance impact due to these changes and so I prefer if the inline+lookaside can be made separate datapath rather than overloading the existing one.

> 
> >
> > Also, if the new ipsec related features are introduced via ipsec-secgw
> > than via rte_ipsec, then it raises doubts over the utility of rte_ipsec library.
> 
> 

Re: [dpdk-dev] [PATCH v3 0/3] add fallback session

[Ananyev, Konstantin]

Hi Anoob,

> > > > > I've few more observations regarding the proposed feature.
> > > > >
> > > > > 1. From what I understood, if an ESP packet ends up on an
> > > > > unprotected interface and doesn't have 'PKT_RX_SEC_OFFLOAD' bit
> > > > > set, then the packet
> > > > would be looked up to see the associated SA and then fallback
> > > > session is figured out and then further processing is done.
> > > > >
> > > > > Can you confirm if I understood the sequence correctly? If yes,
> > > > > then aren't we doing an extra lookup in the s/w? The packet may be
> > > > > looked by the h/w using rte_flow and that information could be
> > > > > used to determine the
> > > > SA. Also, if the ESP packet is expected to be forwarded, then the
> > > > above logic will add an unnecessary lookup even after your h/w has
> > > > detected that the packet need not be security processed.
> > > >
> > > > Not sure I understood your whole statement above.
> > > > AFAIK, right now (with dpdk master) for unprotected iface it works like that:
> > > >
> > > > 1.  slit incoming traffic into 3 groups: ESP packets, IPv4 packets, IPv6
> > packets.
> > > > For ESP packets:
> > > > 2. perform SAD lookup
> > > >     a. if it fails (non SA found for that SPI), then drop the packet.
> > > >     b. otherwise (SA found) process the packet using found SA
> > > >
> > > > What fall-back patch adds:
> > > > Before step 2.b check:
> > > > does that SA has its primary session  with type INLINE-CRYPTO and
> > > > does HW fail to do IPsec realted processing for it (by some reason)?
> > > > If yes, then mark this packet to be processed by fall-back session.
> > > > if (MBUF_NO_SEC_OFFLOAD(pkt) && sa->fallback_sessions > 0) {
> > > >                 uintptr_t intsa = (uintptr_t)sa;
> > > >                 intsa |= IPSEC_SA_OFFLOAD_FALLBACK_FLAG;
> > > >                 result_sa = (void *)intsa;  }
> > > >
> > > > So from my perspective, no additional lookup where introduced.
> > >
> > > [Anoob] For inline processing, h/w does a lookup and figures out the security
> > processing to be done on the packet.
> > > "rte_security_get_userdata()" allows s/w to further segregate that
> > > info. In this approach, I believe we don't consider how such info from h/w can
> > be used.
> >
> > Right now it is not the case with ipsec-secgw:
> 
> [Anoob] Are you saying there is no h/w lookup involved when doing inline crypto processing? Then I'm confused. My understanding is that
> rte_flow with ACTION type as SECURITY is being used to setup h/w lookups.

I am saying that current ipsec-secgw code for each inbound ESP packet *always* does SW lookup.
No matter did HW lookup take place already and does information about that lookup is available
via mbuf in some way or not.
Just look at the ipsec-secgw code yourself:

nb_rx = rte_eth_rx_burst(portid, queueid,  pkts, MAX_PKT_BURST);

if (nb_rx > 0)
	process_pkts(qconf, pkts, nb_rx, portid);

Inside process_pkts()
http://lxr.dpdk.org/dpdk/latest/ident/prepare_one_packet
 it first calls prepare_traffic() which does sort of de-muxing:
put packet in one of 3 arrays: 
   1) ESP packets
   2) non ESP IPv4 packets
   3) non ESP IPv6 packets
Also it checks is PKT_RX_SEC_OFFLOAD set for that packet.
If yes, it retrieves associated security user-data and stores it inside private mbuf area.

Then it goes into process_pkts_inbound()
http://lxr.dpdk.org/dpdk/latest/ident/process_pkts_inbound
and here for all ESP packets:
legacy code path:
ipsec_inbound()->inbound_sa_lookup()->single_inbound_lookup()
for librte_ipsec code path:
inbound_sa_lookup()->single_inbound_lookup()

single_inbound_lookup() is the one that does actual SW lookup for each input packet.
http://lxr.dpdk.org/dpdk/latest/ident/single_inbound_lookup

> 
> > for each inbound ESP packet it *always* does a lookup in SW based SADB, and if
> > lookup fails - drops the packet (see 2.a above).
> > So, I still not see any additional lookups introduced with the patch.
> 
> [Anoob] In case of inline crypto, you could do that. No disagreement. But that doesn't mean that is the only way. If PMDs can retrieve info
> about h/w lookups and pass it on to the upper layers, isn't that the better approach?

Please let's keep our conversation in a constructive way.
We are not discussing what could be done in theory, but a particular patch for ipsec-secgw:
Right now ipsec-secgw does a SW lookup for each inbound ESP packet
(no matter what HW offload does) and this patch doesn't introduce
any extra SW lookups.

If you are not happy with current ipsec-secgw approach, that's fine -
feel free to submit RFC/patch to fix that, or start a discussion in a new thread.
I just don't see why we have to discuss it in context of this patch.  

> 
> >
> > >
> > > > Also AFAIK, right now there is no possibility to configure
> > > > ipsec-secgw to BYPASS some ESP traffic.
> > > > Should we do it (to conform to ipsec RFC) - that's probably subject
> > > > of another discussion.
> > >
> > > [Anoob] The approach (choice of flags) forces a software-based SA
> > > lookup for packets that need to be bypassed instead of leveraging possible
> > hardware SA lookup. I don't think what ipsec-secgw does matters here.
> >
> > I do not agree with that statement.
> > First of all - current patch doesn't introduce any additional SW lookups, see
> > above.
> > Second, if someone would like to add BYPASS for ESP packets into ipsec-secgw, I
> > think he would have to insert new code that do de-mux *before* any ESP
> > related processing starts.
> > Most obvious variant: at prepare_one_packet() when we split our incoming
> > traffic to IPsec and non-IPsec ones.
> > Second variant - at early stage of single_inbound_lookup(), straight after actual
> > SAD lookup failure.
> > In both cases, I don't see how session selection (primary or faillback) would
> > affect us here, actual decision do we want to PROCESS or BYPASS particular ESP
> > packet needs to take place before that.
> > So I still believe we are ok here.
> 
> [Anoob] I don't think you understood what I'm trying to highlight here. You could have a situation when h/w can detect that the packet has
> to be "PROTECT"ed using an "inline crypto" session. But then it might detect that it cannot proceed with inline processing because of some
> issue (fragmentation, h/w queue overflow etc). Now the h/w has already figured out the action to be done to the packet. If PMDs allow this
> to be communicated to the application, the application won't be required to do the lookup.
> 
> In inline crypto, application can ignore the h/w lookup data and do a s/w lookup with the protocol headers as they are not stripped off. It
> was done this way, because API and the associated means to get this info from PMD was not introduced when inline crypto and
> corresponding support in Intel's PMD was added. But inline protocol cannot do the lookup in the application and so the concept of userdata
> etc was introduced. The same can be applied to inline crypto also. Advantage? It could remove the extra lookup done in s/w.
> 
> To summarize, we cannot assume that every inline crypto packet need to looked up in the application to figure out the processing done on
> the packet. There can be applications which relies on h/w lookup data to figure out the processing done.
> 
> Example: Using rte_flow, I can create a rule for ESP packets to be MARKed. There is no security session created for the flow and the
> application intends to forward this flow packets to a different port. With your approach, even these packets would be looked up. The
> packet will have info from the h/w lookup which doesn't get used, because the approach fails to introduce the required concepts.

I think I understood your point.
Basically you saying that in future your PMD for unprocessed ESP packets might provide some additional information
(via rte_security_get_userdata) that might be used by SW - let say to choose BYPASS for such packets.
Current ipsec-secgw doesn't support such ability.
Your concern is that this patch will somehow prevent (or will make it harder) for you to make your future changes.
Correct?
If, so then I said before, I don't think that concern is valid.
Most obvious variant where to add such code is inisde
inside prepare_one_packet() when we split our incoming traffic to IPsec and non-IPsec ones.
To be more specific something like that

if (eth->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) {

                iph4 = (const struct rte_ipv4_hdr *)rte_pktmbuf_adj(pkt,
                        RTE_ETHER_HDR_LEN);
                adjust_ipv4_pktlen(pkt, iph4, 0);

-                if (iph4->next_proto_id == IPPROTO_ESP)
+                if (iph4->next_proto_id == IPPROTO_ESP) {
+		if (pkt->ol_flags & PKT_RX_SEC_OFFLOAD) {
+			/* extract extra info, make decision based on that */
+		} else
			t->ipsec.pkts[(t->ipsec.num)++] = pkt;		
+	   } 

Of course some code reordering might be needed to avoid performance drop,
but I suppose the main idea is clear enough. 
 Second variant - at early stage of single_inbound_lookup(),
either before or straight after actual SAD SW lookup failure.
In both cases, this new code will be executed *before* session selection code.

> 
> Does the above sound like BYPASS? Yes.
> Does ipsec-secgw do it this way? No.

Correct, it doesn't.
You have to submit patches if you'd like to support such ability. 

> Does this approach prevent an application from introducing such a BYPASS? Yes.

That's not correct, I believe, see above.

> 
> >
> > >
> > > For example, ESN was not supported by ipsec-secgw (before library mode
> > > was introduced), but every single library change and application
> > > change was added keeping in mind that ESN is a valid feature for ipsec. So it is
> > one thing to say ipsec-secgw doesn't support ESP bypass and saying the solution
> > doesn't consider a possibility of ESP bypass.
> > >
> > > >
> > > > >
> > > > > 2. The solution proposed here seems like adding the handling in
> > > > > ipsec-secgw instead of ipsec library. In other words, this feature
> > > > > is not getting
> > > > added in ipsec library, which was supposed to simplify the whole
> > > > ipsec usage in DPDK, but fails to handle the case of fragmentation.
> > > >
> > > > What we have right now with ipsec library is SA (low) level API.
> > > > It can handle multi-segment packets properly, but expects someone
> > > > else to do other steps (fragmentation/reassembly).
> > > > ipsec-secgw demonstrates how librte_ip_frag and librte_ipsec can be
> > > > used together to deal with fragmented IPsec traffic in a proper manner.
> > > > Probably in future we'll consider adding some high-level API that
> > > > will be able to do whole processing under hood (SPD/SAD lookup,
> > > > fragment/reassembly, actual IPsec packet processing, matching
> > > > inbound selectors, etc.), but right now it is not the case.
> > > >
> > > > > Also, since the fallback feature is entirely done in the
> > > > > application, it begs the
> > > > question why the same feature is omitted for legacy use case.
> > > >
> > > > Because legacy mode doesn't support multi-seg packets at first place.
> > > > Also it is an extra overhead to support 2 code-paths (legacy and
> > > > library) for the same app, so we'd like in future to deprecate and
> > > > later remove legacy code- path.
> > > > As a first step we propose to make library code-path a default one:
> > > > https://urldefense.proofpoint.com/v2/url?u=http-3A__patches.dpdk.org
> > > >
> > _cover_58247_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=jPfB8rwwviRSxyL
> > Ws2
> > > > n6B-WYLn1v9SyTMrT5EQqh2TU&m=b3E429fuo8P-K5XfH8wG-
> > 7hwr1d8oM4uJGErAkbf
> > > > DvA&s=vz7ioUzJOuzoJdmV7QO0QLPKYn1ytFsj_0eYatbSCrg&e=
> > >
> > > [Anoob] Even if we need the application to do the reassembly, it would
> > > look simple if application uses the same "rte_ipsec_session" to process the
> > fallback packet.
> >
> > I think there is some sort of misunderstanding here.
> > With current librte_ipsec design:
> > rte_ipsec_sa - opaque SW representation of the SA (HW neutral)
> > rte_ipsec_session associates SA (rte_ipsec_sa)with particular HW device
> > (session).
> > Same SA can be referred by multiple sessions.
> >
> > From rte_ipsec.h:
> > /**
> >  * rte_ipsec_session is an aggregate structure that defines particular
> >  * IPsec Security Association IPsec (SA) on given security/crypto device:
> >  * - pointer to the SA object
> >  * - security session action type
> >  * - pointer to security/crypto session, plus other related data
> >  * - session/device specific functions to prepare/process IPsec packets.
> >  */
> >
> > > In that case, the processing required to handle the other packet will
> > > be hidden from the application. Here application has to make sure it chooses
> > the correct session, which I believe should be moved to ipsec library.
> >
> > From my side it is a good thing to let application define its own usage model.
> > I.E. librte_ipsec supports multiple-sessions per SA, upper-layer (app) decides how
> > it will use that feature.
> > Though if you strongly feel that some higher-level API is needed here, and have
> > some good ideas about it  - please go ahead and submit RFC for it.
> 
> [Anoob] If you are okay with application defining how it implements ipsec, then probably there is no use case for lib ipsec?

:)
Once again - library provides an implementation.
Application defines the way to use it, i.e. how it will apply functionality library provides for different use-case scenarios.
As an example: snpritnf() provides user with ability to format strings,
application decides what exactly format to use and for which string.
Same for librte_ipsec, library provides functionality to perform processing for ESP packets.
Application defines which packets to process and what session to use. 

> 
> >
> > >
> > > >
> > > > >
> > > > > 3. It seems like ordering won't be maintained once this processing
> > > > > is done. Again, this is the sequence I understood. Please correct
> > > > > me if I missed
> > > > something,
> > > > >        a. Application receives a bunch of packets (let's say 6
> > > > > packets), in which few are fragmented (P3 & P4) and the rest can
> > > > > be inline
> > > > processed.
> > > > >        b. Application receives P1->P2->P3->P4->P5->P6 (in this,
> > > > > P1, P2, P5, P6 are
> > > > inline processed successfully) and P4 & P5 are the fragments
> > > > >        c. Application groups packets. P1->P2->P5->P6 becomes one
> > > > >group and P3-
> > > > >P4 becomes another and goes for fallback processing.
> > > > > Now how is ordering maintained? I couldn't figure out how that is
> > > > >done in this
> > > > case.
> > > >
> > > > You right, fallback session can introduce packet reordering.
> > > > At least till we'll have ability to process packets in sync mode too.
> > > > See our presentation at dpdk userspace (slides 17, 18):
> > > > https://urldefense.proofpoint.com/v2/url?u=https-3A__static.sched.co
> > > > m_hosted-5Ffiles_dpdkbordeaux2019_8f_DPDK-
> > 2DIPSECu9.pdf&d=DwIFAg&c=n
> > > > KjWec2b6R0mOyPaz7xtfQ&r=jPfB8rwwviRSxyLWs2n6B-
> > WYLn1v9SyTMrT5EQqh2TU&
> > > > m=b3E429fuo8P-K5XfH8wG-
> > 7hwr1d8oM4uJGErAkbfDvA&s=MLRnnYcykjnsqXrHGUuR
> > > > YHc5uDxaAod0Yl7f06EQr1M&e= Right now the only way to deal with it -
> > > > have replay window big enough to sustain reordering and async
> > > > processing latency.
> > > > That's actually another reason why we add this feature into
> > > > ipsec-secgw sample
> > > > app:
> > > > so people can evaluate it on their platforms, determine what replay
> > > > window size would be needed, what issues/slowdowns it might cause, etc.
> > >
> > > [Anoob] This is something I had noticed while going through the code flow.
> > The ordering info is lost because of the approach.
> >
> > Once again, it is a known limitation and we are not trying to hide it from you :)
> 
> [Anoob] Never said it was hidden. Initially I had okayed the approach even though it wasn't a solution suitable for inline protocol. But as
> more cases were considered, few shortcomings were observed, and I was skeptical about how any of that would be solved in the long run.
> 
> My suggestion, if these limitations and the plans to address it were mentioned in the cover letter, we could've saved few cycles here.

Point taken.

> But my reply would still be the same 😊
> 
> > It
> > was outlined here:
> > https://urldefense.proofpoint.com/v2/url?u=https-
> > 3A__static.sched.com_hosted-5Ffiles_dpdkbordeaux2019_8f_DPDK-
> > 2DIPSECu9.pdf&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=jPfB8rwwviRSxyL
> > Ws2n6B-WYLn1v9SyTMrT5EQqh2TU&m=b3E429fuo8P-K5XfH8wG-
> > 7hwr1d8oM4uJGErAkbfDvA&s=MLRnnYcykjnsqXrHGUuRYHc5uDxaAod0Yl7f06E
> > Qr1M&e=
> > And in the latest patch version Marcin clearly stated it inside the AG session:
> > https://urldefense.proofpoint.com/v2/url?u=http-
> > 3A__patches.dpdk.org_patch_60039_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtf
> > Q&r=jPfB8rwwviRSxyLWs2n6B-WYLn1v9SyTMrT5EQqh2TU&m=b3E429fuo8P-
> > K5XfH8wG-
> > 7hwr1d8oM4uJGErAkbfDvA&s=zrn91Vjf_ZElAlDf7IeZGmGevcA6RMwpPTLUCGlgZ
> > cY&e=
> > If you think even further clarification within the doc is needed, please let us
> > know.
> >
> > About the actual packet re-oridering:
> >  AFAIK, for some use-cases it might be acceptable, for others not.
> > Though it is an optional feature, that wouldn't be enabled by default, so user can
> > always choose is it does he needs/wants this feature or not.
> > If/when we'll have CPU_CRYPTO processing mode:
> 
> [Anoob] Shouldn't we get that accepted first then?

I don't think so.
Current implementation does provide expected functionality (with known limitations).
In future, we can try to improve it and/or remove existing limitations.
That's a common iteration development approach that is used though whole DPDK:
- provide initial solution with basic functionality first
- improve/extend with future releases/patches

> 
> Also, the ordering is lost because application is not considering that. Why can't we use re-ordering library? Or an event-dev?

No-one saying it can't be improved in one way or another.
We don't plan to introduce such code right now due to its complexity.
Might be something in future.
Though you are welcome to go ahead and submit your own patches to improve that solution.   

> 
> > https://urldefense.proofpoint.com/v2/url?u=http-
> > 3A__patches.dpdk.org_cover_58862_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtf
> > Q&r=jPfB8rwwviRSxyLWs2n6B-WYLn1v9SyTMrT5EQqh2TU&m=b3E429fuo8P-
> > K5XfH8wG-
> > 7hwr1d8oM4uJGErAkbfDvA&s=k1zef_1uqELXblKN3_qjX04WXNFGAUEFIIJrvKPr6
> > eA&e=
> > we'll add ability for the user to avoid this reordering problem
> >
> > > As we dig deeper, the feature looks hardly complete. The concerning
> > > part is the approach doesn't seem conducive to fixing any of these issues in the
> > future.
> >
> > Again, don't agree with that statement.
> > From my perspective: while current implementation has its limitations (packet
> > reordering, inline-proto/lksd-proto), it is totally functional and provides users
> > with a reference how to deal with such kind of problems.
> 
> [Anoob] Well, that is exactly my problem. It gives a skewed reference on how to deal with the problem, without considering other
> possibilities.

Once again, please feel free and come-up with your own patches, that will address this problem in a smarter way.

> If every application starts adopting this method and starts doing lookup for every ESP packet, h/w based lookups (and the
> associated h/w) would be rendered useless. I really think this series, which happens to be the first step in solving a complex problem, will be
> very useful if all these issues are addressed properly.
> 
> If you can introduce the inline+lookaside via librte_ipsec, its perhaps ok for the patch to have some limitations(like lack of ordering,
> assumptions on ESP etc). However, if the patch is introduced directly into ipsec-secgw, it needs to be more comprehensive and robust.

I think common DPDK policy is exactly opposite:
library/driver has to be as robust and comprehensive as possible.
sample app code allowed to have some limitations (as long as they are documented, etc.). 

> I also expect performance impact due to these changes

Could you be more specific here?
What performance impact do you expect and why?
Can you point to the exact piece of patch code that you believe would cause a degradation.
I don't expect any and performance tests on our platforms didn't show any regression so far.

> and so I prefer if the inline+lookaside can be made separate datapath rather than
> overloading the existing one.

I don't see any good reason for that.
This feature is optional.
User have to enable it manually for each session (via config file).
for all other sessions there is no impact.
What we probably can do - add extra check inside sa.c to allow
fallback sessions only for inline-crypto-offload type of sessions.

> 
> >
> > >
> > > Also, if the new ipsec related features are introduced via ipsec-secgw
> > > than via rte_ipsec, then it raises doubts over the utility of rte_ipsec library.
> >
> >

Re: [dpdk-dev] [PATCH v3 0/3] add fallback session

[Anoob Joseph]

Hi Konstantin,

Following are the main issues I had identified,

1. Doesn't work for inline protocol offload (the plan itself won't scale for that case)
2. Ignores the data from the h/w lookup
3. The fallback support is actually supported in ipsec-secgw (application) instead of librte_ipsec.
4. Reordering issue.
5. Introduces checks in existing data path even when the feature is not properly implemented/is not suitable for other PMDs.

For Issue #1, I get that Intel doesn't have plans to support the feature and won't be submitting anything in that direction. But adding code that further exacerbates the situation is probably not desired.

For Issue #2, since ipsec-secgw is not utilizing the h/w lookup data, the solution here follows suit and is preventing other usage models of the hardware which are allowed by the library.

For Issue #3, librte_ipsec is still experimental and is just 2-3 releases old. It has scope for improvement and all these features would've added value to its utility. No strong feelings here.

For Issue #4, glad that you documented that issue.

For Issue #5, in your reply you had mentioned that there is no additional check done. I see atleast few checks in the datapath. Since the feature itself is experimental, it would be less contentious if it was introduced as a new data path.

Also, to quote from the reply,

> > [Anoob] Shouldn't we get that accepted first then?
> 
> I don't think so.
> Current implementation does provide expected functionality (with known
> limitations).
> In future, we can try to improve it and/or remove existing limitations.
> That's a common iteration development approach that is used though whole
> DPDK:
> - provide initial solution with basic functionality first
> - improve/extend with future releases/patches

If your suggestion is to stick to above guidelines, I don't think there is any point in pushing any further. I leave this to maintainers to decide. I would assume the same rules would apply to every feature that is being attempted.

Thanks,
Anoob

> -----Original Message-----
> From: dev <dev-bounces@dpdk.org> On Behalf Of Ananyev, Konstantin
> Sent: Thursday, October 3, 2019 8:16 PM
> To: Anoob Joseph <anoobj@marvell.com>; Smoczynski, MarcinX
> <marcinx.smoczynski@intel.com>; akhil.goyal@nxp.com
> Cc: dev@dpdk.org; Jerin Jacob Kollanukkaran <jerinj@marvell.com>; Narayana
> Prasad Raju Athreya <pathreya@marvell.com>; Archana Muniganti
> <marchana@marvell.com>
> Subject: Re: [dpdk-dev] [PATCH v3 0/3] add fallback session
> 
> Hi Anoob,
> 
> > > > > > I've few more observations regarding the proposed feature.
> > > > > >
> > > > > > 1. From what I understood, if an ESP packet ends up on an
> > > > > > unprotected interface and doesn't have 'PKT_RX_SEC_OFFLOAD'
> > > > > > bit set, then the packet
> > > > > would be looked up to see the associated SA and then fallback
> > > > > session is figured out and then further processing is done.
> > > > > >
> > > > > > Can you confirm if I understood the sequence correctly? If
> > > > > > yes, then aren't we doing an extra lookup in the s/w? The
> > > > > > packet may be looked by the h/w using rte_flow and that
> > > > > > information could be used to determine the
> > > > > SA. Also, if the ESP packet is expected to be forwarded, then
> > > > > the above logic will add an unnecessary lookup even after your
> > > > > h/w has detected that the packet need not be security processed.
> > > > >
> > > > > Not sure I understood your whole statement above.
> > > > > AFAIK, right now (with dpdk master) for unprotected iface it works like
> that:
> > > > >
> > > > > 1.  slit incoming traffic into 3 groups: ESP packets, IPv4
> > > > > packets, IPv6
> > > packets.
> > > > > For ESP packets:
> > > > > 2. perform SAD lookup
> > > > >     a. if it fails (non SA found for that SPI), then drop the packet.
> > > > >     b. otherwise (SA found) process the packet using found SA
> > > > >
> > > > > What fall-back patch adds:
> > > > > Before step 2.b check:
> > > > > does that SA has its primary session  with type INLINE-CRYPTO
> > > > > and does HW fail to do IPsec realted processing for it (by some reason)?
> > > > > If yes, then mark this packet to be processed by fall-back session.
> > > > > if (MBUF_NO_SEC_OFFLOAD(pkt) && sa->fallback_sessions > 0) {
> > > > >                 uintptr_t intsa = (uintptr_t)sa;
> > > > >                 intsa |= IPSEC_SA_OFFLOAD_FALLBACK_FLAG;
> > > > >                 result_sa = (void *)intsa;  }
> > > > >
> > > > > So from my perspective, no additional lookup where introduced.
> > > >
> > > > [Anoob] For inline processing, h/w does a lookup and figures out
> > > > the security
> > > processing to be done on the packet.
> > > > "rte_security_get_userdata()" allows s/w to further segregate that
> > > > info. In this approach, I believe we don't consider how such info
> > > > from h/w can
> > > be used.
> > >
> > > Right now it is not the case with ipsec-secgw:
> >
> > [Anoob] Are you saying there is no h/w lookup involved when doing
> > inline crypto processing? Then I'm confused. My understanding is that rte_flow
> with ACTION type as SECURITY is being used to setup h/w lookups.
> 
> I am saying that current ipsec-secgw code for each inbound ESP packet
> *always* does SW lookup.
> No matter did HW lookup take place already and does information about that
> lookup is available via mbuf in some way or not.
> Just look at the ipsec-secgw code yourself:
> 
> nb_rx = rte_eth_rx_burst(portid, queueid,  pkts, MAX_PKT_BURST);
> 
> if (nb_rx > 0)
> 	process_pkts(qconf, pkts, nb_rx, portid);
> 
> Inside process_pkts()
> https://urldefense.proofpoint.com/v2/url?u=http-
> 3A__lxr.dpdk.org_dpdk_latest_ident_prepare-5Fone-
> 5Fpacket&d=DwIGaQ&c=nKjWec2b6R0mOyPaz7xtfQ&r=BPcGOOudUMrTDQ9Yb
> gKcOkO5ChYiUPPlPNIEvTOhjNE&m=KVBgvXYlUGyMmu260erISKr73Qt6PBLux0SI
> oSO1i-M&s=opbGGHUqfxy8p_4NvwCG6od6VdV0qN8XafaPiubfO1A&e=
>  it first calls prepare_traffic() which does sort of de-muxing:
> put packet in one of 3 arrays:
>    1) ESP packets
>    2) non ESP IPv4 packets
>    3) non ESP IPv6 packets
> Also it checks is PKT_RX_SEC_OFFLOAD set for that packet.
> If yes, it retrieves associated security user-data and stores it inside private mbuf
> area.
> 
> Then it goes into process_pkts_inbound()
> https://urldefense.proofpoint.com/v2/url?u=http-
> 3A__lxr.dpdk.org_dpdk_latest_ident_process-5Fpkts-
> 5Finbound&d=DwIGaQ&c=nKjWec2b6R0mOyPaz7xtfQ&r=BPcGOOudUMrTDQ9Y
> bgKcOkO5ChYiUPPlPNIEvTOhjNE&m=KVBgvXYlUGyMmu260erISKr73Qt6PBLux0S
> IoSO1i-M&s=2OxMXug7jtUQPcYiDN4nW9WbrMtRfTJx5k-N7ikOeME&e=
> and here for all ESP packets:
> legacy code path:
> ipsec_inbound()->inbound_sa_lookup()->single_inbound_lookup()
> for librte_ipsec code path:
> inbound_sa_lookup()->single_inbound_lookup()
> 
> single_inbound_lookup() is the one that does actual SW lookup for each input
> packet.
> https://urldefense.proofpoint.com/v2/url?u=http-
> 3A__lxr.dpdk.org_dpdk_latest_ident_single-5Finbound-
> 5Flookup&d=DwIGaQ&c=nKjWec2b6R0mOyPaz7xtfQ&r=BPcGOOudUMrTDQ9Yb
> gKcOkO5ChYiUPPlPNIEvTOhjNE&m=KVBgvXYlUGyMmu260erISKr73Qt6PBLux0SI
> oSO1i-M&s=D2ZPZPLbcQSbaluG8KH0F3uUDHfQWNYxw4MOc0fSkI8&e=
> 
> >
> > > for each inbound ESP packet it *always* does a lookup in SW based
> > > SADB, and if lookup fails - drops the packet (see 2.a above).
> > > So, I still not see any additional lookups introduced with the patch.
> >
> > [Anoob] In case of inline crypto, you could do that. No disagreement.
> > But that doesn't mean that is the only way. If PMDs can retrieve info about
> h/w lookups and pass it on to the upper layers, isn't that the better approach?
> 
> Please let's keep our conversation in a constructive way.
> We are not discussing what could be done in theory, but a particular patch for
> ipsec-secgw:
> Right now ipsec-secgw does a SW lookup for each inbound ESP packet (no
> matter what HW offload does) and this patch doesn't introduce any extra SW
> lookups.
> 
> If you are not happy with current ipsec-secgw approach, that's fine - feel free to
> submit RFC/patch to fix that, or start a discussion in a new thread.
> I just don't see why we have to discuss it in context of this patch.
> 
> >
> > >
> > > >
> > > > > Also AFAIK, right now there is no possibility to configure
> > > > > ipsec-secgw to BYPASS some ESP traffic.
> > > > > Should we do it (to conform to ipsec RFC) - that's probably
> > > > > subject of another discussion.
> > > >
> > > > [Anoob] The approach (choice of flags) forces a software-based SA
> > > > lookup for packets that need to be bypassed instead of leveraging
> > > > possible
> > > hardware SA lookup. I don't think what ipsec-secgw does matters here.
> > >
> > > I do not agree with that statement.
> > > First of all - current patch doesn't introduce any additional SW
> > > lookups, see above.
> > > Second, if someone would like to add BYPASS for ESP packets into
> > > ipsec-secgw, I think he would have to insert new code that do de-mux
> > > *before* any ESP related processing starts.
> > > Most obvious variant: at prepare_one_packet() when we split our
> > > incoming traffic to IPsec and non-IPsec ones.
> > > Second variant - at early stage of single_inbound_lookup(), straight
> > > after actual SAD lookup failure.
> > > In both cases, I don't see how session selection (primary or
> > > faillback) would affect us here, actual decision do we want to
> > > PROCESS or BYPASS particular ESP packet needs to take place before that.
> > > So I still believe we are ok here.
> >
> > [Anoob] I don't think you understood what I'm trying to highlight
> > here. You could have a situation when h/w can detect that the packet
> > has to be "PROTECT"ed using an "inline crypto" session. But then it
> > might detect that it cannot proceed with inline processing because of some
> issue (fragmentation, h/w queue overflow etc). Now the h/w has already figured
> out the action to be done to the packet. If PMDs allow this to be communicated
> to the application, the application won't be required to do the lookup.
> >
> > In inline crypto, application can ignore the h/w lookup data and do a
> > s/w lookup with the protocol headers as they are not stripped off. It
> > was done this way, because API and the associated means to get this
> > info from PMD was not introduced when inline crypto and corresponding
> support in Intel's PMD was added. But inline protocol cannot do the lookup in
> the application and so the concept of userdata etc was introduced. The same
> can be applied to inline crypto also. Advantage? It could remove the extra
> lookup done in s/w.
> >
> > To summarize, we cannot assume that every inline crypto packet need to
> > looked up in the application to figure out the processing done on the packet.
> There can be applications which relies on h/w lookup data to figure out the
> processing done.
> >
> > Example: Using rte_flow, I can create a rule for ESP packets to be
> > MARKed. There is no security session created for the flow and the
> > application intends to forward this flow packets to a different port. With your
> approach, even these packets would be looked up. The packet will have info
> from the h/w lookup which doesn't get used, because the approach fails to
> introduce the required concepts.
> 
> I think I understood your point.
> Basically you saying that in future your PMD for unprocessed ESP packets might
> provide some additional information (via rte_security_get_userdata) that might
> be used by SW - let say to choose BYPASS for such packets.
> Current ipsec-secgw doesn't support such ability.
> Your concern is that this patch will somehow prevent (or will make it harder) for
> you to make your future changes.
> Correct?
> If, so then I said before, I don't think that concern is valid.
> Most obvious variant where to add such code is inisde inside
> prepare_one_packet() when we split our incoming traffic to IPsec and non-IPsec
> ones.
> To be more specific something like that
> 
> if (eth->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) {
> 
>                 iph4 = (const struct rte_ipv4_hdr *)rte_pktmbuf_adj(pkt,
>                         RTE_ETHER_HDR_LEN);
>                 adjust_ipv4_pktlen(pkt, iph4, 0);
> 
> -                if (iph4->next_proto_id == IPPROTO_ESP)
> +                if (iph4->next_proto_id == IPPROTO_ESP) {
> +		if (pkt->ol_flags & PKT_RX_SEC_OFFLOAD) {
> +			/* extract extra info, make decision based on that */
> +		} else
> 			t->ipsec.pkts[(t->ipsec.num)++] = pkt;
> +	   }
> 
> Of course some code reordering might be needed to avoid performance drop,
> but I suppose the main idea is clear enough.
>  Second variant - at early stage of single_inbound_lookup(), either before or
> straight after actual SAD SW lookup failure.
> In both cases, this new code will be executed *before* session selection code.
> 
> >
> > Does the above sound like BYPASS? Yes.
> > Does ipsec-secgw do it this way? No.
> 
> Correct, it doesn't.
> You have to submit patches if you'd like to support such ability.
> 
> > Does this approach prevent an application from introducing such a BYPASS?
> Yes.
> 
> That's not correct, I believe, see above.
> 
> >
> > >
> > > >
> > > > For example, ESN was not supported by ipsec-secgw (before library
> > > > mode was introduced), but every single library change and
> > > > application change was added keeping in mind that ESN is a valid
> > > > feature for ipsec. So it is
> > > one thing to say ipsec-secgw doesn't support ESP bypass and saying
> > > the solution doesn't consider a possibility of ESP bypass.
> > > >
> > > > >
> > > > > >
> > > > > > 2. The solution proposed here seems like adding the handling
> > > > > > in ipsec-secgw instead of ipsec library. In other words, this
> > > > > > feature is not getting
> > > > > added in ipsec library, which was supposed to simplify the whole
> > > > > ipsec usage in DPDK, but fails to handle the case of fragmentation.
> > > > >
> > > > > What we have right now with ipsec library is SA (low) level API.
> > > > > It can handle multi-segment packets properly, but expects
> > > > > someone else to do other steps (fragmentation/reassembly).
> > > > > ipsec-secgw demonstrates how librte_ip_frag and librte_ipsec can
> > > > > be used together to deal with fragmented IPsec traffic in a proper
> manner.
> > > > > Probably in future we'll consider adding some high-level API
> > > > > that will be able to do whole processing under hood (SPD/SAD
> > > > > lookup, fragment/reassembly, actual IPsec packet processing,
> > > > > matching inbound selectors, etc.), but right now it is not the case.
> > > > >
> > > > > > Also, since the fallback feature is entirely done in the
> > > > > > application, it begs the
> > > > > question why the same feature is omitted for legacy use case.
> > > > >
> > > > > Because legacy mode doesn't support multi-seg packets at first place.
> > > > > Also it is an extra overhead to support 2 code-paths (legacy and
> > > > > library) for the same app, so we'd like in future to deprecate
> > > > > and later remove legacy code- path.
> > > > > As a first step we propose to make library code-path a default one:
> > > > > https://urldefense.proofpoint.com/v2/url?u=http-3A__patches.dpdk
> > > > > .org
> > > > >
> > >
> _cover_58247_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=jPfB8rwwviRSxyL
> > > Ws2
> > > > > n6B-WYLn1v9SyTMrT5EQqh2TU&m=b3E429fuo8P-K5XfH8wG-
> > > 7hwr1d8oM4uJGErAkbf
> > > > > DvA&s=vz7ioUzJOuzoJdmV7QO0QLPKYn1ytFsj_0eYatbSCrg&e=
> > > >
> > > > [Anoob] Even if we need the application to do the reassembly, it
> > > > would look simple if application uses the same "rte_ipsec_session"
> > > > to process the
> > > fallback packet.
> > >
> > > I think there is some sort of misunderstanding here.
> > > With current librte_ipsec design:
> > > rte_ipsec_sa - opaque SW representation of the SA (HW neutral)
> > > rte_ipsec_session associates SA (rte_ipsec_sa)with particular HW
> > > device (session).
> > > Same SA can be referred by multiple sessions.
> > >
> > > From rte_ipsec.h:
> > > /**
> > >  * rte_ipsec_session is an aggregate structure that defines
> > > particular
> > >  * IPsec Security Association IPsec (SA) on given security/crypto device:
> > >  * - pointer to the SA object
> > >  * - security session action type
> > >  * - pointer to security/crypto session, plus other related data
> > >  * - session/device specific functions to prepare/process IPsec packets.
> > >  */
> > >
> > > > In that case, the processing required to handle the other packet
> > > > will be hidden from the application. Here application has to make
> > > > sure it chooses
> > > the correct session, which I believe should be moved to ipsec library.
> > >
> > > From my side it is a good thing to let application define its own usage model.
> > > I.E. librte_ipsec supports multiple-sessions per SA, upper-layer
> > > (app) decides how it will use that feature.
> > > Though if you strongly feel that some higher-level API is needed
> > > here, and have some good ideas about it  - please go ahead and submit RFC
> for it.
> >
> > [Anoob] If you are okay with application defining how it implements ipsec,
> then probably there is no use case for lib ipsec?
> 
> :)
> Once again - library provides an implementation.
> Application defines the way to use it, i.e. how it will apply functionality library
> provides for different use-case scenarios.
> As an example: snpritnf() provides user with ability to format strings, application
> decides what exactly format to use and for which string.
> Same for librte_ipsec, library provides functionality to perform processing for
> ESP packets.
> Application defines which packets to process and what session to use.
> 
> >
> > >
> > > >
> > > > >
> > > > > >
> > > > > > 3. It seems like ordering won't be maintained once this
> > > > > > processing is done. Again, this is the sequence I understood.
> > > > > > Please correct me if I missed
> > > > > something,
> > > > > >        a. Application receives a bunch of packets (let's say 6
> > > > > > packets), in which few are fragmented (P3 & P4) and the rest
> > > > > > can be inline
> > > > > processed.
> > > > > >        b. Application receives P1->P2->P3->P4->P5->P6 (in
> > > > > > this, P1, P2, P5, P6 are
> > > > > inline processed successfully) and P4 & P5 are the fragments
> > > > > >        c. Application groups packets. P1->P2->P5->P6 becomes
> > > > > >one group and P3-
> > > > > >P4 becomes another and goes for fallback processing.
> > > > > > Now how is ordering maintained? I couldn't figure out how that
> > > > > >is done in this
> > > > > case.
> > > > >
> > > > > You right, fallback session can introduce packet reordering.
> > > > > At least till we'll have ability to process packets in sync mode too.
> > > > > See our presentation at dpdk userspace (slides 17, 18):
> > > > > https://urldefense.proofpoint.com/v2/url?u=https-3A__static.sche
> > > > > d.co
> > > > > m_hosted-5Ffiles_dpdkbordeaux2019_8f_DPDK-
> > > 2DIPSECu9.pdf&d=DwIFAg&c=n
> > > > > KjWec2b6R0mOyPaz7xtfQ&r=jPfB8rwwviRSxyLWs2n6B-
> > > WYLn1v9SyTMrT5EQqh2TU&
> > > > > m=b3E429fuo8P-K5XfH8wG-
> > > 7hwr1d8oM4uJGErAkbfDvA&s=MLRnnYcykjnsqXrHGUuR
> > > > > YHc5uDxaAod0Yl7f06EQr1M&e= Right now the only way to deal with
> > > > > it - have replay window big enough to sustain reordering and
> > > > > async processing latency.
> > > > > That's actually another reason why we add this feature into
> > > > > ipsec-secgw sample
> > > > > app:
> > > > > so people can evaluate it on their platforms, determine what
> > > > > replay window size would be needed, what issues/slowdowns it might
> cause, etc.
> > > >
> > > > [Anoob] This is something I had noticed while going through the code flow.
> > > The ordering info is lost because of the approach.
> > >
> > > Once again, it is a known limitation and we are not trying to hide
> > > it from you :)
> >
> > [Anoob] Never said it was hidden. Initially I had okayed the approach
> > even though it wasn't a solution suitable for inline protocol. But as more cases
> were considered, few shortcomings were observed, and I was skeptical about
> how any of that would be solved in the long run.
> >
> > My suggestion, if these limitations and the plans to address it were mentioned
> in the cover letter, we could've saved few cycles here.
> 
> Point taken.
> 
> > But my reply would still be the same 😊
> >
> > > It
> > > was outlined here:
> > > https://urldefense.proofpoint.com/v2/url?u=https-
> > > 3A__static.sched.com_hosted-5Ffiles_dpdkbordeaux2019_8f_DPDK-
> > >
> 2DIPSECu9.pdf&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=jPfB8rwwviRSxyL
> > > Ws2n6B-WYLn1v9SyTMrT5EQqh2TU&m=b3E429fuo8P-K5XfH8wG-
> > >
> 7hwr1d8oM4uJGErAkbfDvA&s=MLRnnYcykjnsqXrHGUuRYHc5uDxaAod0Yl7f06E
> > > Qr1M&e=
> > > And in the latest patch version Marcin clearly stated it inside the AG session:
> > > https://urldefense.proofpoint.com/v2/url?u=http-
> > >
> 3A__patches.dpdk.org_patch_60039_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtf
> > > Q&r=jPfB8rwwviRSxyLWs2n6B-
> WYLn1v9SyTMrT5EQqh2TU&m=b3E429fuo8P-
> > > K5XfH8wG-
> > >
> 7hwr1d8oM4uJGErAkbfDvA&s=zrn91Vjf_ZElAlDf7IeZGmGevcA6RMwpPTLUCGlgZ
> > > cY&e=
> > > If you think even further clarification within the doc is needed,
> > > please let us know.
> > >
> > > About the actual packet re-oridering:
> > >  AFAIK, for some use-cases it might be acceptable, for others not.
> > > Though it is an optional feature, that wouldn't be enabled by
> > > default, so user can always choose is it does he needs/wants this feature or
> not.
> > > If/when we'll have CPU_CRYPTO processing mode:
> >
> > [Anoob] Shouldn't we get that accepted first then?
> 
> I don't think so.
> Current implementation does provide expected functionality (with known
> limitations).
> In future, we can try to improve it and/or remove existing limitations.
> That's a common iteration development approach that is used though whole
> DPDK:
> - provide initial solution with basic functionality first
> - improve/extend with future releases/patches
> 
> >
> > Also, the ordering is lost because application is not considering that. Why can't
> we use re-ordering library? Or an event-dev?
> 
> No-one saying it can't be improved in one way or another.
> We don't plan to introduce such code right now due to its complexity.
> Might be something in future.
> Though you are welcome to go ahead and submit your own patches to improve
> that solution.
> 
> >
> > > https://urldefense.proofpoint.com/v2/url?u=http-
> > >
> 3A__patches.dpdk.org_cover_58862_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtf
> > > Q&r=jPfB8rwwviRSxyLWs2n6B-
> WYLn1v9SyTMrT5EQqh2TU&m=b3E429fuo8P-
> > > K5XfH8wG-
> > >
> 7hwr1d8oM4uJGErAkbfDvA&s=k1zef_1uqELXblKN3_qjX04WXNFGAUEFIIJrvKPr6
> > > eA&e=
> > > we'll add ability for the user to avoid this reordering problem
> > >
> > > > As we dig deeper, the feature looks hardly complete. The
> > > > concerning part is the approach doesn't seem conducive to fixing
> > > > any of these issues in the
> > > future.
> > >
> > > Again, don't agree with that statement.
> > > From my perspective: while current implementation has its
> > > limitations (packet reordering, inline-proto/lksd-proto), it is
> > > totally functional and provides users with a reference how to deal with such
> kind of problems.
> >
> > [Anoob] Well, that is exactly my problem. It gives a skewed reference
> > on how to deal with the problem, without considering other possibilities.
> 
> Once again, please feel free and come-up with your own patches, that will
> address this problem in a smarter way.
> 
> > If every application starts adopting this method and starts doing
> > lookup for every ESP packet, h/w based lookups (and the associated
> > h/w) would be rendered useless. I really think this series, which happens to be
> the first step in solving a complex problem, will be very useful if all these issues
> are addressed properly.
> >
> > If you can introduce the inline+lookaside via librte_ipsec, its
> > perhaps ok for the patch to have some limitations(like lack of ordering,
> assumptions on ESP etc). However, if the patch is introduced directly into ipsec-
> secgw, it needs to be more comprehensive and robust.
> 
> I think common DPDK policy is exactly opposite:
> library/driver has to be as robust and comprehensive as possible.
> sample app code allowed to have some limitations (as long as they are
> documented, etc.).
> 
> > I also expect performance impact due to these changes
> 
> Could you be more specific here?
> What performance impact do you expect and why?
> Can you point to the exact piece of patch code that you believe would cause a
> degradation.
> I don't expect any and performance tests on our platforms didn't show any
> regression so far.
> 
> > and so I prefer if the inline+lookaside can be made separate datapath
> > rather than overloading the existing one.
> 
> I don't see any good reason for that.
> This feature is optional.
> User have to enable it manually for each session (via config file).
> for all other sessions there is no impact.
> What we probably can do - add extra check inside sa.c to allow fallback sessions
> only for inline-crypto-offload type of sessions.
> 
> >
> > >
> > > >
> > > > Also, if the new ipsec related features are introduced via
> > > > ipsec-secgw than via rte_ipsec, then it raises doubts over the utility of
> rte_ipsec library.
> > >
> > >

Re: [dpdk-dev] [PATCH v3 0/3] add fallback session

[Ananyev, Konstantin]

Hi Anoob,

> 
> Following are the main issues I had identified,
> 
> 1. Doesn't work for inline protocol offload (the plan itself won't scale for that case)
> 2. Ignores the data from the h/w lookup
> 3. The fallback support is actually supported in ipsec-secgw (application) instead of librte_ipsec.
> 4. Reordering issue.
> 5. Introduces checks in existing data path even when the feature is not properly implemented/is not suitable for other PMDs.
> 
> For Issue #1, I get that Intel doesn't have plans to support the feature and won't be submitting anything in that direction.
> But adding code that further exacerbates the situation is probably not desired.

We already discussed why that feature can't be fully supported right now by
inline-proto/lookaside-proto devices:
There is no API defined to sync session state between HW and SW
http://mails.dpdk.org/archives/dev/2019-September/143881.html

I already outlined that gap nearly a year ago:
http://mails.dpdk.org/archives/dev/2018-September/113600.html
Though so far I didn't see any attempt to resolve this problem by Marvell (or NXP) engineers.
Which makes me think that either you guys don't consider it as a problem at all,
or have no desire to work on it.
I never insisted, as I don't  think it is my call by obvious reasons: 
1) right now Intel doesn't have HW that provides inline-proto/lksd-proto capabilities.
2) Intel DPDK team doesn't have access to such HW.
3) There is no PMD inside dpdk.org that implements inline-proto anyway.  

Now, one year later we are in a situation where:
1) there still no PMD at dpdk.org that supports inline-proto.
2) API to sync session state between HW and SW still not defined/implemented.
3) you trying to accuse me and other Intel DPDK team for lack of interest to work on that problem for you
   and threat our patches because of that.

That just doesn't look fair to me.
You shouldn't expect other people patches to provide 'complete' solution for inline-proto,
if (in a year time) you failed to define/implement an API for that.

> For Issue #2, since ipsec-secgw is not utilizing the h/w lookup data, the solution here follows suit and is preventing other usage models of
> the hardware which are allowed by the library.

Same as above, right now there is no PMD that supports inline-proto and no API to extract that extra H/W lookup data.
Though I replied to you and explained *twice*how I think such functionality could be added if
in hypothetical future there would be a PMD that will support these features. 
But you simply ignored my replies and kept threating our patches.
Obviously, I don't think it is a proper attitude for open-source community.
At least I would expect to provide some clear explanation,
why do you think approach I outlined is not good enough.
Even better would be to: 
1) submit an RFC/patch with API you envision and PMD implementation for it.
2) submit an RFC/patch with changes in ipsec-secgw to enable these new features.
Without that in place there is not much to discuss.    
 
> For Issue #3, librte_ipsec is still experimental and is just 2-3 releases old. It has scope for improvement and all these features would've
> added value to its utility. No strong feelings here.
> 
> For Issue #4, glad that you documented that issue.
> 
> For Issue #5, in your reply you had mentioned that there is no additional check done. I see atleast few checks in the datapath. Since the
> feature itself is experimental, it would be less contentious if it was introduced as a new data path.

Once again - you keep making statements without any evidence.
Provide something more concrete here - description of packet and code flow
(function names, exact lines of code) that you think will cause a problem.

> 
> Also, to quote from the reply,
> 
> > > [Anoob] Shouldn't we get that accepted first then?
> >
> > I don't think so.
> > Current implementation does provide expected functionality (with known
> > limitations).
> > In future, we can try to improve it and/or remove existing limitations.
> > That's a common iteration development approach that is used though whole
> > DPDK:
> > - provide initial solution with basic functionality first
> > - improve/extend with future releases/patches
> 
> If your suggestion is to stick to above guidelines,

It is not my suggestion, that's how things work within DPDK right now.
Nearly every dpdk library, driver and sample app was developed based on that principle.
Even ipsec-secgw itself, just look at the git log for it. 

> I don't think there is any point in pushing any further. I leave this to maintainers to decide. I
> would assume the same rules would apply to every feature that is being attempted.

I believe that for #1, #2, #5 your comments are biased, and doesn't provide real picture.
So my suggestion to maintainers would be to ignore them as not valuable.

Konstantin 


> 
> Thanks,
> Anoob
> 
> > -----Original Message-----
> > From: dev <dev-bounces@dpdk.org> On Behalf Of Ananyev, Konstantin
> > Sent: Thursday, October 3, 2019 8:16 PM
> > To: Anoob Joseph <anoobj@marvell.com>; Smoczynski, MarcinX
> > <marcinx.smoczynski@intel.com>; akhil.goyal@nxp.com
> > Cc: dev@dpdk.org; Jerin Jacob Kollanukkaran <jerinj@marvell.com>; Narayana
> > Prasad Raju Athreya <pathreya@marvell.com>; Archana Muniganti
> > <marchana@marvell.com>
> > Subject: Re: [dpdk-dev] [PATCH v3 0/3] add fallback session
> >
> > Hi Anoob,
> >
> > > > > > > I've few more observations regarding the proposed feature.
> > > > > > >
> > > > > > > 1. From what I understood, if an ESP packet ends up on an
> > > > > > > unprotected interface and doesn't have 'PKT_RX_SEC_OFFLOAD'
> > > > > > > bit set, then the packet
> > > > > > would be looked up to see the associated SA and then fallback
> > > > > > session is figured out and then further processing is done.
> > > > > > >
> > > > > > > Can you confirm if I understood the sequence correctly? If
> > > > > > > yes, then aren't we doing an extra lookup in the s/w? The
> > > > > > > packet may be looked by the h/w using rte_flow and that
> > > > > > > information could be used to determine the
> > > > > > SA. Also, if the ESP packet is expected to be forwarded, then
> > > > > > the above logic will add an unnecessary lookup even after your
> > > > > > h/w has detected that the packet need not be security processed.
> > > > > >
> > > > > > Not sure I understood your whole statement above.
> > > > > > AFAIK, right now (with dpdk master) for unprotected iface it works like
> > that:
> > > > > >
> > > > > > 1.  slit incoming traffic into 3 groups: ESP packets, IPv4
> > > > > > packets, IPv6
> > > > packets.
> > > > > > For ESP packets:
> > > > > > 2. perform SAD lookup
> > > > > >     a. if it fails (non SA found for that SPI), then drop the packet.
> > > > > >     b. otherwise (SA found) process the packet using found SA
> > > > > >
> > > > > > What fall-back patch adds:
> > > > > > Before step 2.b check:
> > > > > > does that SA has its primary session  with type INLINE-CRYPTO
> > > > > > and does HW fail to do IPsec realted processing for it (by some reason)?
> > > > > > If yes, then mark this packet to be processed by fall-back session.
> > > > > > if (MBUF_NO_SEC_OFFLOAD(pkt) && sa->fallback_sessions > 0) {
> > > > > >                 uintptr_t intsa = (uintptr_t)sa;
> > > > > >                 intsa |= IPSEC_SA_OFFLOAD_FALLBACK_FLAG;
> > > > > >                 result_sa = (void *)intsa;  }
> > > > > >
> > > > > > So from my perspective, no additional lookup where introduced.
> > > > >
> > > > > [Anoob] For inline processing, h/w does a lookup and figures out
> > > > > the security
> > > > processing to be done on the packet.
> > > > > "rte_security_get_userdata()" allows s/w to further segregate that
> > > > > info. In this approach, I believe we don't consider how such info
> > > > > from h/w can
> > > > be used.
> > > >
> > > > Right now it is not the case with ipsec-secgw:
> > >
> > > [Anoob] Are you saying there is no h/w lookup involved when doing
> > > inline crypto processing? Then I'm confused. My understanding is that rte_flow
> > with ACTION type as SECURITY is being used to setup h/w lookups.
> >
> > I am saying that current ipsec-secgw code for each inbound ESP packet
> > *always* does SW lookup.
> > No matter did HW lookup take place already and does information about that
> > lookup is available via mbuf in some way or not.
> > Just look at the ipsec-secgw code yourself:
> >
> > nb_rx = rte_eth_rx_burst(portid, queueid,  pkts, MAX_PKT_BURST);
> >
> > if (nb_rx > 0)
> > 	process_pkts(qconf, pkts, nb_rx, portid);
> >
> > Inside process_pkts()
> > https://urldefense.proofpoint.com/v2/url?u=http-
> > 3A__lxr.dpdk.org_dpdk_latest_ident_prepare-5Fone-
> > 5Fpacket&d=DwIGaQ&c=nKjWec2b6R0mOyPaz7xtfQ&r=BPcGOOudUMrTDQ9Yb
> > gKcOkO5ChYiUPPlPNIEvTOhjNE&m=KVBgvXYlUGyMmu260erISKr73Qt6PBLux0SI
> > oSO1i-M&s=opbGGHUqfxy8p_4NvwCG6od6VdV0qN8XafaPiubfO1A&e=
> >  it first calls prepare_traffic() which does sort of de-muxing:
> > put packet in one of 3 arrays:
> >    1) ESP packets
> >    2) non ESP IPv4 packets
> >    3) non ESP IPv6 packets
> > Also it checks is PKT_RX_SEC_OFFLOAD set for that packet.
> > If yes, it retrieves associated security user-data and stores it inside private mbuf
> > area.
> >
> > Then it goes into process_pkts_inbound()
> > https://urldefense.proofpoint.com/v2/url?u=http-
> > 3A__lxr.dpdk.org_dpdk_latest_ident_process-5Fpkts-
> > 5Finbound&d=DwIGaQ&c=nKjWec2b6R0mOyPaz7xtfQ&r=BPcGOOudUMrTDQ9Y
> > bgKcOkO5ChYiUPPlPNIEvTOhjNE&m=KVBgvXYlUGyMmu260erISKr73Qt6PBLux0S
> > IoSO1i-M&s=2OxMXug7jtUQPcYiDN4nW9WbrMtRfTJx5k-N7ikOeME&e=
> > and here for all ESP packets:
> > legacy code path:
> > ipsec_inbound()->inbound_sa_lookup()->single_inbound_lookup()
> > for librte_ipsec code path:
> > inbound_sa_lookup()->single_inbound_lookup()
> >
> > single_inbound_lookup() is the one that does actual SW lookup for each input
> > packet.
> > https://urldefense.proofpoint.com/v2/url?u=http-
> > 3A__lxr.dpdk.org_dpdk_latest_ident_single-5Finbound-
> > 5Flookup&d=DwIGaQ&c=nKjWec2b6R0mOyPaz7xtfQ&r=BPcGOOudUMrTDQ9Yb
> > gKcOkO5ChYiUPPlPNIEvTOhjNE&m=KVBgvXYlUGyMmu260erISKr73Qt6PBLux0SI
> > oSO1i-M&s=D2ZPZPLbcQSbaluG8KH0F3uUDHfQWNYxw4MOc0fSkI8&e=
> >
> > >
> > > > for each inbound ESP packet it *always* does a lookup in SW based
> > > > SADB, and if lookup fails - drops the packet (see 2.a above).
> > > > So, I still not see any additional lookups introduced with the patch.
> > >
> > > [Anoob] In case of inline crypto, you could do that. No disagreement.
> > > But that doesn't mean that is the only way. If PMDs can retrieve info about
> > h/w lookups and pass it on to the upper layers, isn't that the better approach?
> >
> > Please let's keep our conversation in a constructive way.
> > We are not discussing what could be done in theory, but a particular patch for
> > ipsec-secgw:
> > Right now ipsec-secgw does a SW lookup for each inbound ESP packet (no
> > matter what HW offload does) and this patch doesn't introduce any extra SW
> > lookups.
> >
> > If you are not happy with current ipsec-secgw approach, that's fine - feel free to
> > submit RFC/patch to fix that, or start a discussion in a new thread.
> > I just don't see why we have to discuss it in context of this patch.
> >
> > >
> > > >
> > > > >
> > > > > > Also AFAIK, right now there is no possibility to configure
> > > > > > ipsec-secgw to BYPASS some ESP traffic.
> > > > > > Should we do it (to conform to ipsec RFC) - that's probably
> > > > > > subject of another discussion.
> > > > >
> > > > > [Anoob] The approach (choice of flags) forces a software-based SA
> > > > > lookup for packets that need to be bypassed instead of leveraging
> > > > > possible
> > > > hardware SA lookup. I don't think what ipsec-secgw does matters here.
> > > >
> > > > I do not agree with that statement.
> > > > First of all - current patch doesn't introduce any additional SW
> > > > lookups, see above.
> > > > Second, if someone would like to add BYPASS for ESP packets into
> > > > ipsec-secgw, I think he would have to insert new code that do de-mux
> > > > *before* any ESP related processing starts.
> > > > Most obvious variant: at prepare_one_packet() when we split our
> > > > incoming traffic to IPsec and non-IPsec ones.
> > > > Second variant - at early stage of single_inbound_lookup(), straight
> > > > after actual SAD lookup failure.
> > > > In both cases, I don't see how session selection (primary or
> > > > faillback) would affect us here, actual decision do we want to
> > > > PROCESS or BYPASS particular ESP packet needs to take place before that.
> > > > So I still believe we are ok here.
> > >
> > > [Anoob] I don't think you understood what I'm trying to highlight
> > > here. You could have a situation when h/w can detect that the packet
> > > has to be "PROTECT"ed using an "inline crypto" session. But then it
> > > might detect that it cannot proceed with inline processing because of some
> > issue (fragmentation, h/w queue overflow etc). Now the h/w has already figured
> > out the action to be done to the packet. If PMDs allow this to be communicated
> > to the application, the application won't be required to do the lookup.
> > >
> > > In inline crypto, application can ignore the h/w lookup data and do a
> > > s/w lookup with the protocol headers as they are not stripped off. It
> > > was done this way, because API and the associated means to get this
> > > info from PMD was not introduced when inline crypto and corresponding
> > support in Intel's PMD was added. But inline protocol cannot do the lookup in
> > the application and so the concept of userdata etc was introduced. The same
> > can be applied to inline crypto also. Advantage? It could remove the extra
> > lookup done in s/w.
> > >
> > > To summarize, we cannot assume that every inline crypto packet need to
> > > looked up in the application to figure out the processing done on the packet.
> > There can be applications which relies on h/w lookup data to figure out the
> > processing done.
> > >
> > > Example: Using rte_flow, I can create a rule for ESP packets to be
> > > MARKed. There is no security session created for the flow and the
> > > application intends to forward this flow packets to a different port. With your
> > approach, even these packets would be looked up. The packet will have info
> > from the h/w lookup which doesn't get used, because the approach fails to
> > introduce the required concepts.
> >
> > I think I understood your point.
> > Basically you saying that in future your PMD for unprocessed ESP packets might
> > provide some additional information (via rte_security_get_userdata) that might
> > be used by SW - let say to choose BYPASS for such packets.
> > Current ipsec-secgw doesn't support such ability.
> > Your concern is that this patch will somehow prevent (or will make it harder) for
> > you to make your future changes.
> > Correct?
> > If, so then I said before, I don't think that concern is valid.
> > Most obvious variant where to add such code is inisde inside
> > prepare_one_packet() when we split our incoming traffic to IPsec and non-IPsec
> > ones.
> > To be more specific something like that
> >
> > if (eth->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) {
> >
> >                 iph4 = (const struct rte_ipv4_hdr *)rte_pktmbuf_adj(pkt,
> >                         RTE_ETHER_HDR_LEN);
> >                 adjust_ipv4_pktlen(pkt, iph4, 0);
> >
> > -                if (iph4->next_proto_id == IPPROTO_ESP)
> > +                if (iph4->next_proto_id == IPPROTO_ESP) {
> > +		if (pkt->ol_flags & PKT_RX_SEC_OFFLOAD) {
> > +			/* extract extra info, make decision based on that */
> > +		} else
> > 			t->ipsec.pkts[(t->ipsec.num)++] = pkt;
> > +	   }
> >
> > Of course some code reordering might be needed to avoid performance drop,
> > but I suppose the main idea is clear enough.
> >  Second variant - at early stage of single_inbound_lookup(), either before or
> > straight after actual SAD SW lookup failure.
> > In both cases, this new code will be executed *before* session selection code.
> >
> > >
> > > Does the above sound like BYPASS? Yes.
> > > Does ipsec-secgw do it this way? No.
> >
> > Correct, it doesn't.
> > You have to submit patches if you'd like to support such ability.
> >
> > > Does this approach prevent an application from introducing such a BYPASS?
> > Yes.
> >
> > That's not correct, I believe, see above.
> >
> > >
> > > >
> > > > >
> > > > > For example, ESN was not supported by ipsec-secgw (before library
> > > > > mode was introduced), but every single library change and
> > > > > application change was added keeping in mind that ESN is a valid
> > > > > feature for ipsec. So it is
> > > > one thing to say ipsec-secgw doesn't support ESP bypass and saying
> > > > the solution doesn't consider a possibility of ESP bypass.
> > > > >
> > > > > >
> > > > > > >
> > > > > > > 2. The solution proposed here seems like adding the handling
> > > > > > > in ipsec-secgw instead of ipsec library. In other words, this
> > > > > > > feature is not getting
> > > > > > added in ipsec library, which was supposed to simplify the whole
> > > > > > ipsec usage in DPDK, but fails to handle the case of fragmentation.
> > > > > >
> > > > > > What we have right now with ipsec library is SA (low) level API.
> > > > > > It can handle multi-segment packets properly, but expects
> > > > > > someone else to do other steps (fragmentation/reassembly).
> > > > > > ipsec-secgw demonstrates how librte_ip_frag and librte_ipsec can
> > > > > > be used together to deal with fragmented IPsec traffic in a proper
> > manner.
> > > > > > Probably in future we'll consider adding some high-level API
> > > > > > that will be able to do whole processing under hood (SPD/SAD
> > > > > > lookup, fragment/reassembly, actual IPsec packet processing,
> > > > > > matching inbound selectors, etc.), but right now it is not the case.
> > > > > >
> > > > > > > Also, since the fallback feature is entirely done in the
> > > > > > > application, it begs the
> > > > > > question why the same feature is omitted for legacy use case.
> > > > > >
> > > > > > Because legacy mode doesn't support multi-seg packets at first place.
> > > > > > Also it is an extra overhead to support 2 code-paths (legacy and
> > > > > > library) for the same app, so we'd like in future to deprecate
> > > > > > and later remove legacy code- path.
> > > > > > As a first step we propose to make library code-path a default one:
> > > > > > https://urldefense.proofpoint.com/v2/url?u=http-3A__patches.dpdk
> > > > > > .org
> > > > > >
> > > >
> > _cover_58247_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=jPfB8rwwviRSxyL
> > > > Ws2
> > > > > > n6B-WYLn1v9SyTMrT5EQqh2TU&m=b3E429fuo8P-K5XfH8wG-
> > > > 7hwr1d8oM4uJGErAkbf
> > > > > > DvA&s=vz7ioUzJOuzoJdmV7QO0QLPKYn1ytFsj_0eYatbSCrg&e=
> > > > >
> > > > > [Anoob] Even if we need the application to do the reassembly, it
> > > > > would look simple if application uses the same "rte_ipsec_session"
> > > > > to process the
> > > > fallback packet.
> > > >
> > > > I think there is some sort of misunderstanding here.
> > > > With current librte_ipsec design:
> > > > rte_ipsec_sa - opaque SW representation of the SA (HW neutral)
> > > > rte_ipsec_session associates SA (rte_ipsec_sa)with particular HW
> > > > device (session).
> > > > Same SA can be referred by multiple sessions.
> > > >
> > > > From rte_ipsec.h:
> > > > /**
> > > >  * rte_ipsec_session is an aggregate structure that defines
> > > > particular
> > > >  * IPsec Security Association IPsec (SA) on given security/crypto device:
> > > >  * - pointer to the SA object
> > > >  * - security session action type
> > > >  * - pointer to security/crypto session, plus other related data
> > > >  * - session/device specific functions to prepare/process IPsec packets.
> > > >  */
> > > >
> > > > > In that case, the processing required to handle the other packet
> > > > > will be hidden from the application. Here application has to make
> > > > > sure it chooses
> > > > the correct session, which I believe should be moved to ipsec library.
> > > >
> > > > From my side it is a good thing to let application define its own usage model.
> > > > I.E. librte_ipsec supports multiple-sessions per SA, upper-layer
> > > > (app) decides how it will use that feature.
> > > > Though if you strongly feel that some higher-level API is needed
> > > > here, and have some good ideas about it  - please go ahead and submit RFC
> > for it.
> > >
> > > [Anoob] If you are okay with application defining how it implements ipsec,
> > then probably there is no use case for lib ipsec?
> >
> > :)
> > Once again - library provides an implementation.
> > Application defines the way to use it, i.e. how it will apply functionality library
> > provides for different use-case scenarios.
> > As an example: snpritnf() provides user with ability to format strings, application
> > decides what exactly format to use and for which string.
> > Same for librte_ipsec, library provides functionality to perform processing for
> > ESP packets.
> > Application defines which packets to process and what session to use.
> >
> > >
> > > >
> > > > >
> > > > > >
> > > > > > >
> > > > > > > 3. It seems like ordering won't be maintained once this
> > > > > > > processing is done. Again, this is the sequence I understood.
> > > > > > > Please correct me if I missed
> > > > > > something,
> > > > > > >        a. Application receives a bunch of packets (let's say 6
> > > > > > > packets), in which few are fragmented (P3 & P4) and the rest
> > > > > > > can be inline
> > > > > > processed.
> > > > > > >        b. Application receives P1->P2->P3->P4->P5->P6 (in
> > > > > > > this, P1, P2, P5, P6 are
> > > > > > inline processed successfully) and P4 & P5 are the fragments
> > > > > > >        c. Application groups packets. P1->P2->P5->P6 becomes
> > > > > > >one group and P3-
> > > > > > >P4 becomes another and goes for fallback processing.
> > > > > > > Now how is ordering maintained? I couldn't figure out how that
> > > > > > >is done in this
> > > > > > case.
> > > > > >
> > > > > > You right, fallback session can introduce packet reordering.
> > > > > > At least till we'll have ability to process packets in sync mode too.
> > > > > > See our presentation at dpdk userspace (slides 17, 18):
> > > > > > https://urldefense.proofpoint.com/v2/url?u=https-3A__static.sche
> > > > > > d.co
> > > > > > m_hosted-5Ffiles_dpdkbordeaux2019_8f_DPDK-
> > > > 2DIPSECu9.pdf&d=DwIFAg&c=n
> > > > > > KjWec2b6R0mOyPaz7xtfQ&r=jPfB8rwwviRSxyLWs2n6B-
> > > > WYLn1v9SyTMrT5EQqh2TU&
> > > > > > m=b3E429fuo8P-K5XfH8wG-
> > > > 7hwr1d8oM4uJGErAkbfDvA&s=MLRnnYcykjnsqXrHGUuR
> > > > > > YHc5uDxaAod0Yl7f06EQr1M&e= Right now the only way to deal with
> > > > > > it - have replay window big enough to sustain reordering and
> > > > > > async processing latency.
> > > > > > That's actually another reason why we add this feature into
> > > > > > ipsec-secgw sample
> > > > > > app:
> > > > > > so people can evaluate it on their platforms, determine what
> > > > > > replay window size would be needed, what issues/slowdowns it might
> > cause, etc.
> > > > >
> > > > > [Anoob] This is something I had noticed while going through the code flow.
> > > > The ordering info is lost because of the approach.
> > > >
> > > > Once again, it is a known limitation and we are not trying to hide
> > > > it from you :)
> > >
> > > [Anoob] Never said it was hidden. Initially I had okayed the approach
> > > even though it wasn't a solution suitable for inline protocol. But as more cases
> > were considered, few shortcomings were observed, and I was skeptical about
> > how any of that would be solved in the long run.
> > >
> > > My suggestion, if these limitations and the plans to address it were mentioned
> > in the cover letter, we could've saved few cycles here.
> >
> > Point taken.
> >
> > > But my reply would still be the same 😊
> > >
> > > > It
> > > > was outlined here:
> > > > https://urldefense.proofpoint.com/v2/url?u=https-
> > > > 3A__static.sched.com_hosted-5Ffiles_dpdkbordeaux2019_8f_DPDK-
> > > >
> > 2DIPSECu9.pdf&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=jPfB8rwwviRSxyL
> > > > Ws2n6B-WYLn1v9SyTMrT5EQqh2TU&m=b3E429fuo8P-K5XfH8wG-
> > > >
> > 7hwr1d8oM4uJGErAkbfDvA&s=MLRnnYcykjnsqXrHGUuRYHc5uDxaAod0Yl7f06E
> > > > Qr1M&e=
> > > > And in the latest patch version Marcin clearly stated it inside the AG session:
> > > > https://urldefense.proofpoint.com/v2/url?u=http-
> > > >
> > 3A__patches.dpdk.org_patch_60039_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtf
> > > > Q&r=jPfB8rwwviRSxyLWs2n6B-
> > WYLn1v9SyTMrT5EQqh2TU&m=b3E429fuo8P-
> > > > K5XfH8wG-
> > > >
> > 7hwr1d8oM4uJGErAkbfDvA&s=zrn91Vjf_ZElAlDf7IeZGmGevcA6RMwpPTLUCGlgZ
> > > > cY&e=
> > > > If you think even further clarification within the doc is needed,
> > > > please let us know.
> > > >
> > > > About the actual packet re-oridering:
> > > >  AFAIK, for some use-cases it might be acceptable, for others not.
> > > > Though it is an optional feature, that wouldn't be enabled by
> > > > default, so user can always choose is it does he needs/wants this feature or
> > not.
> > > > If/when we'll have CPU_CRYPTO processing mode:
> > >
> > > [Anoob] Shouldn't we get that accepted first then?
> >
> > I don't think so.
> > Current implementation does provide expected functionality (with known
> > limitations).
> > In future, we can try to improve it and/or remove existing limitations.
> > That's a common iteration development approach that is used though whole
> > DPDK:
> > - provide initial solution with basic functionality first
> > - improve/extend with future releases/patches
> >
> > >
> > > Also, the ordering is lost because application is not considering that. Why can't
> > we use re-ordering library? Or an event-dev?
> >
> > No-one saying it can't be improved in one way or another.
> > We don't plan to introduce such code right now due to its complexity.
> > Might be something in future.
> > Though you are welcome to go ahead and submit your own patches to improve
> > that solution.
> >
> > >
> > > > https://urldefense.proofpoint.com/v2/url?u=http-
> > > >
> > 3A__patches.dpdk.org_cover_58862_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtf
> > > > Q&r=jPfB8rwwviRSxyLWs2n6B-
> > WYLn1v9SyTMrT5EQqh2TU&m=b3E429fuo8P-
> > > > K5XfH8wG-
> > > >
> > 7hwr1d8oM4uJGErAkbfDvA&s=k1zef_1uqELXblKN3_qjX04WXNFGAUEFIIJrvKPr6
> > > > eA&e=
> > > > we'll add ability for the user to avoid this reordering problem
> > > >
> > > > > As we dig deeper, the feature looks hardly complete. The
> > > > > concerning part is the approach doesn't seem conducive to fixing
> > > > > any of these issues in the
> > > > future.
> > > >
> > > > Again, don't agree with that statement.
> > > > From my perspective: while current implementation has its
> > > > limitations (packet reordering, inline-proto/lksd-proto), it is
> > > > totally functional and provides users with a reference how to deal with such
> > kind of problems.
> > >
> > > [Anoob] Well, that is exactly my problem. It gives a skewed reference
> > > on how to deal with the problem, without considering other possibilities.
> >
> > Once again, please feel free and come-up with your own patches, that will
> > address this problem in a smarter way.
> >
> > > If every application starts adopting this method and starts doing
> > > lookup for every ESP packet, h/w based lookups (and the associated
> > > h/w) would be rendered useless. I really think this series, which happens to be
> > the first step in solving a complex problem, will be very useful if all these issues
> > are addressed properly.
> > >
> > > If you can introduce the inline+lookaside via librte_ipsec, its
> > > perhaps ok for the patch to have some limitations(like lack of ordering,
> > assumptions on ESP etc). However, if the patch is introduced directly into ipsec-
> > secgw, it needs to be more comprehensive and robust.
> >
> > I think common DPDK policy is exactly opposite:
> > library/driver has to be as robust and comprehensive as possible.
> > sample app code allowed to have some limitations (as long as they are
> > documented, etc.).
> >
> > > I also expect performance impact due to these changes
> >
> > Could you be more specific here?
> > What performance impact do you expect and why?
> > Can you point to the exact piece of patch code that you believe would cause a
> > degradation.
> > I don't expect any and performance tests on our platforms didn't show any
> > regression so far.
> >
> > > and so I prefer if the inline+lookaside can be made separate datapath
> > > rather than overloading the existing one.
> >
> > I don't see any good reason for that.
> > This feature is optional.
> > User have to enable it manually for each session (via config file).
> > for all other sessions there is no impact.
> > What we probably can do - add extra check inside sa.c to allow fallback sessions
> > only for inline-crypto-offload type of sessions.
> >
> > >
> > > >
> > > > >
> > > > > Also, if the new ipsec related features are introduced via
> > > > > ipsec-secgw than via rte_ipsec, then it raises doubts over the utility of
> > rte_ipsec library.
> > > >
> > > >

Re: [dpdk-dev] [PATCH v3 0/3] add fallback session

[Anoob Joseph]

Hi Konstantin,

Thanks for clarifying your stand on review comments.

From the comments we(Marvell/Cavium) have received on multiple features, my impression is that the feature need to be correct/fully done/acceptable to the overall community. Features are usually agreed upon if there is solid plan on how it will work in the long run.

Discussions with Radu on what eventually became get_userdata()
http://mails.dpdk.org/archives/dev/2017-November/082039.html

From the mail: 

"> Having something like "get_pkt_metadata" should be fine for inline 
> protocol usage. But we will still need a "get cookie" call to get the 
> cookie, as the application would need something it can interpret.
Why can't you have a get_pkt_metadata that returns the "cookie" - which 
I would call it userdata or similar? What I'm trying to say is that we 
should try to keep it as generic as possible. For example, I wouldn't 
assume that the cookie is stored in pkt->udata64 in the application."

Marvell prefers using pkt->udata64 instead of get_userdata() as it avoids an extra cache line access while trying to get hold of the SA which processed the packet. We agreed to Radu's suggestion even though Intel/Radu didn't have access to an inline protocol enabled h/w and it wasn't affecting any data path that Intel/Radu would be interested in.

Discussions with Mattias on eventmode helper
http://mails.dpdk.org/archives/dev/2019-June/134900.html

"Or even worse, the 
application's developers are forced to do a big-bang switch over to 
using the event and ethernet device APIs directly, in case they can't 
patch DPDK to work around the 
eventmode-assumption-that-didn't-hold-for-them.

You could always add flexibility to the framework (as you encounter a 
need for it), but then it will grow in complexity as well."

Here Mattias resisted the idea of eventmode helper for three reasons,

1. It's purely software, ie, no backing PMD required (which is the same case for librte_ipsec as well).
2. It "could" break for a new feature that "could" get added in eventdev.
3. He "thought" it would grow in complexity.

Marvell/Cavium has respected his opinion and did rework the entire approach which exponentially increased the effort required from our end (one code base integrated into 3 different apps vs 3 different code bases for 3 apps). Please do note that this entire "l2fwd-event" had to be redone 3 times while trying to get a consensus on the approach. I believe you had okeyed the first approach, so you cannot say we had to redo the patches because the patches were done poorly or the feature was incorrect.

I see that there is a shift in the review process these days and the vendors are encouraged to commend only on the features they are interested in. In that case, Marvell will no longer bother with patches which doesn't come in modes which doesn't matter to us. I would expect the same courtesy returned back as well.

FYI, eventmode ipsec-secgw is submitted as an RFC. In line with the comment redressal here, I've documented all the limitations.

> Once again - you keep making statements without any evidence.
> Provide something more concrete here - description of packet and code flow
> (function names, exact lines of code) that you think will cause a problem.

I saw atleast the ones below. It would be frivolous to assume that the feature is added without any data path checks.

+	if (MBUF_NO_SEC_OFFLOAD(pkt) && sa->fallback_sessions > 0) {

+			if (pg->id.val & IPSEC_SA_OFFLOAD_FALLBACK_FLAG) {

@maintainers, Marvell is not blocking this feature anymore assuming that the same yardstick will be used while reviewing patches submitted by us.

Thanks,
Anoob

> -----Original Message-----
> From: Ananyev, Konstantin <konstantin.ananyev@intel.com>
> Sent: Thursday, October 10, 2019 4:25 PM
> To: Anoob Joseph <anoobj@marvell.com>; Smoczynski, MarcinX
> <marcinx.smoczynski@intel.com>; akhil.goyal@nxp.com
> Cc: dev@dpdk.org; Jerin Jacob Kollanukkaran <jerinj@marvell.com>; Narayana
> Prasad Raju Athreya <pathreya@marvell.com>; Archana Muniganti
> <marchana@marvell.com>
> Subject: RE: [dpdk-dev] [PATCH v3 0/3] add fallback session
> 
> 
> Hi Anoob,
> 
> >
> > Following are the main issues I had identified,
> >
> > 1. Doesn't work for inline protocol offload (the plan itself won't
> > scale for that case) 2. Ignores the data from the h/w lookup 3. The
> > fallback support is actually supported in ipsec-secgw (application) instead of
> librte_ipsec.
> > 4. Reordering issue.
> > 5. Introduces checks in existing data path even when the feature is not
> properly implemented/is not suitable for other PMDs.
> >
> > For Issue #1, I get that Intel doesn't have plans to support the feature and
> won't be submitting anything in that direction.
> > But adding code that further exacerbates the situation is probably not desired.
> 
> We already discussed why that feature can't be fully supported right now by
> inline-proto/lookaside-proto devices:
> There is no API defined to sync session state between HW and SW
> https://urldefense.proofpoint.com/v2/url?u=http-
> 3A__mails.dpdk.org_archives_dev_2019-
> 2DSeptember_143881.html&d=DwIGaQ&c=nKjWec2b6R0mOyPaz7xtfQ&r=jPfB8
> rwwviRSxyLWs2n6B-
> WYLn1v9SyTMrT5EQqh2TU&m=UGz2FemapWdsohgNRq2UX5ArFEL_pLGYEVrwe
> IzaUX4&s=ObkJK9pyzOZuwoeulr6mrr6iKQF1NgXu3p2v7XwkYEs&e=
> 
> I already outlined that gap nearly a year ago:
> https://urldefense.proofpoint.com/v2/url?u=http-
> 3A__mails.dpdk.org_archives_dev_2018-
> 2DSeptember_113600.html&d=DwIGaQ&c=nKjWec2b6R0mOyPaz7xtfQ&r=jPfB8
> rwwviRSxyLWs2n6B-
> WYLn1v9SyTMrT5EQqh2TU&m=UGz2FemapWdsohgNRq2UX5ArFEL_pLGYEVrwe
> IzaUX4&s=0x8pmfsS7nT7Xn-a_RvhI-DxtHMdk0zr4jiapi22tVE&e=
> Though so far I didn't see any attempt to resolve this problem by Marvell (or
> NXP) engineers.
> Which makes me think that either you guys don't consider it as a problem at all,
> or have no desire to work on it.
> I never insisted, as I don't  think it is my call by obvious reasons:
> 1) right now Intel doesn't have HW that provides inline-proto/lksd-proto
> capabilities.
> 2) Intel DPDK team doesn't have access to such HW.
> 3) There is no PMD inside dpdk.org that implements inline-proto anyway.
> 
> Now, one year later we are in a situation where:
> 1) there still no PMD at dpdk.org that supports inline-proto.
> 2) API to sync session state between HW and SW still not defined/implemented.
> 3) you trying to accuse me and other Intel DPDK team for lack of interest to
> work on that problem for you
>    and threat our patches because of that.
> 
> That just doesn't look fair to me.
> You shouldn't expect other people patches to provide 'complete' solution for
> inline-proto, if (in a year time) you failed to define/implement an API for that.
> 
> > For Issue #2, since ipsec-secgw is not utilizing the h/w lookup data,
> > the solution here follows suit and is preventing other usage models of the
> hardware which are allowed by the library.
> 
> Same as above, right now there is no PMD that supports inline-proto and no API
> to extract that extra H/W lookup data.
> Though I replied to you and explained *twice*how I think such functionality
> could be added if in hypothetical future there would be a PMD that will support
> these features.
> But you simply ignored my replies and kept threating our patches.
> Obviously, I don't think it is a proper attitude for open-source community.
> At least I would expect to provide some clear explanation, why do you think
> approach I outlined is not good enough.
> Even better would be to:
> 1) submit an RFC/patch with API you envision and PMD implementation for it.
> 2) submit an RFC/patch with changes in ipsec-secgw to enable these new
> features.
> Without that in place there is not much to discuss.
> 
> > For Issue #3, librte_ipsec is still experimental and is just 2-3
> > releases old. It has scope for improvement and all these features would've
> added value to its utility. No strong feelings here.
> >
> > For Issue #4, glad that you documented that issue.
> >
> > For Issue #5, in your reply you had mentioned that there is no
> > additional check done. I see atleast few checks in the datapath. Since the
> feature itself is experimental, it would be less contentious if it was introduced as
> a new data path.
> 
> Once again - you keep making statements without any evidence.
> Provide something more concrete here - description of packet and code flow
> (function names, exact lines of code) that you think will cause a problem.
> 
> >
> > Also, to quote from the reply,
> >
> > > > [Anoob] Shouldn't we get that accepted first then?
> > >
> > > I don't think so.
> > > Current implementation does provide expected functionality (with
> > > known limitations).
> > > In future, we can try to improve it and/or remove existing limitations.
> > > That's a common iteration development approach that is used though
> > > whole
> > > DPDK:
> > > - provide initial solution with basic functionality first
> > > - improve/extend with future releases/patches
> >
> > If your suggestion is to stick to above guidelines,
> 
> It is not my suggestion, that's how things work within DPDK right now.
> Nearly every dpdk library, driver and sample app was developed based on that
> principle.
> Even ipsec-secgw itself, just look at the git log for it.
> 
> > I don't think there is any point in pushing any further. I leave this
> > to maintainers to decide. I would assume the same rules would apply to every
> feature that is being attempted.
> 
> I believe that for #1, #2, #5 your comments are biased, and doesn't provide real
> picture.
> So my suggestion to maintainers would be to ignore them as not valuable.
> 
> Konstantin
> 
> 
> >
> > Thanks,
> > Anoob
> >
> > > -----Original Message-----
> > > From: dev <dev-bounces@dpdk.org> On Behalf Of Ananyev, Konstantin
> > > Sent: Thursday, October 3, 2019 8:16 PM
> > > To: Anoob Joseph <anoobj@marvell.com>; Smoczynski, MarcinX
> > > <marcinx.smoczynski@intel.com>; akhil.goyal@nxp.com
> > > Cc: dev@dpdk.org; Jerin Jacob Kollanukkaran <jerinj@marvell.com>;
> > > Narayana Prasad Raju Athreya <pathreya@marvell.com>; Archana
> > > Muniganti <marchana@marvell.com>
> > > Subject: Re: [dpdk-dev] [PATCH v3 0/3] add fallback session
> > >
> > > Hi Anoob,
> > >
> > > > > > > > I've few more observations regarding the proposed feature.
> > > > > > > >
> > > > > > > > 1. From what I understood, if an ESP packet ends up on an
> > > > > > > > unprotected interface and doesn't have 'PKT_RX_SEC_OFFLOAD'
> > > > > > > > bit set, then the packet
> > > > > > > would be looked up to see the associated SA and then
> > > > > > > fallback session is figured out and then further processing is done.
> > > > > > > >
> > > > > > > > Can you confirm if I understood the sequence correctly? If
> > > > > > > > yes, then aren't we doing an extra lookup in the s/w? The
> > > > > > > > packet may be looked by the h/w using rte_flow and that
> > > > > > > > information could be used to determine the
> > > > > > > SA. Also, if the ESP packet is expected to be forwarded,
> > > > > > > then the above logic will add an unnecessary lookup even
> > > > > > > after your h/w has detected that the packet need not be security
> processed.
> > > > > > >
> > > > > > > Not sure I understood your whole statement above.
> > > > > > > AFAIK, right now (with dpdk master) for unprotected iface it
> > > > > > > works like
> > > that:
> > > > > > >
> > > > > > > 1.  slit incoming traffic into 3 groups: ESP packets, IPv4
> > > > > > > packets, IPv6
> > > > > packets.
> > > > > > > For ESP packets:
> > > > > > > 2. perform SAD lookup
> > > > > > >     a. if it fails (non SA found for that SPI), then drop the packet.
> > > > > > >     b. otherwise (SA found) process the packet using found
> > > > > > > SA
> > > > > > >
> > > > > > > What fall-back patch adds:
> > > > > > > Before step 2.b check:
> > > > > > > does that SA has its primary session  with type
> > > > > > > INLINE-CRYPTO and does HW fail to do IPsec realted processing for it
> (by some reason)?
> > > > > > > If yes, then mark this packet to be processed by fall-back session.
> > > > > > > if (MBUF_NO_SEC_OFFLOAD(pkt) && sa->fallback_sessions > 0) {
> > > > > > >                 uintptr_t intsa = (uintptr_t)sa;
> > > > > > >                 intsa |= IPSEC_SA_OFFLOAD_FALLBACK_FLAG;
> > > > > > >                 result_sa = (void *)intsa;  }
> > > > > > >
> > > > > > > So from my perspective, no additional lookup where introduced.
> > > > > >
> > > > > > [Anoob] For inline processing, h/w does a lookup and figures
> > > > > > out the security
> > > > > processing to be done on the packet.
> > > > > > "rte_security_get_userdata()" allows s/w to further segregate
> > > > > > that info. In this approach, I believe we don't consider how
> > > > > > such info from h/w can
> > > > > be used.
> > > > >
> > > > > Right now it is not the case with ipsec-secgw:
> > > >
> > > > [Anoob] Are you saying there is no h/w lookup involved when doing
> > > > inline crypto processing? Then I'm confused. My understanding is
> > > > that rte_flow
> > > with ACTION type as SECURITY is being used to setup h/w lookups.
> > >
> > > I am saying that current ipsec-secgw code for each inbound ESP
> > > packet
> > > *always* does SW lookup.
> > > No matter did HW lookup take place already and does information
> > > about that lookup is available via mbuf in some way or not.
> > > Just look at the ipsec-secgw code yourself:
> > >
> > > nb_rx = rte_eth_rx_burst(portid, queueid,  pkts, MAX_PKT_BURST);
> > >
> > > if (nb_rx > 0)
> > > 	process_pkts(qconf, pkts, nb_rx, portid);
> > >
> > > Inside process_pkts()
> > > https://urldefense.proofpoint.com/v2/url?u=http-
> > > 3A__lxr.dpdk.org_dpdk_latest_ident_prepare-5Fone-
> > >
> 5Fpacket&d=DwIGaQ&c=nKjWec2b6R0mOyPaz7xtfQ&r=BPcGOOudUMrTDQ9Yb
> > >
> gKcOkO5ChYiUPPlPNIEvTOhjNE&m=KVBgvXYlUGyMmu260erISKr73Qt6PBLux0SI
> > > oSO1i-M&s=opbGGHUqfxy8p_4NvwCG6od6VdV0qN8XafaPiubfO1A&e=
> > >  it first calls prepare_traffic() which does sort of de-muxing:
> > > put packet in one of 3 arrays:
> > >    1) ESP packets
> > >    2) non ESP IPv4 packets
> > >    3) non ESP IPv6 packets
> > > Also it checks is PKT_RX_SEC_OFFLOAD set for that packet.
> > > If yes, it retrieves associated security user-data and stores it
> > > inside private mbuf area.
> > >
> > > Then it goes into process_pkts_inbound()
> > > https://urldefense.proofpoint.com/v2/url?u=http-
> > > 3A__lxr.dpdk.org_dpdk_latest_ident_process-5Fpkts-
> > >
> 5Finbound&d=DwIGaQ&c=nKjWec2b6R0mOyPaz7xtfQ&r=BPcGOOudUMrTDQ9Y
> > >
> bgKcOkO5ChYiUPPlPNIEvTOhjNE&m=KVBgvXYlUGyMmu260erISKr73Qt6PBLux0S
> > > IoSO1i-M&s=2OxMXug7jtUQPcYiDN4nW9WbrMtRfTJx5k-N7ikOeME&e=
> > > and here for all ESP packets:
> > > legacy code path:
> > > ipsec_inbound()->inbound_sa_lookup()->single_inbound_lookup()
> > > for librte_ipsec code path:
> > > inbound_sa_lookup()->single_inbound_lookup()
> > >
> > > single_inbound_lookup() is the one that does actual SW lookup for
> > > each input packet.
> > > https://urldefense.proofpoint.com/v2/url?u=http-
> > > 3A__lxr.dpdk.org_dpdk_latest_ident_single-5Finbound-
> > >
> 5Flookup&d=DwIGaQ&c=nKjWec2b6R0mOyPaz7xtfQ&r=BPcGOOudUMrTDQ9Yb
> > >
> gKcOkO5ChYiUPPlPNIEvTOhjNE&m=KVBgvXYlUGyMmu260erISKr73Qt6PBLux0SI
> > > oSO1i-M&s=D2ZPZPLbcQSbaluG8KH0F3uUDHfQWNYxw4MOc0fSkI8&e=
> > >
> > > >
> > > > > for each inbound ESP packet it *always* does a lookup in SW
> > > > > based SADB, and if lookup fails - drops the packet (see 2.a above).
> > > > > So, I still not see any additional lookups introduced with the patch.
> > > >
> > > > [Anoob] In case of inline crypto, you could do that. No disagreement.
> > > > But that doesn't mean that is the only way. If PMDs can retrieve
> > > > info about
> > > h/w lookups and pass it on to the upper layers, isn't that the better
> approach?
> > >
> > > Please let's keep our conversation in a constructive way.
> > > We are not discussing what could be done in theory, but a particular
> > > patch for
> > > ipsec-secgw:
> > > Right now ipsec-secgw does a SW lookup for each inbound ESP packet
> > > (no matter what HW offload does) and this patch doesn't introduce
> > > any extra SW lookups.
> > >
> > > If you are not happy with current ipsec-secgw approach, that's fine
> > > - feel free to submit RFC/patch to fix that, or start a discussion in a new
> thread.
> > > I just don't see why we have to discuss it in context of this patch.
> > >
> > > >
> > > > >
> > > > > >
> > > > > > > Also AFAIK, right now there is no possibility to configure
> > > > > > > ipsec-secgw to BYPASS some ESP traffic.
> > > > > > > Should we do it (to conform to ipsec RFC) - that's probably
> > > > > > > subject of another discussion.
> > > > > >
> > > > > > [Anoob] The approach (choice of flags) forces a software-based
> > > > > > SA lookup for packets that need to be bypassed instead of
> > > > > > leveraging possible
> > > > > hardware SA lookup. I don't think what ipsec-secgw does matters here.
> > > > >
> > > > > I do not agree with that statement.
> > > > > First of all - current patch doesn't introduce any additional SW
> > > > > lookups, see above.
> > > > > Second, if someone would like to add BYPASS for ESP packets into
> > > > > ipsec-secgw, I think he would have to insert new code that do
> > > > > de-mux
> > > > > *before* any ESP related processing starts.
> > > > > Most obvious variant: at prepare_one_packet() when we split our
> > > > > incoming traffic to IPsec and non-IPsec ones.
> > > > > Second variant - at early stage of single_inbound_lookup(),
> > > > > straight after actual SAD lookup failure.
> > > > > In both cases, I don't see how session selection (primary or
> > > > > faillback) would affect us here, actual decision do we want to
> > > > > PROCESS or BYPASS particular ESP packet needs to take place before
> that.
> > > > > So I still believe we are ok here.
> > > >
> > > > [Anoob] I don't think you understood what I'm trying to highlight
> > > > here. You could have a situation when h/w can detect that the
> > > > packet has to be "PROTECT"ed using an "inline crypto" session. But
> > > > then it might detect that it cannot proceed with inline processing
> > > > because of some
> > > issue (fragmentation, h/w queue overflow etc). Now the h/w has
> > > already figured out the action to be done to the packet. If PMDs
> > > allow this to be communicated to the application, the application won't be
> required to do the lookup.
> > > >
> > > > In inline crypto, application can ignore the h/w lookup data and
> > > > do a s/w lookup with the protocol headers as they are not stripped
> > > > off. It was done this way, because API and the associated means to
> > > > get this info from PMD was not introduced when inline crypto and
> > > > corresponding
> > > support in Intel's PMD was added. But inline protocol cannot do the
> > > lookup in the application and so the concept of userdata etc was
> > > introduced. The same can be applied to inline crypto also.
> > > Advantage? It could remove the extra lookup done in s/w.
> > > >
> > > > To summarize, we cannot assume that every inline crypto packet
> > > > need to looked up in the application to figure out the processing done on
> the packet.
> > > There can be applications which relies on h/w lookup data to figure
> > > out the processing done.
> > > >
> > > > Example: Using rte_flow, I can create a rule for ESP packets to be
> > > > MARKed. There is no security session created for the flow and the
> > > > application intends to forward this flow packets to a different
> > > > port. With your
> > > approach, even these packets would be looked up. The packet will
> > > have info from the h/w lookup which doesn't get used, because the
> > > approach fails to introduce the required concepts.
> > >
> > > I think I understood your point.
> > > Basically you saying that in future your PMD for unprocessed ESP
> > > packets might provide some additional information (via
> > > rte_security_get_userdata) that might be used by SW - let say to choose
> BYPASS for such packets.
> > > Current ipsec-secgw doesn't support such ability.
> > > Your concern is that this patch will somehow prevent (or will make
> > > it harder) for you to make your future changes.
> > > Correct?
> > > If, so then I said before, I don't think that concern is valid.
> > > Most obvious variant where to add such code is inisde inside
> > > prepare_one_packet() when we split our incoming traffic to IPsec and
> > > non-IPsec ones.
> > > To be more specific something like that
> > >
> > > if (eth->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) {
> > >
> > >                 iph4 = (const struct rte_ipv4_hdr *)rte_pktmbuf_adj(pkt,
> > >                         RTE_ETHER_HDR_LEN);
> > >                 adjust_ipv4_pktlen(pkt, iph4, 0);
> > >
> > > -                if (iph4->next_proto_id == IPPROTO_ESP)
> > > +                if (iph4->next_proto_id == IPPROTO_ESP) {
> > > +		if (pkt->ol_flags & PKT_RX_SEC_OFFLOAD) {
> > > +			/* extract extra info, make decision based on that */
> > > +		} else
> > > 			t->ipsec.pkts[(t->ipsec.num)++] = pkt;
> > > +	   }
> > >
> > > Of course some code reordering might be needed to avoid performance
> > > drop, but I suppose the main idea is clear enough.
> > >  Second variant - at early stage of single_inbound_lookup(), either
> > > before or straight after actual SAD SW lookup failure.
> > > In both cases, this new code will be executed *before* session selection
> code.
> > >
> > > >
> > > > Does the above sound like BYPASS? Yes.
> > > > Does ipsec-secgw do it this way? No.
> > >
> > > Correct, it doesn't.
> > > You have to submit patches if you'd like to support such ability.
> > >
> > > > Does this approach prevent an application from introducing such a
> BYPASS?
> > > Yes.
> > >
> > > That's not correct, I believe, see above.
> > >
> > > >
> > > > >
> > > > > >
> > > > > > For example, ESN was not supported by ipsec-secgw (before
> > > > > > library mode was introduced), but every single library change
> > > > > > and application change was added keeping in mind that ESN is a
> > > > > > valid feature for ipsec. So it is
> > > > > one thing to say ipsec-secgw doesn't support ESP bypass and
> > > > > saying the solution doesn't consider a possibility of ESP bypass.
> > > > > >
> > > > > > >
> > > > > > > >
> > > > > > > > 2. The solution proposed here seems like adding the
> > > > > > > > handling in ipsec-secgw instead of ipsec library. In other
> > > > > > > > words, this feature is not getting
> > > > > > > added in ipsec library, which was supposed to simplify the
> > > > > > > whole ipsec usage in DPDK, but fails to handle the case of
> fragmentation.
> > > > > > >
> > > > > > > What we have right now with ipsec library is SA (low) level API.
> > > > > > > It can handle multi-segment packets properly, but expects
> > > > > > > someone else to do other steps (fragmentation/reassembly).
> > > > > > > ipsec-secgw demonstrates how librte_ip_frag and librte_ipsec
> > > > > > > can be used together to deal with fragmented IPsec traffic
> > > > > > > in a proper
> > > manner.
> > > > > > > Probably in future we'll consider adding some high-level API
> > > > > > > that will be able to do whole processing under hood (SPD/SAD
> > > > > > > lookup, fragment/reassembly, actual IPsec packet processing,
> > > > > > > matching inbound selectors, etc.), but right now it is not the case.
> > > > > > >
> > > > > > > > Also, since the fallback feature is entirely done in the
> > > > > > > > application, it begs the
> > > > > > > question why the same feature is omitted for legacy use case.
> > > > > > >
> > > > > > > Because legacy mode doesn't support multi-seg packets at first place.
> > > > > > > Also it is an extra overhead to support 2 code-paths (legacy
> > > > > > > and
> > > > > > > library) for the same app, so we'd like in future to
> > > > > > > deprecate and later remove legacy code- path.
> > > > > > > As a first step we propose to make library code-path a default one:
> > > > > > > https://urldefense.proofpoint.com/v2/url?u=http-3A__patches.
> > > > > > > dpdk
> > > > > > > .org
> > > > > > >
> > > > >
> > >
> _cover_58247_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=jPfB8rwwviRSxyL
> > > > > Ws2
> > > > > > > n6B-WYLn1v9SyTMrT5EQqh2TU&m=b3E429fuo8P-K5XfH8wG-
> > > > > 7hwr1d8oM4uJGErAkbf
> > > > > > > DvA&s=vz7ioUzJOuzoJdmV7QO0QLPKYn1ytFsj_0eYatbSCrg&e=
> > > > > >
> > > > > > [Anoob] Even if we need the application to do the reassembly,
> > > > > > it would look simple if application uses the same "rte_ipsec_session"
> > > > > > to process the
> > > > > fallback packet.
> > > > >
> > > > > I think there is some sort of misunderstanding here.
> > > > > With current librte_ipsec design:
> > > > > rte_ipsec_sa - opaque SW representation of the SA (HW neutral)
> > > > > rte_ipsec_session associates SA (rte_ipsec_sa)with particular HW
> > > > > device (session).
> > > > > Same SA can be referred by multiple sessions.
> > > > >
> > > > > From rte_ipsec.h:
> > > > > /**
> > > > >  * rte_ipsec_session is an aggregate structure that defines
> > > > > particular
> > > > >  * IPsec Security Association IPsec (SA) on given security/crypto device:
> > > > >  * - pointer to the SA object
> > > > >  * - security session action type
> > > > >  * - pointer to security/crypto session, plus other related data
> > > > >  * - session/device specific functions to prepare/process IPsec packets.
> > > > >  */
> > > > >
> > > > > > In that case, the processing required to handle the other
> > > > > > packet will be hidden from the application. Here application
> > > > > > has to make sure it chooses
> > > > > the correct session, which I believe should be moved to ipsec library.
> > > > >
> > > > > From my side it is a good thing to let application define its own usage
> model.
> > > > > I.E. librte_ipsec supports multiple-sessions per SA, upper-layer
> > > > > (app) decides how it will use that feature.
> > > > > Though if you strongly feel that some higher-level API is needed
> > > > > here, and have some good ideas about it  - please go ahead and
> > > > > submit RFC
> > > for it.
> > > >
> > > > [Anoob] If you are okay with application defining how it
> > > > implements ipsec,
> > > then probably there is no use case for lib ipsec?
> > >
> > > :)
> > > Once again - library provides an implementation.
> > > Application defines the way to use it, i.e. how it will apply
> > > functionality library provides for different use-case scenarios.
> > > As an example: snpritnf() provides user with ability to format
> > > strings, application decides what exactly format to use and for which string.
> > > Same for librte_ipsec, library provides functionality to perform
> > > processing for ESP packets.
> > > Application defines which packets to process and what session to use.
> > >
> > > >
> > > > >
> > > > > >
> > > > > > >
> > > > > > > >
> > > > > > > > 3. It seems like ordering won't be maintained once this
> > > > > > > > processing is done. Again, this is the sequence I understood.
> > > > > > > > Please correct me if I missed
> > > > > > > something,
> > > > > > > >        a. Application receives a bunch of packets (let's
> > > > > > > > say 6 packets), in which few are fragmented (P3 & P4) and
> > > > > > > > the rest can be inline
> > > > > > > processed.
> > > > > > > >        b. Application receives P1->P2->P3->P4->P5->P6 (in
> > > > > > > > this, P1, P2, P5, P6 are
> > > > > > > inline processed successfully) and P4 & P5 are the fragments
> > > > > > > >        c. Application groups packets. P1->P2->P5->P6
> > > > > > > >becomes one group and P3-
> > > > > > > >P4 becomes another and goes for fallback processing.
> > > > > > > > Now how is ordering maintained? I couldn't figure out how
> > > > > > > >that is done in this
> > > > > > > case.
> > > > > > >
> > > > > > > You right, fallback session can introduce packet reordering.
> > > > > > > At least till we'll have ability to process packets in sync mode too.
> > > > > > > See our presentation at dpdk userspace (slides 17, 18):
> > > > > > > https://urldefense.proofpoint.com/v2/url?u=https-3A__static.
> > > > > > > sche
> > > > > > > d.co
> > > > > > > m_hosted-5Ffiles_dpdkbordeaux2019_8f_DPDK-
> > > > > 2DIPSECu9.pdf&d=DwIFAg&c=n
> > > > > > > KjWec2b6R0mOyPaz7xtfQ&r=jPfB8rwwviRSxyLWs2n6B-
> > > > > WYLn1v9SyTMrT5EQqh2TU&
> > > > > > > m=b3E429fuo8P-K5XfH8wG-
> > > > > 7hwr1d8oM4uJGErAkbfDvA&s=MLRnnYcykjnsqXrHGUuR
> > > > > > > YHc5uDxaAod0Yl7f06EQr1M&e= Right now the only way to deal
> > > > > > > with it - have replay window big enough to sustain
> > > > > > > reordering and async processing latency.
> > > > > > > That's actually another reason why we add this feature into
> > > > > > > ipsec-secgw sample
> > > > > > > app:
> > > > > > > so people can evaluate it on their platforms, determine what
> > > > > > > replay window size would be needed, what issues/slowdowns it
> > > > > > > might
> > > cause, etc.
> > > > > >
> > > > > > [Anoob] This is something I had noticed while going through the code
> flow.
> > > > > The ordering info is lost because of the approach.
> > > > >
> > > > > Once again, it is a known limitation and we are not trying to
> > > > > hide it from you :)
> > > >
> > > > [Anoob] Never said it was hidden. Initially I had okayed the
> > > > approach even though it wasn't a solution suitable for inline
> > > > protocol. But as more cases
> > > were considered, few shortcomings were observed, and I was skeptical
> > > about how any of that would be solved in the long run.
> > > >
> > > > My suggestion, if these limitations and the plans to address it
> > > > were mentioned
> > > in the cover letter, we could've saved few cycles here.
> > >
> > > Point taken.
> > >
> > > > But my reply would still be the same 😊
> > > >
> > > > > It
> > > > > was outlined here:
> > > > > https://urldefense.proofpoint.com/v2/url?u=https-
> > > > > 3A__static.sched.com_hosted-5Ffiles_dpdkbordeaux2019_8f_DPDK-
> > > > >
> > >
> 2DIPSECu9.pdf&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=jPfB8rwwviRSxyL
> > > > > Ws2n6B-WYLn1v9SyTMrT5EQqh2TU&m=b3E429fuo8P-K5XfH8wG-
> > > > >
> > >
> 7hwr1d8oM4uJGErAkbfDvA&s=MLRnnYcykjnsqXrHGUuRYHc5uDxaAod0Yl7f06E
> > > > > Qr1M&e=
> > > > > And in the latest patch version Marcin clearly stated it inside the AG
> session:
> > > > > https://urldefense.proofpoint.com/v2/url?u=http-
> > > > >
> > >
> 3A__patches.dpdk.org_patch_60039_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtf
> > > > > Q&r=jPfB8rwwviRSxyLWs2n6B-
> > > WYLn1v9SyTMrT5EQqh2TU&m=b3E429fuo8P-
> > > > > K5XfH8wG-
> > > > >
> > >
> 7hwr1d8oM4uJGErAkbfDvA&s=zrn91Vjf_ZElAlDf7IeZGmGevcA6RMwpPTLUCGlgZ
> > > > > cY&e=
> > > > > If you think even further clarification within the doc is
> > > > > needed, please let us know.
> > > > >
> > > > > About the actual packet re-oridering:
> > > > >  AFAIK, for some use-cases it might be acceptable, for others not.
> > > > > Though it is an optional feature, that wouldn't be enabled by
> > > > > default, so user can always choose is it does he needs/wants
> > > > > this feature or
> > > not.
> > > > > If/when we'll have CPU_CRYPTO processing mode:
> > > >
> > > > [Anoob] Shouldn't we get that accepted first then?
> > >
> > > I don't think so.
> > > Current implementation does provide expected functionality (with
> > > known limitations).
> > > In future, we can try to improve it and/or remove existing limitations.
> > > That's a common iteration development approach that is used though
> > > whole
> > > DPDK:
> > > - provide initial solution with basic functionality first
> > > - improve/extend with future releases/patches
> > >
> > > >
> > > > Also, the ordering is lost because application is not considering
> > > > that. Why can't
> > > we use re-ordering library? Or an event-dev?
> > >
> > > No-one saying it can't be improved in one way or another.
> > > We don't plan to introduce such code right now due to its complexity.
> > > Might be something in future.
> > > Though you are welcome to go ahead and submit your own patches to
> > > improve that solution.
> > >
> > > >
> > > > > https://urldefense.proofpoint.com/v2/url?u=http-
> > > > >
> > >
> 3A__patches.dpdk.org_cover_58862_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtf
> > > > > Q&r=jPfB8rwwviRSxyLWs2n6B-
> > > WYLn1v9SyTMrT5EQqh2TU&m=b3E429fuo8P-
> > > > > K5XfH8wG-
> > > > >
> > >
> 7hwr1d8oM4uJGErAkbfDvA&s=k1zef_1uqELXblKN3_qjX04WXNFGAUEFIIJrvKPr6
> > > > > eA&e=
> > > > > we'll add ability for the user to avoid this reordering problem
> > > > >
> > > > > > As we dig deeper, the feature looks hardly complete. The
> > > > > > concerning part is the approach doesn't seem conducive to
> > > > > > fixing any of these issues in the
> > > > > future.
> > > > >
> > > > > Again, don't agree with that statement.
> > > > > From my perspective: while current implementation has its
> > > > > limitations (packet reordering, inline-proto/lksd-proto), it is
> > > > > totally functional and provides users with a reference how to
> > > > > deal with such
> > > kind of problems.
> > > >
> > > > [Anoob] Well, that is exactly my problem. It gives a skewed
> > > > reference on how to deal with the problem, without considering other
> possibilities.
> > >
> > > Once again, please feel free and come-up with your own patches, that
> > > will address this problem in a smarter way.
> > >
> > > > If every application starts adopting this method and starts doing
> > > > lookup for every ESP packet, h/w based lookups (and the associated
> > > > h/w) would be rendered useless. I really think this series, which
> > > > happens to be
> > > the first step in solving a complex problem, will be very useful if
> > > all these issues are addressed properly.
> > > >
> > > > If you can introduce the inline+lookaside via librte_ipsec, its
> > > > perhaps ok for the patch to have some limitations(like lack of
> > > > ordering,
> > > assumptions on ESP etc). However, if the patch is introduced
> > > directly into ipsec- secgw, it needs to be more comprehensive and robust.
> > >
> > > I think common DPDK policy is exactly opposite:
> > > library/driver has to be as robust and comprehensive as possible.
> > > sample app code allowed to have some limitations (as long as they
> > > are documented, etc.).
> > >
> > > > I also expect performance impact due to these changes
> > >
> > > Could you be more specific here?
> > > What performance impact do you expect and why?
> > > Can you point to the exact piece of patch code that you believe
> > > would cause a degradation.
> > > I don't expect any and performance tests on our platforms didn't
> > > show any regression so far.
> > >
> > > > and so I prefer if the inline+lookaside can be made separate
> > > > datapath rather than overloading the existing one.
> > >
> > > I don't see any good reason for that.
> > > This feature is optional.
> > > User have to enable it manually for each session (via config file).
> > > for all other sessions there is no impact.
> > > What we probably can do - add extra check inside sa.c to allow
> > > fallback sessions only for inline-crypto-offload type of sessions.
> > >
> > > >
> > > > >
> > > > > >
> > > > > > Also, if the new ipsec related features are introduced via
> > > > > > ipsec-secgw than via rte_ipsec, then it raises doubts over the
> > > > > > utility of
> > > rte_ipsec library.
> > > > >
> > > > >

Re: [dpdk-dev] [PATCH v3 0/3] add fallback session

[Ananyev, Konstantin]

> > Once again - you keep making statements without any evidence.
> > Provide something more concrete here - description of packet and code flow
> > (function names, exact lines of code) that you think will cause a problem.
> 
> I saw atleast the ones below. It would be frivolous to assume that the feature is added without any data path checks.
> 
> +	if (MBUF_NO_SEC_OFFLOAD(pkt) && sa->fallback_sessions > 0) {


Ok here we have extra 2 comparsions for data (pkt->ol_flags and sa->fallback_sessions),
that already would be in the cache and .
For common path both will be evaluated to 0, we didn't observe any perf difference because of that.
I would expect the same for other archs.
If you would see something on your boxes - let us know.
We could think what we can do here...
Probably at least swap the comparisons or so... 
But honestly, I don't expect any observable slowdown.

> 
> +			if (pg->id.val & IPSEC_SA_OFFLOAD_FALLBACK_FLAG) {



> 
> @maintainers, Marvell is not blocking this feature anymore assuming that the same yardstick will be used while reviewing patches
> submitted by us.
> 
> Thanks,
> Anoob
> 
> > -----Original Message-----
> > From: Ananyev, Konstantin <konstantin.ananyev@intel.com>
> > Sent: Thursday, October 10, 2019 4:25 PM
> > To: Anoob Joseph <anoobj@marvell.com>; Smoczynski, MarcinX
> > <marcinx.smoczynski@intel.com>; akhil.goyal@nxp.com
> > Cc: dev@dpdk.org; Jerin Jacob Kollanukkaran <jerinj@marvell.com>; Narayana
> > Prasad Raju Athreya <pathreya@marvell.com>; Archana Muniganti
> > <marchana@marvell.com>
> > Subject: RE: [dpdk-dev] [PATCH v3 0/3] add fallback session
> >
> >
> > Hi Anoob,
> >
> > >
> > > Following are the main issues I had identified,
> > >
> > > 1. Doesn't work for inline protocol offload (the plan itself won't
> > > scale for that case) 2. Ignores the data from the h/w lookup 3. The
> > > fallback support is actually supported in ipsec-secgw (application) instead of
> > librte_ipsec.
> > > 4. Reordering issue.
> > > 5. Introduces checks in existing data path even when the feature is not
> > properly implemented/is not suitable for other PMDs.
> > >
> > > For Issue #1, I get that Intel doesn't have plans to support the feature and
> > won't be submitting anything in that direction.
> > > But adding code that further exacerbates the situation is probably not desired.
> >
> > We already discussed why that feature can't be fully supported right now by
> > inline-proto/lookaside-proto devices:
> > There is no API defined to sync session state between HW and SW
> > https://urldefense.proofpoint.com/v2/url?u=http-
> > 3A__mails.dpdk.org_archives_dev_2019-
> > 2DSeptember_143881.html&d=DwIGaQ&c=nKjWec2b6R0mOyPaz7xtfQ&r=jPfB8
> > rwwviRSxyLWs2n6B-
> > WYLn1v9SyTMrT5EQqh2TU&m=UGz2FemapWdsohgNRq2UX5ArFEL_pLGYEVrwe
> > IzaUX4&s=ObkJK9pyzOZuwoeulr6mrr6iKQF1NgXu3p2v7XwkYEs&e=
> >
> > I already outlined that gap nearly a year ago:
> > https://urldefense.proofpoint.com/v2/url?u=http-
> > 3A__mails.dpdk.org_archives_dev_2018-
> > 2DSeptember_113600.html&d=DwIGaQ&c=nKjWec2b6R0mOyPaz7xtfQ&r=jPfB8
> > rwwviRSxyLWs2n6B-
> > WYLn1v9SyTMrT5EQqh2TU&m=UGz2FemapWdsohgNRq2UX5ArFEL_pLGYEVrwe
> > IzaUX4&s=0x8pmfsS7nT7Xn-a_RvhI-DxtHMdk0zr4jiapi22tVE&e=
> > Though so far I didn't see any attempt to resolve this problem by Marvell (or
> > NXP) engineers.
> > Which makes me think that either you guys don't consider it as a problem at all,
> > or have no desire to work on it.
> > I never insisted, as I don't  think it is my call by obvious reasons:
> > 1) right now Intel doesn't have HW that provides inline-proto/lksd-proto
> > capabilities.
> > 2) Intel DPDK team doesn't have access to such HW.
> > 3) There is no PMD inside dpdk.org that implements inline-proto anyway.
> >
> > Now, one year later we are in a situation where:
> > 1) there still no PMD at dpdk.org that supports inline-proto.
> > 2) API to sync session state between HW and SW still not defined/implemented.
> > 3) you trying to accuse me and other Intel DPDK team for lack of interest to
> > work on that problem for you
> >    and threat our patches because of that.
> >
> > That just doesn't look fair to me.
> > You shouldn't expect other people patches to provide 'complete' solution for
> > inline-proto, if (in a year time) you failed to define/implement an API for that.
> >
> > > For Issue #2, since ipsec-secgw is not utilizing the h/w lookup data,
> > > the solution here follows suit and is preventing other usage models of the
> > hardware which are allowed by the library.
> >
> > Same as above, right now there is no PMD that supports inline-proto and no API
> > to extract that extra H/W lookup data.
> > Though I replied to you and explained *twice*how I think such functionality
> > could be added if in hypothetical future there would be a PMD that will support
> > these features.
> > But you simply ignored my replies and kept threating our patches.
> > Obviously, I don't think it is a proper attitude for open-source community.
> > At least I would expect to provide some clear explanation, why do you think
> > approach I outlined is not good enough.
> > Even better would be to:
> > 1) submit an RFC/patch with API you envision and PMD implementation for it.
> > 2) submit an RFC/patch with changes in ipsec-secgw to enable these new
> > features.
> > Without that in place there is not much to discuss.
> >
> > > For Issue #3, librte_ipsec is still experimental and is just 2-3
> > > releases old. It has scope for improvement and all these features would've
> > added value to its utility. No strong feelings here.
> > >
> > > For Issue #4, glad that you documented that issue.
> > >
> > > For Issue #5, in your reply you had mentioned that there is no
> > > additional check done. I see atleast few checks in the datapath. Since the
> > feature itself is experimental, it would be less contentious if it was introduced as
> > a new data path.
> >
> > Once again - you keep making statements without any evidence.
> > Provide something more concrete here - description of packet and code flow
> > (function names, exact lines of code) that you think will cause a problem.
> >
> > >
> > > Also, to quote from the reply,
> > >
> > > > > [Anoob] Shouldn't we get that accepted first then?
> > > >
> > > > I don't think so.
> > > > Current implementation does provide expected functionality (with
> > > > known limitations).
> > > > In future, we can try to improve it and/or remove existing limitations.
> > > > That's a common iteration development approach that is used though
> > > > whole
> > > > DPDK:
> > > > - provide initial solution with basic functionality first
> > > > - improve/extend with future releases/patches
> > >
> > > If your suggestion is to stick to above guidelines,
> >
> > It is not my suggestion, that's how things work within DPDK right now.
> > Nearly every dpdk library, driver and sample app was developed based on that
> > principle.
> > Even ipsec-secgw itself, just look at the git log for it.
> >
> > > I don't think there is any point in pushing any further. I leave this
> > > to maintainers to decide. I would assume the same rules would apply to every
> > feature that is being attempted.
> >
> > I believe that for #1, #2, #5 your comments are biased, and doesn't provide real
> > picture.
> > So my suggestion to maintainers would be to ignore them as not valuable.
> >
> > Konstantin
> >
> >
> > >
> > > Thanks,
> > > Anoob
> > >
> > > > -----Original Message-----
> > > > From: dev <dev-bounces@dpdk.org> On Behalf Of Ananyev, Konstantin
> > > > Sent: Thursday, October 3, 2019 8:16 PM
> > > > To: Anoob Joseph <anoobj@marvell.com>; Smoczynski, MarcinX
> > > > <marcinx.smoczynski@intel.com>; akhil.goyal@nxp.com
> > > > Cc: dev@dpdk.org; Jerin Jacob Kollanukkaran <jerinj@marvell.com>;
> > > > Narayana Prasad Raju Athreya <pathreya@marvell.com>; Archana
> > > > Muniganti <marchana@marvell.com>
> > > > Subject: Re: [dpdk-dev] [PATCH v3 0/3] add fallback session
> > > >
> > > > Hi Anoob,
> > > >
> > > > > > > > > I've few more observations regarding the proposed feature.
> > > > > > > > >
> > > > > > > > > 1. From what I understood, if an ESP packet ends up on an
> > > > > > > > > unprotected interface and doesn't have 'PKT_RX_SEC_OFFLOAD'
> > > > > > > > > bit set, then the packet
> > > > > > > > would be looked up to see the associated SA and then
> > > > > > > > fallback session is figured out and then further processing is done.
> > > > > > > > >
> > > > > > > > > Can you confirm if I understood the sequence correctly? If
> > > > > > > > > yes, then aren't we doing an extra lookup in the s/w? The
> > > > > > > > > packet may be looked by the h/w using rte_flow and that
> > > > > > > > > information could be used to determine the
> > > > > > > > SA. Also, if the ESP packet is expected to be forwarded,
> > > > > > > > then the above logic will add an unnecessary lookup even
> > > > > > > > after your h/w has detected that the packet need not be security
> > processed.
> > > > > > > >
> > > > > > > > Not sure I understood your whole statement above.
> > > > > > > > AFAIK, right now (with dpdk master) for unprotected iface it
> > > > > > > > works like
> > > > that:
> > > > > > > >
> > > > > > > > 1.  slit incoming traffic into 3 groups: ESP packets, IPv4
> > > > > > > > packets, IPv6
> > > > > > packets.
> > > > > > > > For ESP packets:
> > > > > > > > 2. perform SAD lookup
> > > > > > > >     a. if it fails (non SA found for that SPI), then drop the packet.
> > > > > > > >     b. otherwise (SA found) process the packet using found
> > > > > > > > SA
> > > > > > > >
> > > > > > > > What fall-back patch adds:
> > > > > > > > Before step 2.b check:
> > > > > > > > does that SA has its primary session  with type
> > > > > > > > INLINE-CRYPTO and does HW fail to do IPsec realted processing for it
> > (by some reason)?
> > > > > > > > If yes, then mark this packet to be processed by fall-back session.
> > > > > > > > if (MBUF_NO_SEC_OFFLOAD(pkt) && sa->fallback_sessions > 0) {
> > > > > > > >                 uintptr_t intsa = (uintptr_t)sa;
> > > > > > > >                 intsa |= IPSEC_SA_OFFLOAD_FALLBACK_FLAG;
> > > > > > > >                 result_sa = (void *)intsa;  }
> > > > > > > >
> > > > > > > > So from my perspective, no additional lookup where introduced.
> > > > > > >
> > > > > > > [Anoob] For inline processing, h/w does a lookup and figures
> > > > > > > out the security
> > > > > > processing to be done on the packet.
> > > > > > > "rte_security_get_userdata()" allows s/w to further segregate
> > > > > > > that info. In this approach, I believe we don't consider how
> > > > > > > such info from h/w can
> > > > > > be used.
> > > > > >
> > > > > > Right now it is not the case with ipsec-secgw:
> > > > >
> > > > > [Anoob] Are you saying there is no h/w lookup involved when doing
> > > > > inline crypto processing? Then I'm confused. My understanding is
> > > > > that rte_flow
> > > > with ACTION type as SECURITY is being used to setup h/w lookups.
> > > >
> > > > I am saying that current ipsec-secgw code for each inbound ESP
> > > > packet
> > > > *always* does SW lookup.
> > > > No matter did HW lookup take place already and does information
> > > > about that lookup is available via mbuf in some way or not.
> > > > Just look at the ipsec-secgw code yourself:
> > > >
> > > > nb_rx = rte_eth_rx_burst(portid, queueid,  pkts, MAX_PKT_BURST);
> > > >
> > > > if (nb_rx > 0)
> > > > 	process_pkts(qconf, pkts, nb_rx, portid);
> > > >
> > > > Inside process_pkts()
> > > > https://urldefense.proofpoint.com/v2/url?u=http-
> > > > 3A__lxr.dpdk.org_dpdk_latest_ident_prepare-5Fone-
> > > >
> > 5Fpacket&d=DwIGaQ&c=nKjWec2b6R0mOyPaz7xtfQ&r=BPcGOOudUMrTDQ9Yb
> > > >
> > gKcOkO5ChYiUPPlPNIEvTOhjNE&m=KVBgvXYlUGyMmu260erISKr73Qt6PBLux0SI
> > > > oSO1i-M&s=opbGGHUqfxy8p_4NvwCG6od6VdV0qN8XafaPiubfO1A&e=
> > > >  it first calls prepare_traffic() which does sort of de-muxing:
> > > > put packet in one of 3 arrays:
> > > >    1) ESP packets
> > > >    2) non ESP IPv4 packets
> > > >    3) non ESP IPv6 packets
> > > > Also it checks is PKT_RX_SEC_OFFLOAD set for that packet.
> > > > If yes, it retrieves associated security user-data and stores it
> > > > inside private mbuf area.
> > > >
> > > > Then it goes into process_pkts_inbound()
> > > > https://urldefense.proofpoint.com/v2/url?u=http-
> > > > 3A__lxr.dpdk.org_dpdk_latest_ident_process-5Fpkts-
> > > >
> > 5Finbound&d=DwIGaQ&c=nKjWec2b6R0mOyPaz7xtfQ&r=BPcGOOudUMrTDQ9Y
> > > >
> > bgKcOkO5ChYiUPPlPNIEvTOhjNE&m=KVBgvXYlUGyMmu260erISKr73Qt6PBLux0S
> > > > IoSO1i-M&s=2OxMXug7jtUQPcYiDN4nW9WbrMtRfTJx5k-N7ikOeME&e=
> > > > and here for all ESP packets:
> > > > legacy code path:
> > > > ipsec_inbound()->inbound_sa_lookup()->single_inbound_lookup()
> > > > for librte_ipsec code path:
> > > > inbound_sa_lookup()->single_inbound_lookup()
> > > >
> > > > single_inbound_lookup() is the one that does actual SW lookup for
> > > > each input packet.
> > > > https://urldefense.proofpoint.com/v2/url?u=http-
> > > > 3A__lxr.dpdk.org_dpdk_latest_ident_single-5Finbound-
> > > >
> > 5Flookup&d=DwIGaQ&c=nKjWec2b6R0mOyPaz7xtfQ&r=BPcGOOudUMrTDQ9Yb
> > > >
> > gKcOkO5ChYiUPPlPNIEvTOhjNE&m=KVBgvXYlUGyMmu260erISKr73Qt6PBLux0SI
> > > > oSO1i-M&s=D2ZPZPLbcQSbaluG8KH0F3uUDHfQWNYxw4MOc0fSkI8&e=
> > > >
> > > > >
> > > > > > for each inbound ESP packet it *always* does a lookup in SW
> > > > > > based SADB, and if lookup fails - drops the packet (see 2.a above).
> > > > > > So, I still not see any additional lookups introduced with the patch.
> > > > >
> > > > > [Anoob] In case of inline crypto, you could do that. No disagreement.
> > > > > But that doesn't mean that is the only way. If PMDs can retrieve
> > > > > info about
> > > > h/w lookups and pass it on to the upper layers, isn't that the better
> > approach?
> > > >
> > > > Please let's keep our conversation in a constructive way.
> > > > We are not discussing what could be done in theory, but a particular
> > > > patch for
> > > > ipsec-secgw:
> > > > Right now ipsec-secgw does a SW lookup for each inbound ESP packet
> > > > (no matter what HW offload does) and this patch doesn't introduce
> > > > any extra SW lookups.
> > > >
> > > > If you are not happy with current ipsec-secgw approach, that's fine
> > > > - feel free to submit RFC/patch to fix that, or start a discussion in a new
> > thread.
> > > > I just don't see why we have to discuss it in context of this patch.
> > > >
> > > > >
> > > > > >
> > > > > > >
> > > > > > > > Also AFAIK, right now there is no possibility to configure
> > > > > > > > ipsec-secgw to BYPASS some ESP traffic.
> > > > > > > > Should we do it (to conform to ipsec RFC) - that's probably
> > > > > > > > subject of another discussion.
> > > > > > >
> > > > > > > [Anoob] The approach (choice of flags) forces a software-based
> > > > > > > SA lookup for packets that need to be bypassed instead of
> > > > > > > leveraging possible
> > > > > > hardware SA lookup. I don't think what ipsec-secgw does matters here.
> > > > > >
> > > > > > I do not agree with that statement.
> > > > > > First of all - current patch doesn't introduce any additional SW
> > > > > > lookups, see above.
> > > > > > Second, if someone would like to add BYPASS for ESP packets into
> > > > > > ipsec-secgw, I think he would have to insert new code that do
> > > > > > de-mux
> > > > > > *before* any ESP related processing starts.
> > > > > > Most obvious variant: at prepare_one_packet() when we split our
> > > > > > incoming traffic to IPsec and non-IPsec ones.
> > > > > > Second variant - at early stage of single_inbound_lookup(),
> > > > > > straight after actual SAD lookup failure.
> > > > > > In both cases, I don't see how session selection (primary or
> > > > > > faillback) would affect us here, actual decision do we want to
> > > > > > PROCESS or BYPASS particular ESP packet needs to take place before
> > that.
> > > > > > So I still believe we are ok here.
> > > > >
> > > > > [Anoob] I don't think you understood what I'm trying to highlight
> > > > > here. You could have a situation when h/w can detect that the
> > > > > packet has to be "PROTECT"ed using an "inline crypto" session. But
> > > > > then it might detect that it cannot proceed with inline processing
> > > > > because of some
> > > > issue (fragmentation, h/w queue overflow etc). Now the h/w has
> > > > already figured out the action to be done to the packet. If PMDs
> > > > allow this to be communicated to the application, the application won't be
> > required to do the lookup.
> > > > >
> > > > > In inline crypto, application can ignore the h/w lookup data and
> > > > > do a s/w lookup with the protocol headers as they are not stripped
> > > > > off. It was done this way, because API and the associated means to
> > > > > get this info from PMD was not introduced when inline crypto and
> > > > > corresponding
> > > > support in Intel's PMD was added. But inline protocol cannot do the
> > > > lookup in the application and so the concept of userdata etc was
> > > > introduced. The same can be applied to inline crypto also.
> > > > Advantage? It could remove the extra lookup done in s/w.
> > > > >
> > > > > To summarize, we cannot assume that every inline crypto packet
> > > > > need to looked up in the application to figure out the processing done on
> > the packet.
> > > > There can be applications which relies on h/w lookup data to figure
> > > > out the processing done.
> > > > >
> > > > > Example: Using rte_flow, I can create a rule for ESP packets to be
> > > > > MARKed. There is no security session created for the flow and the
> > > > > application intends to forward this flow packets to a different
> > > > > port. With your
> > > > approach, even these packets would be looked up. The packet will
> > > > have info from the h/w lookup which doesn't get used, because the
> > > > approach fails to introduce the required concepts.
> > > >
> > > > I think I understood your point.
> > > > Basically you saying that in future your PMD for unprocessed ESP
> > > > packets might provide some additional information (via
> > > > rte_security_get_userdata) that might be used by SW - let say to choose
> > BYPASS for such packets.
> > > > Current ipsec-secgw doesn't support such ability.
> > > > Your concern is that this patch will somehow prevent (or will make
> > > > it harder) for you to make your future changes.
> > > > Correct?
> > > > If, so then I said before, I don't think that concern is valid.
> > > > Most obvious variant where to add such code is inisde inside
> > > > prepare_one_packet() when we split our incoming traffic to IPsec and
> > > > non-IPsec ones.
> > > > To be more specific something like that
> > > >
> > > > if (eth->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) {
> > > >
> > > >                 iph4 = (const struct rte_ipv4_hdr *)rte_pktmbuf_adj(pkt,
> > > >                         RTE_ETHER_HDR_LEN);
> > > >                 adjust_ipv4_pktlen(pkt, iph4, 0);
> > > >
> > > > -                if (iph4->next_proto_id == IPPROTO_ESP)
> > > > +                if (iph4->next_proto_id == IPPROTO_ESP) {
> > > > +		if (pkt->ol_flags & PKT_RX_SEC_OFFLOAD) {
> > > > +			/* extract extra info, make decision based on that */
> > > > +		} else
> > > > 			t->ipsec.pkts[(t->ipsec.num)++] = pkt;
> > > > +	   }
> > > >
> > > > Of course some code reordering might be needed to avoid performance
> > > > drop, but I suppose the main idea is clear enough.
> > > >  Second variant - at early stage of single_inbound_lookup(), either
> > > > before or straight after actual SAD SW lookup failure.
> > > > In both cases, this new code will be executed *before* session selection
> > code.
> > > >
> > > > >
> > > > > Does the above sound like BYPASS? Yes.
> > > > > Does ipsec-secgw do it this way? No.
> > > >
> > > > Correct, it doesn't.
> > > > You have to submit patches if you'd like to support such ability.
> > > >
> > > > > Does this approach prevent an application from introducing such a
> > BYPASS?
> > > > Yes.
> > > >
> > > > That's not correct, I believe, see above.
> > > >
> > > > >
> > > > > >
> > > > > > >
> > > > > > > For example, ESN was not supported by ipsec-secgw (before
> > > > > > > library mode was introduced), but every single library change
> > > > > > > and application change was added keeping in mind that ESN is a
> > > > > > > valid feature for ipsec. So it is
> > > > > > one thing to say ipsec-secgw doesn't support ESP bypass and
> > > > > > saying the solution doesn't consider a possibility of ESP bypass.
> > > > > > >
> > > > > > > >
> > > > > > > > >
> > > > > > > > > 2. The solution proposed here seems like adding the
> > > > > > > > > handling in ipsec-secgw instead of ipsec library. In other
> > > > > > > > > words, this feature is not getting
> > > > > > > > added in ipsec library, which was supposed to simplify the
> > > > > > > > whole ipsec usage in DPDK, but fails to handle the case of
> > fragmentation.
> > > > > > > >
> > > > > > > > What we have right now with ipsec library is SA (low) level API.
> > > > > > > > It can handle multi-segment packets properly, but expects
> > > > > > > > someone else to do other steps (fragmentation/reassembly).
> > > > > > > > ipsec-secgw demonstrates how librte_ip_frag and librte_ipsec
> > > > > > > > can be used together to deal with fragmented IPsec traffic
> > > > > > > > in a proper
> > > > manner.
> > > > > > > > Probably in future we'll consider adding some high-level API
> > > > > > > > that will be able to do whole processing under hood (SPD/SAD
> > > > > > > > lookup, fragment/reassembly, actual IPsec packet processing,
> > > > > > > > matching inbound selectors, etc.), but right now it is not the case.
> > > > > > > >
> > > > > > > > > Also, since the fallback feature is entirely done in the
> > > > > > > > > application, it begs the
> > > > > > > > question why the same feature is omitted for legacy use case.
> > > > > > > >
> > > > > > > > Because legacy mode doesn't support multi-seg packets at first place.
> > > > > > > > Also it is an extra overhead to support 2 code-paths (legacy
> > > > > > > > and
> > > > > > > > library) for the same app, so we'd like in future to
> > > > > > > > deprecate and later remove legacy code- path.
> > > > > > > > As a first step we propose to make library code-path a default one:
> > > > > > > > https://urldefense.proofpoint.com/v2/url?u=http-3A__patches.
> > > > > > > > dpdk
> > > > > > > > .org
> > > > > > > >
> > > > > >
> > > >
> > _cover_58247_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=jPfB8rwwviRSxyL
> > > > > > Ws2
> > > > > > > > n6B-WYLn1v9SyTMrT5EQqh2TU&m=b3E429fuo8P-K5XfH8wG-
> > > > > > 7hwr1d8oM4uJGErAkbf
> > > > > > > > DvA&s=vz7ioUzJOuzoJdmV7QO0QLPKYn1ytFsj_0eYatbSCrg&e=
> > > > > > >
> > > > > > > [Anoob] Even if we need the application to do the reassembly,
> > > > > > > it would look simple if application uses the same "rte_ipsec_session"
> > > > > > > to process the
> > > > > > fallback packet.
> > > > > >
> > > > > > I think there is some sort of misunderstanding here.
> > > > > > With current librte_ipsec design:
> > > > > > rte_ipsec_sa - opaque SW representation of the SA (HW neutral)
> > > > > > rte_ipsec_session associates SA (rte_ipsec_sa)with particular HW
> > > > > > device (session).
> > > > > > Same SA can be referred by multiple sessions.
> > > > > >
> > > > > > From rte_ipsec.h:
> > > > > > /**
> > > > > >  * rte_ipsec_session is an aggregate structure that defines
> > > > > > particular
> > > > > >  * IPsec Security Association IPsec (SA) on given security/crypto device:
> > > > > >  * - pointer to the SA object
> > > > > >  * - security session action type
> > > > > >  * - pointer to security/crypto session, plus other related data
> > > > > >  * - session/device specific functions to prepare/process IPsec packets.
> > > > > >  */
> > > > > >
> > > > > > > In that case, the processing required to handle the other
> > > > > > > packet will be hidden from the application. Here application
> > > > > > > has to make sure it chooses
> > > > > > the correct session, which I believe should be moved to ipsec library.
> > > > > >
> > > > > > From my side it is a good thing to let application define its own usage
> > model.
> > > > > > I.E. librte_ipsec supports multiple-sessions per SA, upper-layer
> > > > > > (app) decides how it will use that feature.
> > > > > > Though if you strongly feel that some higher-level API is needed
> > > > > > here, and have some good ideas about it  - please go ahead and
> > > > > > submit RFC
> > > > for it.
> > > > >
> > > > > [Anoob] If you are okay with application defining how it
> > > > > implements ipsec,
> > > > then probably there is no use case for lib ipsec?
> > > >
> > > > :)
> > > > Once again - library provides an implementation.
> > > > Application defines the way to use it, i.e. how it will apply
> > > > functionality library provides for different use-case scenarios.
> > > > As an example: snpritnf() provides user with ability to format
> > > > strings, application decides what exactly format to use and for which string.
> > > > Same for librte_ipsec, library provides functionality to perform
> > > > processing for ESP packets.
> > > > Application defines which packets to process and what session to use.
> > > >
> > > > >
> > > > > >
> > > > > > >
> > > > > > > >
> > > > > > > > >
> > > > > > > > > 3. It seems like ordering won't be maintained once this
> > > > > > > > > processing is done. Again, this is the sequence I understood.
> > > > > > > > > Please correct me if I missed
> > > > > > > > something,
> > > > > > > > >        a. Application receives a bunch of packets (let's
> > > > > > > > > say 6 packets), in which few are fragmented (P3 & P4) and
> > > > > > > > > the rest can be inline
> > > > > > > > processed.
> > > > > > > > >        b. Application receives P1->P2->P3->P4->P5->P6 (in
> > > > > > > > > this, P1, P2, P5, P6 are
> > > > > > > > inline processed successfully) and P4 & P5 are the fragments
> > > > > > > > >        c. Application groups packets. P1->P2->P5->P6
> > > > > > > > >becomes one group and P3-
> > > > > > > > >P4 becomes another and goes for fallback processing.
> > > > > > > > > Now how is ordering maintained? I couldn't figure out how
> > > > > > > > >that is done in this
> > > > > > > > case.
> > > > > > > >
> > > > > > > > You right, fallback session can introduce packet reordering.
> > > > > > > > At least till we'll have ability to process packets in sync mode too.
> > > > > > > > See our presentation at dpdk userspace (slides 17, 18):
> > > > > > > > https://urldefense.proofpoint.com/v2/url?u=https-3A__static.
> > > > > > > > sche
> > > > > > > > d.co
> > > > > > > > m_hosted-5Ffiles_dpdkbordeaux2019_8f_DPDK-
> > > > > > 2DIPSECu9.pdf&d=DwIFAg&c=n
> > > > > > > > KjWec2b6R0mOyPaz7xtfQ&r=jPfB8rwwviRSxyLWs2n6B-
> > > > > > WYLn1v9SyTMrT5EQqh2TU&
> > > > > > > > m=b3E429fuo8P-K5XfH8wG-
> > > > > > 7hwr1d8oM4uJGErAkbfDvA&s=MLRnnYcykjnsqXrHGUuR
> > > > > > > > YHc5uDxaAod0Yl7f06EQr1M&e= Right now the only way to deal
> > > > > > > > with it - have replay window big enough to sustain
> > > > > > > > reordering and async processing latency.
> > > > > > > > That's actually another reason why we add this feature into
> > > > > > > > ipsec-secgw sample
> > > > > > > > app:
> > > > > > > > so people can evaluate it on their platforms, determine what
> > > > > > > > replay window size would be needed, what issues/slowdowns it
> > > > > > > > might
> > > > cause, etc.
> > > > > > >
> > > > > > > [Anoob] This is something I had noticed while going through the code
> > flow.
> > > > > > The ordering info is lost because of the approach.
> > > > > >
> > > > > > Once again, it is a known limitation and we are not trying to
> > > > > > hide it from you :)
> > > > >
> > > > > [Anoob] Never said it was hidden. Initially I had okayed the
> > > > > approach even though it wasn't a solution suitable for inline
> > > > > protocol. But as more cases
> > > > were considered, few shortcomings were observed, and I was skeptical
> > > > about how any of that would be solved in the long run.
> > > > >
> > > > > My suggestion, if these limitations and the plans to address it
> > > > > were mentioned
> > > > in the cover letter, we could've saved few cycles here.
> > > >
> > > > Point taken.
> > > >
> > > > > But my reply would still be the same 😊
> > > > >
> > > > > > It
> > > > > > was outlined here:
> > > > > > https://urldefense.proofpoint.com/v2/url?u=https-
> > > > > > 3A__static.sched.com_hosted-5Ffiles_dpdkbordeaux2019_8f_DPDK-
> > > > > >
> > > >
> > 2DIPSECu9.pdf&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=jPfB8rwwviRSxyL
> > > > > > Ws2n6B-WYLn1v9SyTMrT5EQqh2TU&m=b3E429fuo8P-K5XfH8wG-
> > > > > >
> > > >
> > 7hwr1d8oM4uJGErAkbfDvA&s=MLRnnYcykjnsqXrHGUuRYHc5uDxaAod0Yl7f06E
> > > > > > Qr1M&e=
> > > > > > And in the latest patch version Marcin clearly stated it inside the AG
> > session:
> > > > > > https://urldefense.proofpoint.com/v2/url?u=http-
> > > > > >
> > > >
> > 3A__patches.dpdk.org_patch_60039_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtf
> > > > > > Q&r=jPfB8rwwviRSxyLWs2n6B-
> > > > WYLn1v9SyTMrT5EQqh2TU&m=b3E429fuo8P-
> > > > > > K5XfH8wG-
> > > > > >
> > > >
> > 7hwr1d8oM4uJGErAkbfDvA&s=zrn91Vjf_ZElAlDf7IeZGmGevcA6RMwpPTLUCGlgZ
> > > > > > cY&e=
> > > > > > If you think even further clarification within the doc is
> > > > > > needed, please let us know.
> > > > > >
> > > > > > About the actual packet re-oridering:
> > > > > >  AFAIK, for some use-cases it might be acceptable, for others not.
> > > > > > Though it is an optional feature, that wouldn't be enabled by
> > > > > > default, so user can always choose is it does he needs/wants
> > > > > > this feature or
> > > > not.
> > > > > > If/when we'll have CPU_CRYPTO processing mode:
> > > > >
> > > > > [Anoob] Shouldn't we get that accepted first then?
> > > >
> > > > I don't think so.
> > > > Current implementation does provide expected functionality (with
> > > > known limitations).
> > > > In future, we can try to improve it and/or remove existing limitations.
> > > > That's a common iteration development approach that is used though
> > > > whole
> > > > DPDK:
> > > > - provide initial solution with basic functionality first
> > > > - improve/extend with future releases/patches
> > > >
> > > > >
> > > > > Also, the ordering is lost because application is not considering
> > > > > that. Why can't
> > > > we use re-ordering library? Or an event-dev?
> > > >
> > > > No-one saying it can't be improved in one way or another.
> > > > We don't plan to introduce such code right now due to its complexity.
> > > > Might be something in future.
> > > > Though you are welcome to go ahead and submit your own patches to
> > > > improve that solution.
> > > >
> > > > >
> > > > > > https://urldefense.proofpoint.com/v2/url?u=http-
> > > > > >
> > > >
> > 3A__patches.dpdk.org_cover_58862_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtf
> > > > > > Q&r=jPfB8rwwviRSxyLWs2n6B-
> > > > WYLn1v9SyTMrT5EQqh2TU&m=b3E429fuo8P-
> > > > > > K5XfH8wG-
> > > > > >
> > > >
> > 7hwr1d8oM4uJGErAkbfDvA&s=k1zef_1uqELXblKN3_qjX04WXNFGAUEFIIJrvKPr6
> > > > > > eA&e=
> > > > > > we'll add ability for the user to avoid this reordering problem
> > > > > >
> > > > > > > As we dig deeper, the feature looks hardly complete. The
> > > > > > > concerning part is the approach doesn't seem conducive to
> > > > > > > fixing any of these issues in the
> > > > > > future.
> > > > > >
> > > > > > Again, don't agree with that statement.
> > > > > > From my perspective: while current implementation has its
> > > > > > limitations (packet reordering, inline-proto/lksd-proto), it is
> > > > > > totally functional and provides users with a reference how to
> > > > > > deal with such
> > > > kind of problems.
> > > > >
> > > > > [Anoob] Well, that is exactly my problem. It gives a skewed
> > > > > reference on how to deal with the problem, without considering other
> > possibilities.
> > > >
> > > > Once again, please feel free and come-up with your own patches, that
> > > > will address this problem in a smarter way.
> > > >
> > > > > If every application starts adopting this method and starts doing
> > > > > lookup for every ESP packet, h/w based lookups (and the associated
> > > > > h/w) would be rendered useless. I really think this series, which
> > > > > happens to be
> > > > the first step in solving a complex problem, will be very useful if
> > > > all these issues are addressed properly.
> > > > >
> > > > > If you can introduce the inline+lookaside via librte_ipsec, its
> > > > > perhaps ok for the patch to have some limitations(like lack of
> > > > > ordering,
> > > > assumptions on ESP etc). However, if the patch is introduced
> > > > directly into ipsec- secgw, it needs to be more comprehensive and robust.
> > > >
> > > > I think common DPDK policy is exactly opposite:
> > > > library/driver has to be as robust and comprehensive as possible.
> > > > sample app code allowed to have some limitations (as long as they
> > > > are documented, etc.).
> > > >
> > > > > I also expect performance impact due to these changes
> > > >
> > > > Could you be more specific here?
> > > > What performance impact do you expect and why?
> > > > Can you point to the exact piece of patch code that you believe
> > > > would cause a degradation.
> > > > I don't expect any and performance tests on our platforms didn't
> > > > show any regression so far.
> > > >
> > > > > and so I prefer if the inline+lookaside can be made separate
> > > > > datapath rather than overloading the existing one.
> > > >
> > > > I don't see any good reason for that.
> > > > This feature is optional.
> > > > User have to enable it manually for each session (via config file).
> > > > for all other sessions there is no impact.
> > > > What we probably can do - add extra check inside sa.c to allow
> > > > fallback sessions only for inline-crypto-offload type of sessions.
> > > >
> > > > >
> > > > > >
> > > > > > >
> > > > > > > Also, if the new ipsec related features are introduced via
> > > > > > > ipsec-secgw than via rte_ipsec, then it raises doubts over the
> > > > > > > utility of
> > > > rte_ipsec library.
> > > > > >
> > > > > >

Re: [dpdk-dev] [PATCH v3 0/3] add fallback session

[Ananyev, Konstantin]

> 
> Hi Konstantin,
> 
> Thanks for clarifying your stand on review comments.
> 
> From the comments we(Marvell/Cavium) have received on multiple features, my impression is that the feature need to be correct/fully
> done/acceptable to the overall community. Features are usually agreed upon if there is solid plan on how it will work in the long run.
> 
> Discussions with Radu on what eventually became get_userdata()
> http://mails.dpdk.org/archives/dev/2017-November/082039.html
> 
> From the mail:
> 
> "> Having something like "get_pkt_metadata" should be fine for inline
> > protocol usage. But we will still need a "get cookie" call to get the
> > cookie, as the application would need something it can interpret.
> Why can't you have a get_pkt_metadata that returns the "cookie" - which
> I would call it userdata or similar? What I'm trying to say is that we
> should try to keep it as generic as possible. For example, I wouldn't
> assume that the cookie is stored in pkt->udata64 in the application."
> 
> Marvell prefers using pkt->udata64 instead of get_userdata() as it avoids an extra cache line access while trying to get hold of the SA which
> processed the packet. We agreed to Radu's suggestion even though Intel/Radu didn't have access to an inline protocol enabled h/w and it
> wasn't affecting any data path that Intel/Radu would be interested in.
> 
> Discussions with Mattias on eventmode helper
> http://mails.dpdk.org/archives/dev/2019-June/134900.html
> 
> "Or even worse, the
> application's developers are forced to do a big-bang switch over to
> using the event and ethernet device APIs directly, in case they can't
> patch DPDK to work around the
> eventmode-assumption-that-didn't-hold-for-them.
> 
> You could always add flexibility to the framework (as you encounter a
> need for it), but then it will grow in complexity as well."
> 
> Here Mattias resisted the idea of eventmode helper for three reasons,
> 
> 1. It's purely software, ie, no backing PMD required (which is the same case for librte_ipsec as well).
> 2. It "could" break for a new feature that "could" get added in eventdev.
> 3. He "thought" it would grow in complexity.

Sorry, but why we have to discuss other people comments on your previous patches here?
How does it relate to the patch under discussion?
From my perspective, it doesn't at all.
 
> Marvell/Cavium has respected his opinion and did rework the entire approach which exponentially increased the effort required from our
> end (one code base integrated into 3 different apps vs 3 different code bases for 3 apps). Please do note that this entire "l2fwd-event" had
> to be redone 3 times while trying to get a consensus on the approach. I believe you had okeyed the first approach, so you cannot say we
> had to redo the patches because the patches were done poorly or the feature was incorrect.

Once again, if you felt that these comments were inappropriate and caused extra unnecessary work,
why do you complain about it here?
Why didn't you flag that at original patch review time?

> I see that there is a shift in the review process these days and the vendors are encouraged to commend only on the features they are
> interested in. In that case, Marvell will no longer bother with patches which doesn't come in modes which doesn't matter to us. I would
> expect the same courtesy returned back as well.

I personally not aware about such shift.
I still think that as community member you are free to review any patches you'd like.
Moreover as a good citizen of the community you are encouraged to review other people patches.
Though I think, these reviews need to be relevant, concentrate on patch subject,
and take into account real picture.  

Let say about your comment that current fall-back session doesn't work for inline-proto:
True, it doesn't.
Though the reason for that -  required for that API is not defined and not implemented.
As API is not in place (and no signs that it would be anytime soon),
we'll have to go ahead and submit our code without inline-proto/lookaside-proto support.
Though, if such API would be defined and implemented -
for sure we will have to respect and support it, as we did in other places for inline-proto devices.
And yes, I expect Marvell team to lead/start that API work, because right now you are the only ones
within DPDK world who claimed to have inline-proto devices.
Other teams simply don't have enough knowledge (and need) for that.

> 
> FYI, eventmode ipsec-secgw is submitted as an RFC. In line with the comment redressal here, I've documented all the limitations.
> 
> > Once again - you keep making statements without any evidence.
> > Provide something more concrete here - description of packet and code flow
> > (function names, exact lines of code) that you think will cause a problem.
> 
> I saw atleast the ones below. It would be frivolous to assume that the feature is added without any data path checks.
> 
> +	if (MBUF_NO_SEC_OFFLOAD(pkt) && sa->fallback_sessions > 0) {
> 
> +			if (pg->id.val & IPSEC_SA_OFFLOAD_FALLBACK_FLAG) {
> 
> @maintainers, Marvell is not blocking this feature anymore assuming that the same yardstick will be used while reviewing patches
> submitted by us.
> 
> Thanks,
> Anoob
> 
> > -----Original Message-----
> > From: Ananyev, Konstantin <konstantin.ananyev@intel.com>
> > Sent: Thursday, October 10, 2019 4:25 PM
> > To: Anoob Joseph <anoobj@marvell.com>; Smoczynski, MarcinX
> > <marcinx.smoczynski@intel.com>; akhil.goyal@nxp.com
> > Cc: dev@dpdk.org; Jerin Jacob Kollanukkaran <jerinj@marvell.com>; Narayana
> > Prasad Raju Athreya <pathreya@marvell.com>; Archana Muniganti
> > <marchana@marvell.com>
> > Subject: RE: [dpdk-dev] [PATCH v3 0/3] add fallback session
> >
> >
> > Hi Anoob,
> >
> > >
> > > Following are the main issues I had identified,
> > >
> > > 1. Doesn't work for inline protocol offload (the plan itself won't
> > > scale for that case) 2. Ignores the data from the h/w lookup 3. The
> > > fallback support is actually supported in ipsec-secgw (application) instead of
> > librte_ipsec.
> > > 4. Reordering issue.
> > > 5. Introduces checks in existing data path even when the feature is not
> > properly implemented/is not suitable for other PMDs.
> > >
> > > For Issue #1, I get that Intel doesn't have plans to support the feature and
> > won't be submitting anything in that direction.
> > > But adding code that further exacerbates the situation is probably not desired.
> >
> > We already discussed why that feature can't be fully supported right now by
> > inline-proto/lookaside-proto devices:
> > There is no API defined to sync session state between HW and SW
> > https://urldefense.proofpoint.com/v2/url?u=http-
> > 3A__mails.dpdk.org_archives_dev_2019-
> > 2DSeptember_143881.html&d=DwIGaQ&c=nKjWec2b6R0mOyPaz7xtfQ&r=jPfB8
> > rwwviRSxyLWs2n6B-
> > WYLn1v9SyTMrT5EQqh2TU&m=UGz2FemapWdsohgNRq2UX5ArFEL_pLGYEVrwe
> > IzaUX4&s=ObkJK9pyzOZuwoeulr6mrr6iKQF1NgXu3p2v7XwkYEs&e=
> >
> > I already outlined that gap nearly a year ago:
> > https://urldefense.proofpoint.com/v2/url?u=http-
> > 3A__mails.dpdk.org_archives_dev_2018-
> > 2DSeptember_113600.html&d=DwIGaQ&c=nKjWec2b6R0mOyPaz7xtfQ&r=jPfB8
> > rwwviRSxyLWs2n6B-
> > WYLn1v9SyTMrT5EQqh2TU&m=UGz2FemapWdsohgNRq2UX5ArFEL_pLGYEVrwe
> > IzaUX4&s=0x8pmfsS7nT7Xn-a_RvhI-DxtHMdk0zr4jiapi22tVE&e=
> > Though so far I didn't see any attempt to resolve this problem by Marvell (or
> > NXP) engineers.
> > Which makes me think that either you guys don't consider it as a problem at all,
> > or have no desire to work on it.
> > I never insisted, as I don't  think it is my call by obvious reasons:
> > 1) right now Intel doesn't have HW that provides inline-proto/lksd-proto
> > capabilities.
> > 2) Intel DPDK team doesn't have access to such HW.
> > 3) There is no PMD inside dpdk.org that implements inline-proto anyway.
> >
> > Now, one year later we are in a situation where:
> > 1) there still no PMD at dpdk.org that supports inline-proto.
> > 2) API to sync session state between HW and SW still not defined/implemented.
> > 3) you trying to accuse me and other Intel DPDK team for lack of interest to
> > work on that problem for you
> >    and threat our patches because of that.
> >
> > That just doesn't look fair to me.
> > You shouldn't expect other people patches to provide 'complete' solution for
> > inline-proto, if (in a year time) you failed to define/implement an API for that.
> >
> > > For Issue #2, since ipsec-secgw is not utilizing the h/w lookup data,
> > > the solution here follows suit and is preventing other usage models of the
> > hardware which are allowed by the library.
> >
> > Same as above, right now there is no PMD that supports inline-proto and no API
> > to extract that extra H/W lookup data.
> > Though I replied to you and explained *twice*how I think such functionality
> > could be added if in hypothetical future there would be a PMD that will support
> > these features.
> > But you simply ignored my replies and kept threating our patches.
> > Obviously, I don't think it is a proper attitude for open-source community.
> > At least I would expect to provide some clear explanation, why do you think
> > approach I outlined is not good enough.
> > Even better would be to:
> > 1) submit an RFC/patch with API you envision and PMD implementation for it.
> > 2) submit an RFC/patch with changes in ipsec-secgw to enable these new
> > features.
> > Without that in place there is not much to discuss.
> >
> > > For Issue #3, librte_ipsec is still experimental and is just 2-3
> > > releases old. It has scope for improvement and all these features would've
> > added value to its utility. No strong feelings here.
> > >
> > > For Issue #4, glad that you documented that issue.
> > >
> > > For Issue #5, in your reply you had mentioned that there is no
> > > additional check done. I see atleast few checks in the datapath. Since the
> > feature itself is experimental, it would be less contentious if it was introduced as
> > a new data path.
> >
> > Once again - you keep making statements without any evidence.
> > Provide something more concrete here - description of packet and code flow
> > (function names, exact lines of code) that you think will cause a problem.
> >
> > >
> > > Also, to quote from the reply,
> > >
> > > > > [Anoob] Shouldn't we get that accepted first then?
> > > >
> > > > I don't think so.
> > > > Current implementation does provide expected functionality (with
> > > > known limitations).
> > > > In future, we can try to improve it and/or remove existing limitations.
> > > > That's a common iteration development approach that is used though
> > > > whole
> > > > DPDK:
> > > > - provide initial solution with basic functionality first
> > > > - improve/extend with future releases/patches
> > >
> > > If your suggestion is to stick to above guidelines,
> >
> > It is not my suggestion, that's how things work within DPDK right now.
> > Nearly every dpdk library, driver and sample app was developed based on that
> > principle.
> > Even ipsec-secgw itself, just look at the git log for it.
> >
> > > I don't think there is any point in pushing any further. I leave this
> > > to maintainers to decide. I would assume the same rules would apply to every
> > feature that is being attempted.
> >
> > I believe that for #1, #2, #5 your comments are biased, and doesn't provide real
> > picture.
> > So my suggestion to maintainers would be to ignore them as not valuable.
> >
> > Konstantin
> >
> >
> > >
> > > Thanks,
> > > Anoob
> > >
> > > > -----Original Message-----
> > > > From: dev <dev-bounces@dpdk.org> On Behalf Of Ananyev, Konstantin
> > > > Sent: Thursday, October 3, 2019 8:16 PM
> > > > To: Anoob Joseph <anoobj@marvell.com>; Smoczynski, MarcinX
> > > > <marcinx.smoczynski@intel.com>; akhil.goyal@nxp.com
> > > > Cc: dev@dpdk.org; Jerin Jacob Kollanukkaran <jerinj@marvell.com>;
> > > > Narayana Prasad Raju Athreya <pathreya@marvell.com>; Archana
> > > > Muniganti <marchana@marvell.com>
> > > > Subject: Re: [dpdk-dev] [PATCH v3 0/3] add fallback session
> > > >
> > > > Hi Anoob,
> > > >
> > > > > > > > > I've few more observations regarding the proposed feature.
> > > > > > > > >
> > > > > > > > > 1. From what I understood, if an ESP packet ends up on an
> > > > > > > > > unprotected interface and doesn't have 'PKT_RX_SEC_OFFLOAD'
> > > > > > > > > bit set, then the packet
> > > > > > > > would be looked up to see the associated SA and then
> > > > > > > > fallback session is figured out and then further processing is done.
> > > > > > > > >
> > > > > > > > > Can you confirm if I understood the sequence correctly? If
> > > > > > > > > yes, then aren't we doing an extra lookup in the s/w? The
> > > > > > > > > packet may be looked by the h/w using rte_flow and that
> > > > > > > > > information could be used to determine the
> > > > > > > > SA. Also, if the ESP packet is expected to be forwarded,
> > > > > > > > then the above logic will add an unnecessary lookup even
> > > > > > > > after your h/w has detected that the packet need not be security
> > processed.
> > > > > > > >
> > > > > > > > Not sure I understood your whole statement above.
> > > > > > > > AFAIK, right now (with dpdk master) for unprotected iface it
> > > > > > > > works like
> > > > that:
> > > > > > > >
> > > > > > > > 1.  slit incoming traffic into 3 groups: ESP packets, IPv4
> > > > > > > > packets, IPv6
> > > > > > packets.
> > > > > > > > For ESP packets:
> > > > > > > > 2. perform SAD lookup
> > > > > > > >     a. if it fails (non SA found for that SPI), then drop the packet.
> > > > > > > >     b. otherwise (SA found) process the packet using found
> > > > > > > > SA
> > > > > > > >
> > > > > > > > What fall-back patch adds:
> > > > > > > > Before step 2.b check:
> > > > > > > > does that SA has its primary session  with type
> > > > > > > > INLINE-CRYPTO and does HW fail to do IPsec realted processing for it
> > (by some reason)?
> > > > > > > > If yes, then mark this packet to be processed by fall-back session.
> > > > > > > > if (MBUF_NO_SEC_OFFLOAD(pkt) && sa->fallback_sessions > 0) {
> > > > > > > >                 uintptr_t intsa = (uintptr_t)sa;
> > > > > > > >                 intsa |= IPSEC_SA_OFFLOAD_FALLBACK_FLAG;
> > > > > > > >                 result_sa = (void *)intsa;  }
> > > > > > > >
> > > > > > > > So from my perspective, no additional lookup where introduced.
> > > > > > >
> > > > > > > [Anoob] For inline processing, h/w does a lookup and figures
> > > > > > > out the security
> > > > > > processing to be done on the packet.
> > > > > > > "rte_security_get_userdata()" allows s/w to further segregate
> > > > > > > that info. In this approach, I believe we don't consider how
> > > > > > > such info from h/w can
> > > > > > be used.
> > > > > >
> > > > > > Right now it is not the case with ipsec-secgw:
> > > > >
> > > > > [Anoob] Are you saying there is no h/w lookup involved when doing
> > > > > inline crypto processing? Then I'm confused. My understanding is
> > > > > that rte_flow
> > > > with ACTION type as SECURITY is being used to setup h/w lookups.
> > > >
> > > > I am saying that current ipsec-secgw code for each inbound ESP
> > > > packet
> > > > *always* does SW lookup.
> > > > No matter did HW lookup take place already and does information
> > > > about that lookup is available via mbuf in some way or not.
> > > > Just look at the ipsec-secgw code yourself:
> > > >
> > > > nb_rx = rte_eth_rx_burst(portid, queueid,  pkts, MAX_PKT_BURST);
> > > >
> > > > if (nb_rx > 0)
> > > > 	process_pkts(qconf, pkts, nb_rx, portid);
> > > >
> > > > Inside process_pkts()
> > > > https://urldefense.proofpoint.com/v2/url?u=http-
> > > > 3A__lxr.dpdk.org_dpdk_latest_ident_prepare-5Fone-
> > > >
> > 5Fpacket&d=DwIGaQ&c=nKjWec2b6R0mOyPaz7xtfQ&r=BPcGOOudUMrTDQ9Yb
> > > >
> > gKcOkO5ChYiUPPlPNIEvTOhjNE&m=KVBgvXYlUGyMmu260erISKr73Qt6PBLux0SI
> > > > oSO1i-M&s=opbGGHUqfxy8p_4NvwCG6od6VdV0qN8XafaPiubfO1A&e=
> > > >  it first calls prepare_traffic() which does sort of de-muxing:
> > > > put packet in one of 3 arrays:
> > > >    1) ESP packets
> > > >    2) non ESP IPv4 packets
> > > >    3) non ESP IPv6 packets
> > > > Also it checks is PKT_RX_SEC_OFFLOAD set for that packet.
> > > > If yes, it retrieves associated security user-data and stores it
> > > > inside private mbuf area.
> > > >
> > > > Then it goes into process_pkts_inbound()
> > > > https://urldefense.proofpoint.com/v2/url?u=http-
> > > > 3A__lxr.dpdk.org_dpdk_latest_ident_process-5Fpkts-
> > > >
> > 5Finbound&d=DwIGaQ&c=nKjWec2b6R0mOyPaz7xtfQ&r=BPcGOOudUMrTDQ9Y
> > > >
> > bgKcOkO5ChYiUPPlPNIEvTOhjNE&m=KVBgvXYlUGyMmu260erISKr73Qt6PBLux0S
> > > > IoSO1i-M&s=2OxMXug7jtUQPcYiDN4nW9WbrMtRfTJx5k-N7ikOeME&e=
> > > > and here for all ESP packets:
> > > > legacy code path:
> > > > ipsec_inbound()->inbound_sa_lookup()->single_inbound_lookup()
> > > > for librte_ipsec code path:
> > > > inbound_sa_lookup()->single_inbound_lookup()
> > > >
> > > > single_inbound_lookup() is the one that does actual SW lookup for
> > > > each input packet.
> > > > https://urldefense.proofpoint.com/v2/url?u=http-
> > > > 3A__lxr.dpdk.org_dpdk_latest_ident_single-5Finbound-
> > > >
> > 5Flookup&d=DwIGaQ&c=nKjWec2b6R0mOyPaz7xtfQ&r=BPcGOOudUMrTDQ9Yb
> > > >
> > gKcOkO5ChYiUPPlPNIEvTOhjNE&m=KVBgvXYlUGyMmu260erISKr73Qt6PBLux0SI
> > > > oSO1i-M&s=D2ZPZPLbcQSbaluG8KH0F3uUDHfQWNYxw4MOc0fSkI8&e=
> > > >
> > > > >
> > > > > > for each inbound ESP packet it *always* does a lookup in SW
> > > > > > based SADB, and if lookup fails - drops the packet (see 2.a above).
> > > > > > So, I still not see any additional lookups introduced with the patch.
> > > > >
> > > > > [Anoob] In case of inline crypto, you could do that. No disagreement.
> > > > > But that doesn't mean that is the only way. If PMDs can retrieve
> > > > > info about
> > > > h/w lookups and pass it on to the upper layers, isn't that the better
> > approach?
> > > >
> > > > Please let's keep our conversation in a constructive way.
> > > > We are not discussing what could be done in theory, but a particular
> > > > patch for
> > > > ipsec-secgw:
> > > > Right now ipsec-secgw does a SW lookup for each inbound ESP packet
> > > > (no matter what HW offload does) and this patch doesn't introduce
> > > > any extra SW lookups.
> > > >
> > > > If you are not happy with current ipsec-secgw approach, that's fine
> > > > - feel free to submit RFC/patch to fix that, or start a discussion in a new
> > thread.
> > > > I just don't see why we have to discuss it in context of this patch.
> > > >
> > > > >
> > > > > >
> > > > > > >
> > > > > > > > Also AFAIK, right now there is no possibility to configure
> > > > > > > > ipsec-secgw to BYPASS some ESP traffic.
> > > > > > > > Should we do it (to conform to ipsec RFC) - that's probably
> > > > > > > > subject of another discussion.
> > > > > > >
> > > > > > > [Anoob] The approach (choice of flags) forces a software-based
> > > > > > > SA lookup for packets that need to be bypassed instead of
> > > > > > > leveraging possible
> > > > > > hardware SA lookup. I don't think what ipsec-secgw does matters here.
> > > > > >
> > > > > > I do not agree with that statement.
> > > > > > First of all - current patch doesn't introduce any additional SW
> > > > > > lookups, see above.
> > > > > > Second, if someone would like to add BYPASS for ESP packets into
> > > > > > ipsec-secgw, I think he would have to insert new code that do
> > > > > > de-mux
> > > > > > *before* any ESP related processing starts.
> > > > > > Most obvious variant: at prepare_one_packet() when we split our
> > > > > > incoming traffic to IPsec and non-IPsec ones.
> > > > > > Second variant - at early stage of single_inbound_lookup(),
> > > > > > straight after actual SAD lookup failure.
> > > > > > In both cases, I don't see how session selection (primary or
> > > > > > faillback) would affect us here, actual decision do we want to
> > > > > > PROCESS or BYPASS particular ESP packet needs to take place before
> > that.
> > > > > > So I still believe we are ok here.
> > > > >
> > > > > [Anoob] I don't think you understood what I'm trying to highlight
> > > > > here. You could have a situation when h/w can detect that the
> > > > > packet has to be "PROTECT"ed using an "inline crypto" session. But
> > > > > then it might detect that it cannot proceed with inline processing
> > > > > because of some
> > > > issue (fragmentation, h/w queue overflow etc). Now the h/w has
> > > > already figured out the action to be done to the packet. If PMDs
> > > > allow this to be communicated to the application, the application won't be
> > required to do the lookup.
> > > > >
> > > > > In inline crypto, application can ignore the h/w lookup data and
> > > > > do a s/w lookup with the protocol headers as they are not stripped
> > > > > off. It was done this way, because API and the associated means to
> > > > > get this info from PMD was not introduced when inline crypto and
> > > > > corresponding
> > > > support in Intel's PMD was added. But inline protocol cannot do the
> > > > lookup in the application and so the concept of userdata etc was
> > > > introduced. The same can be applied to inline crypto also.
> > > > Advantage? It could remove the extra lookup done in s/w.
> > > > >
> > > > > To summarize, we cannot assume that every inline crypto packet
> > > > > need to looked up in the application to figure out the processing done on
> > the packet.
> > > > There can be applications which relies on h/w lookup data to figure
> > > > out the processing done.
> > > > >
> > > > > Example: Using rte_flow, I can create a rule for ESP packets to be
> > > > > MARKed. There is no security session created for the flow and the
> > > > > application intends to forward this flow packets to a different
> > > > > port. With your
> > > > approach, even these packets would be looked up. The packet will
> > > > have info from the h/w lookup which doesn't get used, because the
> > > > approach fails to introduce the required concepts.
> > > >
> > > > I think I understood your point.
> > > > Basically you saying that in future your PMD for unprocessed ESP
> > > > packets might provide some additional information (via
> > > > rte_security_get_userdata) that might be used by SW - let say to choose
> > BYPASS for such packets.
> > > > Current ipsec-secgw doesn't support such ability.
> > > > Your concern is that this patch will somehow prevent (or will make
> > > > it harder) for you to make your future changes.
> > > > Correct?
> > > > If, so then I said before, I don't think that concern is valid.
> > > > Most obvious variant where to add such code is inisde inside
> > > > prepare_one_packet() when we split our incoming traffic to IPsec and
> > > > non-IPsec ones.
> > > > To be more specific something like that
> > > >
> > > > if (eth->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) {
> > > >
> > > >                 iph4 = (const struct rte_ipv4_hdr *)rte_pktmbuf_adj(pkt,
> > > >                         RTE_ETHER_HDR_LEN);
> > > >                 adjust_ipv4_pktlen(pkt, iph4, 0);
> > > >
> > > > -                if (iph4->next_proto_id == IPPROTO_ESP)
> > > > +                if (iph4->next_proto_id == IPPROTO_ESP) {
> > > > +		if (pkt->ol_flags & PKT_RX_SEC_OFFLOAD) {
> > > > +			/* extract extra info, make decision based on that */
> > > > +		} else
> > > > 			t->ipsec.pkts[(t->ipsec.num)++] = pkt;
> > > > +	   }
> > > >
> > > > Of course some code reordering might be needed to avoid performance
> > > > drop, but I suppose the main idea is clear enough.
> > > >  Second variant - at early stage of single_inbound_lookup(), either
> > > > before or straight after actual SAD SW lookup failure.
> > > > In both cases, this new code will be executed *before* session selection
> > code.
> > > >
> > > > >
> > > > > Does the above sound like BYPASS? Yes.
> > > > > Does ipsec-secgw do it this way? No.
> > > >
> > > > Correct, it doesn't.
> > > > You have to submit patches if you'd like to support such ability.
> > > >
> > > > > Does this approach prevent an application from introducing such a
> > BYPASS?
> > > > Yes.
> > > >
> > > > That's not correct, I believe, see above.
> > > >
> > > > >
> > > > > >
> > > > > > >
> > > > > > > For example, ESN was not supported by ipsec-secgw (before
> > > > > > > library mode was introduced), but every single library change
> > > > > > > and application change was added keeping in mind that ESN is a
> > > > > > > valid feature for ipsec. So it is
> > > > > > one thing to say ipsec-secgw doesn't support ESP bypass and
> > > > > > saying the solution doesn't consider a possibility of ESP bypass.
> > > > > > >
> > > > > > > >
> > > > > > > > >
> > > > > > > > > 2. The solution proposed here seems like adding the
> > > > > > > > > handling in ipsec-secgw instead of ipsec library. In other
> > > > > > > > > words, this feature is not getting
> > > > > > > > added in ipsec library, which was supposed to simplify the
> > > > > > > > whole ipsec usage in DPDK, but fails to handle the case of
> > fragmentation.
> > > > > > > >
> > > > > > > > What we have right now with ipsec library is SA (low) level API.
> > > > > > > > It can handle multi-segment packets properly, but expects
> > > > > > > > someone else to do other steps (fragmentation/reassembly).
> > > > > > > > ipsec-secgw demonstrates how librte_ip_frag and librte_ipsec
> > > > > > > > can be used together to deal with fragmented IPsec traffic
> > > > > > > > in a proper
> > > > manner.
> > > > > > > > Probably in future we'll consider adding some high-level API
> > > > > > > > that will be able to do whole processing under hood (SPD/SAD
> > > > > > > > lookup, fragment/reassembly, actual IPsec packet processing,
> > > > > > > > matching inbound selectors, etc.), but right now it is not the case.
> > > > > > > >
> > > > > > > > > Also, since the fallback feature is entirely done in the
> > > > > > > > > application, it begs the
> > > > > > > > question why the same feature is omitted for legacy use case.
> > > > > > > >
> > > > > > > > Because legacy mode doesn't support multi-seg packets at first place.
> > > > > > > > Also it is an extra overhead to support 2 code-paths (legacy
> > > > > > > > and
> > > > > > > > library) for the same app, so we'd like in future to
> > > > > > > > deprecate and later remove legacy code- path.
> > > > > > > > As a first step we propose to make library code-path a default one:
> > > > > > > > https://urldefense.proofpoint.com/v2/url?u=http-3A__patches.
> > > > > > > > dpdk
> > > > > > > > .org
> > > > > > > >
> > > > > >
> > > >
> > _cover_58247_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=jPfB8rwwviRSxyL
> > > > > > Ws2
> > > > > > > > n6B-WYLn1v9SyTMrT5EQqh2TU&m=b3E429fuo8P-K5XfH8wG-
> > > > > > 7hwr1d8oM4uJGErAkbf
> > > > > > > > DvA&s=vz7ioUzJOuzoJdmV7QO0QLPKYn1ytFsj_0eYatbSCrg&e=
> > > > > > >
> > > > > > > [Anoob] Even if we need the application to do the reassembly,
> > > > > > > it would look simple if application uses the same "rte_ipsec_session"
> > > > > > > to process the
> > > > > > fallback packet.
> > > > > >
> > > > > > I think there is some sort of misunderstanding here.
> > > > > > With current librte_ipsec design:
> > > > > > rte_ipsec_sa - opaque SW representation of the SA (HW neutral)
> > > > > > rte_ipsec_session associates SA (rte_ipsec_sa)with particular HW
> > > > > > device (session).
> > > > > > Same SA can be referred by multiple sessions.
> > > > > >
> > > > > > From rte_ipsec.h:
> > > > > > /**
> > > > > >  * rte_ipsec_session is an aggregate structure that defines
> > > > > > particular
> > > > > >  * IPsec Security Association IPsec (SA) on given security/crypto device:
> > > > > >  * - pointer to the SA object
> > > > > >  * - security session action type
> > > > > >  * - pointer to security/crypto session, plus other related data
> > > > > >  * - session/device specific functions to prepare/process IPsec packets.
> > > > > >  */
> > > > > >
> > > > > > > In that case, the processing required to handle the other
> > > > > > > packet will be hidden from the application. Here application
> > > > > > > has to make sure it chooses
> > > > > > the correct session, which I believe should be moved to ipsec library.
> > > > > >
> > > > > > From my side it is a good thing to let application define its own usage
> > model.
> > > > > > I.E. librte_ipsec supports multiple-sessions per SA, upper-layer
> > > > > > (app) decides how it will use that feature.
> > > > > > Though if you strongly feel that some higher-level API is needed
> > > > > > here, and have some good ideas about it  - please go ahead and
> > > > > > submit RFC
> > > > for it.
> > > > >
> > > > > [Anoob] If you are okay with application defining how it
> > > > > implements ipsec,
> > > > then probably there is no use case for lib ipsec?
> > > >
> > > > :)
> > > > Once again - library provides an implementation.
> > > > Application defines the way to use it, i.e. how it will apply
> > > > functionality library provides for different use-case scenarios.
> > > > As an example: snpritnf() provides user with ability to format
> > > > strings, application decides what exactly format to use and for which string.
> > > > Same for librte_ipsec, library provides functionality to perform
> > > > processing for ESP packets.
> > > > Application defines which packets to process and what session to use.
> > > >
> > > > >
> > > > > >
> > > > > > >
> > > > > > > >
> > > > > > > > >
> > > > > > > > > 3. It seems like ordering won't be maintained once this
> > > > > > > > > processing is done. Again, this is the sequence I understood.
> > > > > > > > > Please correct me if I missed
> > > > > > > > something,
> > > > > > > > >        a. Application receives a bunch of packets (let's
> > > > > > > > > say 6 packets), in which few are fragmented (P3 & P4) and
> > > > > > > > > the rest can be inline
> > > > > > > > processed.
> > > > > > > > >        b. Application receives P1->P2->P3->P4->P5->P6 (in
> > > > > > > > > this, P1, P2, P5, P6 are
> > > > > > > > inline processed successfully) and P4 & P5 are the fragments
> > > > > > > > >        c. Application groups packets. P1->P2->P5->P6
> > > > > > > > >becomes one group and P3-
> > > > > > > > >P4 becomes another and goes for fallback processing.
> > > > > > > > > Now how is ordering maintained? I couldn't figure out how
> > > > > > > > >that is done in this
> > > > > > > > case.
> > > > > > > >
> > > > > > > > You right, fallback session can introduce packet reordering.
> > > > > > > > At least till we'll have ability to process packets in sync mode too.
> > > > > > > > See our presentation at dpdk userspace (slides 17, 18):
> > > > > > > > https://urldefense.proofpoint.com/v2/url?u=https-3A__static.
> > > > > > > > sche
> > > > > > > > d.co
> > > > > > > > m_hosted-5Ffiles_dpdkbordeaux2019_8f_DPDK-
> > > > > > 2DIPSECu9.pdf&d=DwIFAg&c=n
> > > > > > > > KjWec2b6R0mOyPaz7xtfQ&r=jPfB8rwwviRSxyLWs2n6B-
> > > > > > WYLn1v9SyTMrT5EQqh2TU&
> > > > > > > > m=b3E429fuo8P-K5XfH8wG-
> > > > > > 7hwr1d8oM4uJGErAkbfDvA&s=MLRnnYcykjnsqXrHGUuR
> > > > > > > > YHc5uDxaAod0Yl7f06EQr1M&e= Right now the only way to deal
> > > > > > > > with it - have replay window big enough to sustain
> > > > > > > > reordering and async processing latency.
> > > > > > > > That's actually another reason why we add this feature into
> > > > > > > > ipsec-secgw sample
> > > > > > > > app:
> > > > > > > > so people can evaluate it on their platforms, determine what
> > > > > > > > replay window size would be needed, what issues/slowdowns it
> > > > > > > > might
> > > > cause, etc.
> > > > > > >
> > > > > > > [Anoob] This is something I had noticed while going through the code
> > flow.
> > > > > > The ordering info is lost because of the approach.
> > > > > >
> > > > > > Once again, it is a known limitation and we are not trying to
> > > > > > hide it from you :)
> > > > >
> > > > > [Anoob] Never said it was hidden. Initially I had okayed the
> > > > > approach even though it wasn't a solution suitable for inline
> > > > > protocol. But as more cases
> > > > were considered, few shortcomings were observed, and I was skeptical
> > > > about how any of that would be solved in the long run.
> > > > >
> > > > > My suggestion, if these limitations and the plans to address it
> > > > > were mentioned
> > > > in the cover letter, we could've saved few cycles here.
> > > >
> > > > Point taken.
> > > >
> > > > > But my reply would still be the same 😊
> > > > >
> > > > > > It
> > > > > > was outlined here:
> > > > > > https://urldefense.proofpoint.com/v2/url?u=https-
> > > > > > 3A__static.sched.com_hosted-5Ffiles_dpdkbordeaux2019_8f_DPDK-
> > > > > >
> > > >
> > 2DIPSECu9.pdf&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=jPfB8rwwviRSxyL
> > > > > > Ws2n6B-WYLn1v9SyTMrT5EQqh2TU&m=b3E429fuo8P-K5XfH8wG-
> > > > > >
> > > >
> > 7hwr1d8oM4uJGErAkbfDvA&s=MLRnnYcykjnsqXrHGUuRYHc5uDxaAod0Yl7f06E
> > > > > > Qr1M&e=
> > > > > > And in the latest patch version Marcin clearly stated it inside the AG
> > session:
> > > > > > https://urldefense.proofpoint.com/v2/url?u=http-
> > > > > >
> > > >
> > 3A__patches.dpdk.org_patch_60039_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtf
> > > > > > Q&r=jPfB8rwwviRSxyLWs2n6B-
> > > > WYLn1v9SyTMrT5EQqh2TU&m=b3E429fuo8P-
> > > > > > K5XfH8wG-
> > > > > >
> > > >
> > 7hwr1d8oM4uJGErAkbfDvA&s=zrn91Vjf_ZElAlDf7IeZGmGevcA6RMwpPTLUCGlgZ
> > > > > > cY&e=
> > > > > > If you think even further clarification within the doc is
> > > > > > needed, please let us know.
> > > > > >
> > > > > > About the actual packet re-oridering:
> > > > > >  AFAIK, for some use-cases it might be acceptable, for others not.
> > > > > > Though it is an optional feature, that wouldn't be enabled by
> > > > > > default, so user can always choose is it does he needs/wants
> > > > > > this feature or
> > > > not.
> > > > > > If/when we'll have CPU_CRYPTO processing mode:
> > > > >
> > > > > [Anoob] Shouldn't we get that accepted first then?
> > > >
> > > > I don't think so.
> > > > Current implementation does provide expected functionality (with
> > > > known limitations).
> > > > In future, we can try to improve it and/or remove existing limitations.
> > > > That's a common iteration development approach that is used though
> > > > whole
> > > > DPDK:
> > > > - provide initial solution with basic functionality first
> > > > - improve/extend with future releases/patches
> > > >
> > > > >
> > > > > Also, the ordering is lost because application is not considering
> > > > > that. Why can't
> > > > we use re-ordering library? Or an event-dev?
> > > >
> > > > No-one saying it can't be improved in one way or another.
> > > > We don't plan to introduce such code right now due to its complexity.
> > > > Might be something in future.
> > > > Though you are welcome to go ahead and submit your own patches to
> > > > improve that solution.
> > > >
> > > > >
> > > > > > https://urldefense.proofpoint.com/v2/url?u=http-
> > > > > >
> > > >
> > 3A__patches.dpdk.org_cover_58862_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtf
> > > > > > Q&r=jPfB8rwwviRSxyLWs2n6B-
> > > > WYLn1v9SyTMrT5EQqh2TU&m=b3E429fuo8P-
> > > > > > K5XfH8wG-
> > > > > >
> > > >
> > 7hwr1d8oM4uJGErAkbfDvA&s=k1zef_1uqELXblKN3_qjX04WXNFGAUEFIIJrvKPr6
> > > > > > eA&e=
> > > > > > we'll add ability for the user to avoid this reordering problem
> > > > > >
> > > > > > > As we dig deeper, the feature looks hardly complete. The
> > > > > > > concerning part is the approach doesn't seem conducive to
> > > > > > > fixing any of these issues in the
> > > > > > future.
> > > > > >
> > > > > > Again, don't agree with that statement.
> > > > > > From my perspective: while current implementation has its
> > > > > > limitations (packet reordering, inline-proto/lksd-proto), it is
> > > > > > totally functional and provides users with a reference how to
> > > > > > deal with such
> > > > kind of problems.
> > > > >
> > > > > [Anoob] Well, that is exactly my problem. It gives a skewed
> > > > > reference on how to deal with the problem, without considering other
> > possibilities.
> > > >
> > > > Once again, please feel free and come-up with your own patches, that
> > > > will address this problem in a smarter way.
> > > >
> > > > > If every application starts adopting this method and starts doing
> > > > > lookup for every ESP packet, h/w based lookups (and the associated
> > > > > h/w) would be rendered useless. I really think this series, which
> > > > > happens to be
> > > > the first step in solving a complex problem, will be very useful if
> > > > all these issues are addressed properly.
> > > > >
> > > > > If you can introduce the inline+lookaside via librte_ipsec, its
> > > > > perhaps ok for the patch to have some limitations(like lack of
> > > > > ordering,
> > > > assumptions on ESP etc). However, if the patch is introduced
> > > > directly into ipsec- secgw, it needs to be more comprehensive and robust.
> > > >
> > > > I think common DPDK policy is exactly opposite:
> > > > library/driver has to be as robust and comprehensive as possible.
> > > > sample app code allowed to have some limitations (as long as they
> > > > are documented, etc.).
> > > >
> > > > > I also expect performance impact due to these changes
> > > >
> > > > Could you be more specific here?
> > > > What performance impact do you expect and why?
> > > > Can you point to the exact piece of patch code that you believe
> > > > would cause a degradation.
> > > > I don't expect any and performance tests on our platforms didn't
> > > > show any regression so far.
> > > >
> > > > > and so I prefer if the inline+lookaside can be made separate
> > > > > datapath rather than overloading the existing one.
> > > >
> > > > I don't see any good reason for that.
> > > > This feature is optional.
> > > > User have to enable it manually for each session (via config file).
> > > > for all other sessions there is no impact.
> > > > What we probably can do - add extra check inside sa.c to allow
> > > > fallback sessions only for inline-crypto-offload type of sessions.
> > > >
> > > > >
> > > > > >
> > > > > > >
> > > > > > > Also, if the new ipsec related features are introduced via
> > > > > > > ipsec-secgw than via rte_ipsec, then it raises doubts over the
> > > > > > > utility of
> > > > rte_ipsec library.
> > > > > >
> > > > > >

[dpdk-dev] [PATCH v4 0/4] add fallback session

[Marcin Smoczynski]

Add fallback session feature allowing to process packets that inline
processor is unable to handle (e.g. fragmented traffic). Processing
takes place in a secondary session defined for SA in a configuration
file.

This feature is limited to ingress IPsec traffic only. IPsec
anti-replay window and ESN are supported in conjunction with fallback
session when following conditions are met:
 * primary session is 'inline-crypto-offload,
 * fallback sessions is 'lookaside-none'.
Due to different processing times of inline and lookaside modes,
fallback session introduces some packet reordering, therefore when
using with IPsec window, its value should be increased.

v3 to v4 changes:
 - add info about packet reordering to the documentation regarding
   fallback session
 - add patch with --frag-ttl command line option which allows to change
   fragment lifetime

v2 to v3 changes:
 - doc and commit log update - explicitly state feature limitations

v1 to v2 changes:
 - disable fallback offload for outbound SAs
 - add test scripts

Marcin Smoczynski (4):
  examples/ipsec-secgw: ipsec_sa structure cleanup
  examples/ipsec-secgw: add fallback session feature
  examples/ipsec-secgw: add frag TTL cmdline option
  examples/ipsec-secgw: add offload fallback tests

 doc/guides/sample_app_ug/ipsec_secgw.rst      |  31 +++-
 examples/ipsec-secgw/esp.c                    |  35 ++--
 examples/ipsec-secgw/ipsec-secgw.c            |  56 ++++--
 examples/ipsec-secgw/ipsec.c                  |  99 ++++++-----
 examples/ipsec-secgw/ipsec.h                  |  61 +++++--
 examples/ipsec-secgw/ipsec_process.c          | 113 +++++++-----
 examples/ipsec-secgw/sa.c                     | 164 +++++++++++++-----
 .../test/trs_aesgcm_common_defs.sh            |   4 +-
 .../trs_aesgcm_inline_crypto_fallback_defs.sh |   5 +
 .../test/tun_aesgcm_common_defs.sh            |   6 +-
 .../tun_aesgcm_inline_crypto_fallback_defs.sh |   5 +
 11 files changed, 402 insertions(+), 177 deletions(-)
 create mode 100644 examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
 create mode 100644 examples/ipsec-secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh

-- 
2.17.1

[dpdk-dev] [PATCH v4 1/4] examples/ipsec-secgw: ipsec_sa structure cleanup

[Marcin Smoczynski]

Cleanup ipsec_sa structure by removing every field that is already in
the rte_ipsec_session structure:
 * cryptodev/security session union
 * action type
 * offload flags
 * security context
References to abovementioned fields are changed to direct references
to matching fields of rte_ipsec_session structure.

Such refactoring is needed to introduce many sessions per SA feature,
e.g. fallback session for inline offload processing.

Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
---
 examples/ipsec-secgw/esp.c           | 35 +++++++----
 examples/ipsec-secgw/ipsec.c         | 91 +++++++++++++++-------------
 examples/ipsec-secgw/ipsec.h         | 27 ++++++---
 examples/ipsec-secgw/ipsec_process.c | 30 ++++-----
 examples/ipsec-secgw/sa.c            | 66 ++++++++++----------
 5 files changed, 137 insertions(+), 112 deletions(-)

diff --git a/examples/ipsec-secgw/esp.c b/examples/ipsec-secgw/esp.c
index d6d7b1256..c1b49da1e 100644
--- a/examples/ipsec-secgw/esp.c
+++ b/examples/ipsec-secgw/esp.c
@@ -30,7 +30,8 @@ esp_inbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 	int32_t payload_len, ip_hdr_len;
 
 	RTE_ASSERT(sa != NULL);
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
+	if (ipsec_get_action_type(sa) ==
+			RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
 		return 0;
 
 	RTE_ASSERT(m != NULL);
@@ -148,13 +149,16 @@ esp_inbound_post(struct rte_mbuf *m, struct ipsec_sa *sa,
 	uint8_t *nexthdr, *pad_len;
 	uint8_t *padding;
 	uint16_t i;
+	struct rte_ipsec_session *ips;
 
 	RTE_ASSERT(m != NULL);
 	RTE_ASSERT(sa != NULL);
 	RTE_ASSERT(cop != NULL);
 
-	if ((sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
-			(sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
+	ips = ipsec_get_session(sa);
+
+	if ((ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
+			(ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
 		if (m->ol_flags & PKT_RX_SEC_OFFLOAD) {
 			if (m->ol_flags & PKT_RX_SEC_OFFLOAD_FAILED)
 				cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
@@ -169,8 +173,8 @@ esp_inbound_post(struct rte_mbuf *m, struct ipsec_sa *sa,
 		return -1;
 	}
 
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO &&
-	    sa->ol_flags & RTE_SECURITY_RX_HW_TRAILER_OFFLOAD) {
+	if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO &&
+		ips->security.ol_flags & RTE_SECURITY_RX_HW_TRAILER_OFFLOAD) {
 		nexthdr = &m->inner_esp_next_proto;
 	} else {
 		nexthdr = rte_pktmbuf_mtod_offset(m, uint8_t*,
@@ -225,10 +229,12 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 	struct rte_crypto_sym_op *sym_cop;
 	int32_t i;
 	uint16_t pad_payload_len, pad_len, ip_hdr_len;
+	struct rte_ipsec_session *ips;
 
 	RTE_ASSERT(m != NULL);
 	RTE_ASSERT(sa != NULL);
 
+	ips = ipsec_get_session(sa);
 	ip_hdr_len = 0;
 
 	ip4 = rte_pktmbuf_mtod(m, struct ip *);
@@ -277,9 +283,10 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 	}
 
 	/* Add trailer padding if it is not constructed by HW */
-	if (sa->type != RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
-	    (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO &&
-	     !(sa->ol_flags & RTE_SECURITY_TX_HW_TRAILER_OFFLOAD))) {
+	if (ips->type != RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+		(ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO &&
+		 !(ips->security.ol_flags &
+			 RTE_SECURITY_TX_HW_TRAILER_OFFLOAD))) {
 		padding = (uint8_t *)rte_pktmbuf_append(m, pad_len +
 							sa->digest_len);
 		if (unlikely(padding == NULL)) {
@@ -344,8 +351,9 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 		}
 	}
 
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
-		if (sa->ol_flags & RTE_SECURITY_TX_HW_TRAILER_OFFLOAD) {
+	if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		if (ips->security.ol_flags &
+				RTE_SECURITY_TX_HW_TRAILER_OFFLOAD) {
 			/* Set the inner esp next protocol for HW trailer */
 			m->inner_esp_next_proto = nlp;
 			m->packet_type |= RTE_PTYPE_TUNNEL_ESP;
@@ -448,11 +456,14 @@ esp_outbound_post(struct rte_mbuf *m,
 		  struct ipsec_sa *sa,
 		  struct rte_crypto_op *cop)
 {
+	enum rte_security_session_action_type type;
 	RTE_ASSERT(m != NULL);
 	RTE_ASSERT(sa != NULL);
 
-	if ((sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
-			(sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
+	type = ipsec_get_action_type(sa);
+
+	if ((type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
+			(type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
 		m->ol_flags |= PKT_TX_SEC_OFFLOAD;
 	} else {
 		RTE_ASSERT(cop != NULL);
diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index dc85adfe5..8c60bd84b 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -40,7 +40,8 @@ set_ipsec_conf(struct ipsec_sa *sa, struct rte_security_ipsec_xform *ipsec)
 }
 
 int
-create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips)
 {
 	struct rte_cryptodev_info cdev_info;
 	unsigned long cdev_id_qp = 0;
@@ -71,9 +72,9 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			ipsec_ctx->tbl[cdev_id_qp].id,
 			ipsec_ctx->tbl[cdev_id_qp].qp);
 
-	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
+	if (ips->type != RTE_SECURITY_ACTION_TYPE_NONE) {
 		struct rte_security_session_conf sess_conf = {
-			.action_type = sa->type,
+			.action_type = ips->type,
 			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 			{.ipsec = {
 				.spi = sa->spi,
@@ -90,7 +91,7 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 
 		};
 
-		if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
+		if (ips->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
 			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
 							rte_cryptodev_get_sec_ctx(
 							ipsec_ctx->tbl[cdev_id_qp].id);
@@ -98,9 +99,9 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			/* Set IPsec parameters in conf */
 			set_ipsec_conf(sa, &(sess_conf.ipsec));
 
-			sa->sec_session = rte_security_session_create(ctx,
+			ips->security.ses = rte_security_session_create(ctx,
 					&sess_conf, ipsec_ctx->session_priv_pool);
-			if (sa->sec_session == NULL) {
+			if (ips->security.ses == NULL) {
 				RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 				return -1;
@@ -110,10 +111,10 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			return -1;
 		}
 	} else {
-		sa->crypto_session = rte_cryptodev_sym_session_create(
+		ips->crypto.ses = rte_cryptodev_sym_session_create(
 				ipsec_ctx->session_pool);
 		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
-				sa->crypto_session, sa->xforms,
+				ips->crypto.ses, sa->xforms,
 				ipsec_ctx->session_priv_pool);
 
 		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
@@ -126,12 +127,13 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 }
 
 int
-create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips)
 {
 	int32_t ret = 0;
 	struct rte_security_ctx *sec_ctx;
 	struct rte_security_session_conf sess_conf = {
-		.action_type = sa->type,
+		.action_type = ips->type,
 		.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 		{.ipsec = {
 			.spi = sa->spi,
@@ -151,7 +153,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on port %u\n",
 		sa->spi, sa->portid);
 
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+	if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
 		struct rte_flow_error err;
 		const struct rte_security_capability *sec_cap;
 		int ret = 0;
@@ -165,9 +167,9 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 			return -1;
 		}
 
-		sa->sec_session = rte_security_session_create(sec_ctx,
+		ips->security.ses = rte_security_session_create(sec_ctx,
 				&sess_conf, skt_ctx->session_pool);
-		if (sa->sec_session == NULL) {
+		if (ips->security.ses == NULL) {
 			RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 			return -1;
@@ -177,7 +179,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 
 		/* iterate until ESP tunnel*/
 		while (sec_cap->action != RTE_SECURITY_ACTION_TYPE_NONE) {
-			if (sec_cap->action == sa->type &&
+			if (sec_cap->action == ips->type &&
 			    sec_cap->protocol ==
 				RTE_SECURITY_PROTOCOL_IPSEC &&
 			    sec_cap->ipsec.mode ==
@@ -193,8 +195,8 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 			return -1;
 		}
 
-		sa->ol_flags = sec_cap->ol_flags;
-		sa->security_ctx = sec_ctx;
+		ips->security.ol_flags = sec_cap->ol_flags;
+		ips->security.ctx = sec_ctx;
 		sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
 
 		if (IS_IP6(sa->flags)) {
@@ -223,7 +225,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 		sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
 
 		sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
-		sa->action[0].conf = sa->sec_session;
+		sa->action[0].conf = ips->security.ses;
 
 		sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
 
@@ -282,8 +284,8 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 			if (ret)
 				goto flow_create_failure;
 		} else if (sa->attr.egress &&
-			   (sa->ol_flags &
-				    RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
+				(ips->security.ol_flags &
+					RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
 			sa->action[1].type =
 					RTE_FLOW_ACTION_TYPE_PASSTHRU;
 			sa->action[2].type =
@@ -299,7 +301,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 				err.message);
 			return -1;
 		}
-	} else if (sa->type ==	RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
+	} else if (ips->type ==	RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
 		const struct rte_security_capability *sec_cap;
 
 		sec_ctx = (struct rte_security_ctx *)
@@ -327,9 +329,9 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 
 		sess_conf.userdata = (void *) sa;
 
-		sa->sec_session = rte_security_session_create(sec_ctx,
+		ips->security.ses = rte_security_session_create(sec_ctx,
 					&sess_conf, skt_ctx->session_pool);
-		if (sa->sec_session == NULL) {
+		if (ips->security.ses == NULL) {
 			RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 			return -1;
@@ -345,7 +347,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 		/* iterate until ESP tunnel*/
 		while (sec_cap->action !=
 				RTE_SECURITY_ACTION_TYPE_NONE) {
-			if (sec_cap->action == sa->type &&
+			if (sec_cap->action == ips->type &&
 			    sec_cap->protocol ==
 				RTE_SECURITY_PROTOCOL_IPSEC &&
 			    sec_cap->ipsec.mode ==
@@ -361,8 +363,8 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 			return -1;
 		}
 
-		sa->ol_flags = sec_cap->ol_flags;
-		sa->security_ctx = sec_ctx;
+		ips->security.ol_flags = sec_cap->ol_flags;
+		ips->security.ctx = sec_ctx;
 	}
 	sa->cdev_id_qp = 0;
 
@@ -409,6 +411,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 	struct ipsec_mbuf_metadata *priv;
 	struct rte_crypto_sym_op *sym_cop;
 	struct ipsec_sa *sa;
+	struct rte_ipsec_session *ips;
 
 	for (i = 0; i < nb_pkts; i++) {
 		if (unlikely(sas[i] == NULL)) {
@@ -422,16 +425,17 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 		priv = get_priv(pkts[i]);
 		sa = sas[i];
 		priv->sa = sa;
+		ips = ipsec_get_session(sa);
 
-		switch (sa->type) {
+		switch (ips->type) {
 		case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL:
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
 
-			if ((unlikely(sa->sec_session == NULL)) &&
-				create_lookaside_session(ipsec_ctx, sa)) {
+			if ((unlikely(ips->security.ses == NULL)) &&
+				create_lookaside_session(ipsec_ctx, sa, ips)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -440,7 +444,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			sym_cop->m_src = pkts[i];
 
 			rte_security_attach_session(&priv->cop,
-					sa->sec_session);
+				ips->security.ses);
 			break;
 		case RTE_SECURITY_ACTION_TYPE_NONE:
 
@@ -449,14 +453,14 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			rte_prefetch0(&priv->sym_cop);
 
-			if ((unlikely(sa->crypto_session == NULL)) &&
-				create_lookaside_session(ipsec_ctx, sa)) {
+			if ((unlikely(ips->crypto.ses == NULL)) &&
+				create_lookaside_session(ipsec_ctx, sa, ips)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
 
 			rte_crypto_op_attach_sym_session(&priv->cop,
-					sa->crypto_session);
+					ips->crypto.ses);
 
 			ret = xform_func(pkts[i], sa, &priv->cop);
 			if (unlikely(ret)) {
@@ -465,21 +469,22 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 			break;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
-			RTE_ASSERT(sa->sec_session != NULL);
+			RTE_ASSERT(ips->security.ses != NULL);
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
-			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
+			if (ips->security.ol_flags &
+				RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
-						sa->security_ctx,
-						sa->sec_session, pkts[i], NULL);
+					ips->security.ctx, ips->security.ses,
+					pkts[i], NULL);
 			continue;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
-			RTE_ASSERT(sa->sec_session != NULL);
+			RTE_ASSERT(ips->security.ses != NULL);
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
 			rte_security_attach_session(&priv->cop,
-					sa->sec_session);
+					ips->security.ses);
 
 			ret = xform_func(pkts[i], sa, &priv->cop);
 			if (unlikely(ret)) {
@@ -488,10 +493,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
-			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
+			if (ips->security.ol_flags &
+				RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
-						sa->security_ctx,
-						sa->sec_session, pkts[i], NULL);
+					ips->security.ctx, ips->security.ses,
+					pkts[i], NULL);
 			continue;
 		}
 
@@ -560,7 +566,8 @@ ipsec_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			RTE_ASSERT(sa != NULL);
 
-			if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {
+			if (ipsec_get_action_type(sa) ==
+				RTE_SECURITY_ACTION_TYPE_NONE) {
 				ret = xform_func(pkt, sa, cops[j]);
 				if (unlikely(ret)) {
 					rte_pktmbuf_free(pkt);
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index 1efa6e488..a4ad81b0e 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -87,10 +87,6 @@ struct ipsec_sa {
 	uint32_t cdev_id_qp;
 	uint64_t seq;
 	uint32_t salt;
-	union {
-		struct rte_cryptodev_sym_session *crypto_session;
-		struct rte_security_session *sec_session;
-	};
 	enum rte_crypto_cipher_algorithm cipher_algo;
 	enum rte_crypto_auth_algorithm auth_algo;
 	enum rte_crypto_aead_algorithm aead_algo;
@@ -114,11 +110,8 @@ struct ipsec_sa {
 		struct rte_crypto_sym_xform *xforms;
 		struct rte_security_ipsec_xform *sec_xform;
 	};
-	enum rte_security_session_action_type type;
 	enum rte_security_ipsec_sa_direction direction;
 	uint16_t portid;
-	struct rte_security_ctx *security_ctx;
-	uint32_t ol_flags;
 
 #define MAX_RTE_FLOW_PATTERN (4)
 #define MAX_RTE_FLOW_ACTIONS (3)
@@ -284,6 +277,20 @@ get_sym_cop(struct rte_crypto_op *cop)
 	return (cop + 1);
 }
 
+static inline struct rte_ipsec_session *
+ipsec_get_session(struct ipsec_sa *sa)
+{
+	return &sa->ips;
+}
+
+static inline enum rte_security_session_action_type
+ipsec_get_action_type(struct ipsec_sa *sa)
+{
+	struct rte_ipsec_session *ips;
+	ips = ipsec_get_session(sa);
+	return ips->type;
+}
+
 int
 inbound_sa_check(struct sa_ctx *sa_ctx, struct rte_mbuf *m, uint32_t sa_idx);
 
@@ -338,9 +345,11 @@ void
 enqueue_cop_burst(struct cdev_qp *cqp);
 
 int
-create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips);
 
 int
-create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa);
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips);
 
 #endif /* __IPSEC_H__ */
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index 868f1a28d..239d81ef6 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -94,22 +94,16 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
 
 	/* setup crypto section */
 	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		if (sa->crypto_session == NULL) {
-			rc = create_lookaside_session(ctx, sa);
-			if (rc != 0)
-				return rc;
-		}
-		ss->crypto.ses = sa->crypto_session;
+		RTE_ASSERT(ss->crypto.ses == NULL);
+		rc = create_lookaside_session(ctx, sa, ss);
+		if (rc != 0)
+			return rc;
 	/* setup session action type */
-	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
-		if (sa->sec_session == NULL) {
-			rc = create_lookaside_session(ctx, sa);
-			if (rc != 0)
-				return rc;
-		}
-		ss->security.ses = sa->sec_session;
-		ss->security.ctx = sa->security_ctx;
-		ss->security.ol_flags = sa->ol_flags;
+	} else if (ss->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
+		RTE_ASSERT(ss->security.ses == NULL);
+		rc = create_lookaside_session(ctx, sa, ss);
+		if (rc != 0)
+			return rc;
 	} else
 		RTE_ASSERT(0);
 
@@ -218,7 +212,7 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 		pg = grp + i;
 		sa = pg->id.ptr;
 
-		ips = &sa->ips;
+		ips = ipsec_get_session(sa);
 
 		/* no valid HW session for that SA, try to create one */
 		if (sa == NULL || (ips->crypto.ses == NULL &&
@@ -226,8 +220,8 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 			k = 0;
 
 		/* process packets inline */
-		else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
-				sa->type ==
+		else if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+				ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
 
 			satp = rte_ipsec_sa_type(ips->sa);
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index c3cf3bd1f..a3d8e4545 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -224,6 +224,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	struct parse_status *status)
 {
 	struct ipsec_sa *rule = NULL;
+	struct rte_ipsec_session *ips;
 	uint32_t ti; /*token index*/
 	uint32_t *ri /*rule index*/;
 	uint32_t cipher_algo_p = 0;
@@ -262,6 +263,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	if (atoi(tokens[1]) == INVALID_SPI)
 		return;
 	rule->spi = atoi(tokens[1]);
+	ips = ipsec_get_session(rule);
 
 	for (ti = 2; ti < n_tokens; ti++) {
 		if (strcmp(tokens[ti], "mode") == 0) {
@@ -558,18 +560,18 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 				return;
 
 			if (strcmp(tokens[ti], "inline-crypto-offload") == 0)
-				rule->type =
+				ips->type =
 					RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO;
 			else if (strcmp(tokens[ti],
 					"inline-protocol-offload") == 0)
-				rule->type =
+				ips->type =
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL;
 			else if (strcmp(tokens[ti],
 					"lookaside-protocol-offload") == 0)
-				rule->type =
+				ips->type =
 				RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL;
 			else if (strcmp(tokens[ti], "no-offload") == 0)
-				rule->type = RTE_SECURITY_ACTION_TYPE_NONE;
+				ips->type = RTE_SECURITY_ACTION_TYPE_NONE;
 			else {
 				APP_CHECK(0, status, "Invalid input \"%s\"",
 						tokens[ti]);
@@ -624,11 +626,11 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	if (status->status < 0)
 		return;
 
-	if ((rule->type != RTE_SECURITY_ACTION_TYPE_NONE) && (portid_p == 0))
+	if ((ips->type != RTE_SECURITY_ACTION_TYPE_NONE) && (portid_p == 0))
 		printf("Missing portid option, falling back to non-offload\n");
 
 	if (!type_p || !portid_p) {
-		rule->type = RTE_SECURITY_ACTION_TYPE_NONE;
+		ips->type = RTE_SECURITY_ACTION_TYPE_NONE;
 		rule->portid = -1;
 	}
 
@@ -640,6 +642,7 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 {
 	uint32_t i;
 	uint8_t a, b, c, d;
+	const struct rte_ipsec_session *ips;
 
 	printf("\tspi_%s(%3u):", inbound?"in":"out", sa->spi);
 
@@ -695,8 +698,10 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 		printf("Transport ");
 		break;
 	}
+
+	ips = &sa->ips;
 	printf(" type:");
-	switch (sa->type) {
+	switch (ips->type) {
 	case RTE_SECURITY_ACTION_TYPE_NONE:
 		printf("no-offload ");
 		break;
@@ -876,6 +881,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 	uint16_t iv_length, aad_length;
 	int inline_status;
 	int32_t rc;
+	struct rte_ipsec_session *ips;
 
 	/* for ESN upper 32 bits of SQN also need to be part of AAD */
 	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
@@ -890,9 +896,10 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 		}
 		*sa = entries[i];
 		sa->seq = 0;
+		ips = ipsec_get_session(sa);
 
-		if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
-			sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
+			ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
 			if (check_eth_dev_caps(sa->portid, inbound))
 				return -EINVAL;
 		}
@@ -907,7 +914,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 			sa->dst.ip.ip4 = rte_cpu_to_be_32(sa->dst.ip.ip4);
 			break;
 		case TRANSPORT:
-			if (sa->type ==
+			if (ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
 				inline_status =
 					sa_add_address_inline_crypto(sa);
@@ -918,6 +925,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 		}
 
 		if (sa->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) {
+			struct rte_ipsec_session *ips;
 			iv_length = 16;
 
 			sa_ctx->xf[idx].a.type = RTE_CRYPTO_SYM_XFORM_AEAD;
@@ -938,11 +946,12 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			sa->xforms = &sa_ctx->xf[idx].a;
 
-			if (sa->type ==
+			ips = ipsec_get_session(sa);
+			if (ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
-				sa->type ==
+				ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
-				rc = create_inline_session(skt_ctx, sa);
+				rc = create_inline_session(skt_ctx, sa, ips);
 				if (rc != 0) {
 					RTE_LOG(ERR, IPSEC_ESP,
 						"create_inline_session() failed\n");
@@ -1100,23 +1109,11 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm, const struct ipsec_sa *ss,
 }
 
 static int
-fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
-	const struct ipsec_sa *lsa)
+fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa)
 {
 	int32_t rc = 0;
 
 	ss->sa = sa;
-	ss->type = lsa->type;
-
-	/* setup crypto section */
-	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		ss->crypto.ses = lsa->crypto_session;
-	/* setup session action type */
-	} else {
-		ss->security.ses = lsa->sec_session;
-		ss->security.ctx = lsa->security_ctx;
-		ss->security.ol_flags = lsa->ol_flags;
-	}
 
 	if (ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
 		ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
@@ -1138,6 +1135,7 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 {
 	int rc;
 	struct rte_ipsec_sa_prm prm;
+	struct rte_ipsec_session *ips;
 	struct rte_ipv4_hdr v4  = {
 		.version_ihl = IPVERSION << 4 |
 			sizeof(v4) / RTE_IPV4_IHL_MULTIPLIER,
@@ -1162,7 +1160,10 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
+	/* init processing session */
+	ips = ipsec_get_session(lsa);
+	rc = fill_ipsec_session(ips, sa);
+
 	return rc;
 }
 
@@ -1386,6 +1387,7 @@ sa_check_offloads(uint16_t port_id, uint64_t *rx_offloads,
 {
 	struct ipsec_sa *rule;
 	uint32_t idx_sa;
+	enum rte_security_session_action_type rule_type;
 
 	*rx_offloads = 0;
 	*tx_offloads = 0;
@@ -1393,8 +1395,9 @@ sa_check_offloads(uint16_t port_id, uint64_t *rx_offloads,
 	/* Check for inbound rules that use offloads and use this port */
 	for (idx_sa = 0; idx_sa < nb_sa_in; idx_sa++) {
 		rule = &sa_in[idx_sa];
-		if ((rule->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
-				rule->type ==
+		rule_type = ipsec_get_action_type(rule);
+		if ((rule_type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+				rule_type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
 				&& rule->portid == port_id)
 			*rx_offloads |= DEV_RX_OFFLOAD_SECURITY;
@@ -1403,8 +1406,9 @@ sa_check_offloads(uint16_t port_id, uint64_t *rx_offloads,
 	/* Check for outbound rules that use offloads and use this port */
 	for (idx_sa = 0; idx_sa < nb_sa_out; idx_sa++) {
 		rule = &sa_out[idx_sa];
-		if ((rule->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
-				rule->type ==
+		rule_type = ipsec_get_action_type(rule);
+		if ((rule_type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+				rule_type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
 				&& rule->portid == port_id)
 			*tx_offloads |= DEV_TX_OFFLOAD_SECURITY;
-- 
2.17.1

[dpdk-dev] [PATCH v4 2/4] examples/ipsec-secgw: add fallback session feature

[Marcin Smoczynski]

Inline processing is limited to a specified subset of traffic. It is
often unable to handle more complicated situations, such as fragmented
traffic. When using inline processing such traffic is dropped.

Introduce fallback session for inline processing allowing processing
packets that normally would be dropped. A fallback session is
configured by adding 'fallback' keyword with 'lookaside-none' or
'lookaside-protocol' parameter to an SA configuration.

Using IPsec anti-replay window or ESN feature with fallback session is
not yet supported when primary session is of type
'inline-protocol-offload' or fallback session is 'lookaside-protocol'
because SA sequence number is not synchronized between software and
hardware sessions. Fallback sessions are also limited to ingress IPsec
traffic.

Fallback session feature is not available in the legacy mode.

Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
---
 doc/guides/sample_app_ug/ipsec_secgw.rst |  24 ++++-
 examples/ipsec-secgw/esp.c               |   4 +-
 examples/ipsec-secgw/ipsec-secgw.c       |  16 ++--
 examples/ipsec-secgw/ipsec.c             |  10 +--
 examples/ipsec-secgw/ipsec.h             |  40 +++++++--
 examples/ipsec-secgw/ipsec_process.c     |  85 ++++++++++++------
 examples/ipsec-secgw/sa.c                | 110 +++++++++++++++++++----
 7 files changed, 225 insertions(+), 64 deletions(-)

diff --git a/doc/guides/sample_app_ug/ipsec_secgw.rst b/doc/guides/sample_app_ug/ipsec_secgw.rst
index ad2d79e75..21b4b4418 100644
--- a/doc/guides/sample_app_ug/ipsec_secgw.rst
+++ b/doc/guides/sample_app_ug/ipsec_secgw.rst
@@ -401,7 +401,7 @@ The SA rule syntax is shown as follows:
 .. code-block:: console
 
     sa <dir> <spi> <cipher_algo> <cipher_key> <auth_algo> <auth_key>
-    <mode> <src_ip> <dst_ip> <action_type> <port_id>
+    <mode> <src_ip> <dst_ip> <action_type> <port_id> <fallback>
 
 where each options means:
 
@@ -573,6 +573,28 @@ where each options means:
 
    * *port_id X* X is a valid device number in decimal
 
+ ``<fallback>``
+
+ * Action type for ingress IPsec packets that inline processor failed to
+   process. IPsec anti-replay window and ESN is supported with fallback
+   processing session only when primary session is *lookaside-crypto-offload*
+   and fallback session is *lookaside-none*.
+
+   If used in conjunction with IPsec window, its width needs be increased
+   due to different processing times of inline and lookaside modes which
+   results in packet reordering.
+
+ * Optional: Yes.
+
+ * Available options:
+
+   * *lookaside-none*: use automatically chosen cryptodev to process packets
+   * *lookaside-protocol*: lookaside protocol hardware offload
+
+ * Syntax:
+
+   * *fallback lookaside-none*
+   * *fallback lookaside-protocol*
 
 Example SA rules:
 
diff --git a/examples/ipsec-secgw/esp.c b/examples/ipsec-secgw/esp.c
index c1b49da1e..bfa7ff721 100644
--- a/examples/ipsec-secgw/esp.c
+++ b/examples/ipsec-secgw/esp.c
@@ -155,7 +155,7 @@ esp_inbound_post(struct rte_mbuf *m, struct ipsec_sa *sa,
 	RTE_ASSERT(sa != NULL);
 	RTE_ASSERT(cop != NULL);
 
-	ips = ipsec_get_session(sa);
+	ips = ipsec_get_primary_session(sa);
 
 	if ((ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
 			(ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
@@ -234,7 +234,7 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 	RTE_ASSERT(m != NULL);
 	RTE_ASSERT(sa != NULL);
 
-	ips = ipsec_get_session(sa);
+	ips = ipsec_get_primary_session(sa);
 	ip_hdr_len = 0;
 
 	ip4 = rte_pktmbuf_mtod(m, struct ip *);
diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index 0d1fd6af6..641ed3767 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -189,6 +189,7 @@ static uint32_t mtu_size = RTE_ETHER_MTU;
 
 /* application wide librte_ipsec/SA parameters */
 struct app_sa_prm app_sa_prm = {.enable = 0};
+static const char *cfgfile;
 
 struct lcore_rx_queue {
 	uint16_t port_id;
@@ -1462,12 +1463,7 @@ parse_args(int32_t argc, char **argv)
 				print_usage(prgname);
 				return -1;
 			}
-			if (parse_cfg_file(optarg) < 0) {
-				printf("parsing file \"%s\" failed\n",
-					optarg);
-				print_usage(prgname);
-				return -1;
-			}
+			cfgfile = optarg;
 			f_present = 1;
 			break;
 		case 'j':
@@ -2398,6 +2394,14 @@ main(int32_t argc, char **argv)
 	if (ret < 0)
 		rte_exit(EXIT_FAILURE, "Invalid parameters\n");
 
+	/* parse configuration file */
+	if (parse_cfg_file(cfgfile) < 0) {
+		printf("parsing file \"%s\" failed\n",
+			optarg);
+		print_usage(argv[0]);
+		return -1;
+	}
+
 	if ((unprotected_port_mask & enabled_port_mask) !=
 			unprotected_port_mask)
 		rte_exit(EXIT_FAILURE, "Invalid unprotected portmask 0x%x\n",
diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index 8c60bd84b..8b0441028 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -404,7 +404,7 @@ enqueue_cop(struct cdev_qp *cqp, struct rte_crypto_op *cop)
 
 static inline void
 ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
-		struct rte_mbuf *pkts[], struct ipsec_sa *sas[],
+		struct rte_mbuf *pkts[], void *sas[],
 		uint16_t nb_pkts)
 {
 	int32_t ret = 0, i;
@@ -423,9 +423,9 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 		rte_prefetch0(pkts[i]);
 
 		priv = get_priv(pkts[i]);
-		sa = sas[i];
+		sa = ipsec_mask_saptr(sas[i]);
 		priv->sa = sa;
-		ips = ipsec_get_session(sa);
+		ips = ipsec_get_primary_session(sa);
 
 		switch (ips->type) {
 		case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL:
@@ -586,7 +586,7 @@ uint16_t
 ipsec_inbound(struct ipsec_ctx *ctx, struct rte_mbuf *pkts[],
 		uint16_t nb_pkts, uint16_t len)
 {
-	struct ipsec_sa *sas[nb_pkts];
+	void *sas[nb_pkts];
 
 	inbound_sa_lookup(ctx->sa_ctx, pkts, sas, nb_pkts);
 
@@ -606,7 +606,7 @@ uint16_t
 ipsec_outbound(struct ipsec_ctx *ctx, struct rte_mbuf *pkts[],
 		uint32_t sa_idx[], uint16_t nb_pkts, uint16_t len)
 {
-	struct ipsec_sa *sas[nb_pkts];
+	void *sas[nb_pkts];
 
 	outbound_sa_lookup(ctx->sa_ctx, sa_idx, sas, nb_pkts);
 
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index a4ad81b0e..8e075216c 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -81,12 +81,32 @@ struct app_sa_prm {
 
 extern struct app_sa_prm app_sa_prm;
 
+enum {
+	IPSEC_SESSION_PRIMARY = 0,
+	IPSEC_SESSION_FALLBACK = 1,
+	IPSEC_SESSION_MAX
+};
+
+#define IPSEC_SA_OFFLOAD_FALLBACK_FLAG (1)
+
+static inline struct ipsec_sa *
+ipsec_mask_saptr(void *ptr)
+{
+	uintptr_t i = (uintptr_t)ptr;
+	static const uintptr_t mask = IPSEC_SA_OFFLOAD_FALLBACK_FLAG;
+
+	i &= ~mask;
+
+	return (struct ipsec_sa *)i;
+}
+
 struct ipsec_sa {
-	struct rte_ipsec_session ips; /* one session per sa for now */
+	struct rte_ipsec_session sessions[IPSEC_SESSION_MAX];
 	uint32_t spi;
 	uint32_t cdev_id_qp;
 	uint64_t seq;
 	uint32_t salt;
+	uint32_t fallback_sessions;
 	enum rte_crypto_cipher_algorithm cipher_algo;
 	enum rte_crypto_auth_algorithm auth_algo;
 	enum rte_crypto_aead_algorithm aead_algo;
@@ -210,7 +230,7 @@ struct cnt_blk {
 struct traffic_type {
 	const uint8_t *data[MAX_PKT_BURST * 2];
 	struct rte_mbuf *pkts[MAX_PKT_BURST * 2];
-	struct ipsec_sa *saptr[MAX_PKT_BURST * 2];
+	void *saptr[MAX_PKT_BURST * 2];
 	uint32_t res[MAX_PKT_BURST * 2];
 	uint32_t num;
 };
@@ -278,16 +298,22 @@ get_sym_cop(struct rte_crypto_op *cop)
 }
 
 static inline struct rte_ipsec_session *
-ipsec_get_session(struct ipsec_sa *sa)
+ipsec_get_primary_session(struct ipsec_sa *sa)
+{
+	return &sa->sessions[IPSEC_SESSION_PRIMARY];
+}
+
+static inline struct rte_ipsec_session *
+ipsec_get_fallback_session(struct ipsec_sa *sa)
 {
-	return &sa->ips;
+	return &sa->sessions[IPSEC_SESSION_FALLBACK];
 }
 
 static inline enum rte_security_session_action_type
 ipsec_get_action_type(struct ipsec_sa *sa)
 {
 	struct rte_ipsec_session *ips;
-	ips = ipsec_get_session(sa);
+	ips = ipsec_get_primary_session(sa);
 	return ips->type;
 }
 
@@ -296,11 +322,11 @@ inbound_sa_check(struct sa_ctx *sa_ctx, struct rte_mbuf *m, uint32_t sa_idx);
 
 void
 inbound_sa_lookup(struct sa_ctx *sa_ctx, struct rte_mbuf *pkts[],
-		struct ipsec_sa *sa[], uint16_t nb_pkts);
+		void *sa[], uint16_t nb_pkts);
 
 void
 outbound_sa_lookup(struct sa_ctx *sa_ctx, uint32_t sa_idx[],
-		struct ipsec_sa *sa[], uint16_t nb_pkts);
+		void *sa[], uint16_t nb_pkts);
 
 void
 sp4_init(struct socket_ctx *ctx, int32_t socket_id);
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index 239d81ef6..2eb5c8b34 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -18,7 +18,6 @@
 	(((t) & RTE_IPSEC_SATP_IPV_MASK) == RTE_IPSEC_SATP_IPV4)) || \
 	((t) & RTE_IPSEC_SATP_MODE_MASK) == RTE_IPSEC_SATP_MODE_TUNLV4)
 
-
 /* helper routine to free bulk of packets */
 static inline void
 free_pkts(struct rte_mbuf *mb[], uint32_t n)
@@ -118,7 +117,7 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
  * group input packets byt the SA they belong to.
  */
 static uint32_t
-sa_group(struct ipsec_sa *sa_ptr[], struct rte_mbuf *pkts[],
+sa_group(void *sa_ptr[], struct rte_mbuf *pkts[],
 	struct rte_ipsec_group grp[], uint32_t num)
 {
 	uint32_t i, n, spi;
@@ -185,6 +184,37 @@ copy_to_trf(struct ipsec_traffic *trf, uint64_t satp, struct rte_mbuf *mb[],
 	out->num += num;
 }
 
+static uint32_t
+ipsec_prepare_crypto_group(struct ipsec_ctx *ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips, struct rte_mbuf **m,
+		unsigned int cnt)
+{
+	struct cdev_qp *cqp;
+	struct rte_crypto_op *cop[cnt];
+	uint32_t j, k;
+	struct ipsec_mbuf_metadata *priv;
+
+	cqp = &ctx->tbl[sa->cdev_id_qp];
+
+	/* for that app each mbuf has it's own crypto op */
+	for (j = 0; j != cnt; j++) {
+		priv = get_priv(m[j]);
+		cop[j] = &priv->cop;
+		/*
+		 * this is just to satisfy inbound_sa_check()
+		 * should be removed in future.
+		 */
+		priv->sa = sa;
+	}
+
+	/* prepare and enqueue crypto ops */
+	k = rte_ipsec_pkt_crypto_prepare(ips, m, cop, cnt);
+	if (k != 0)
+		enqueue_cop_bulk(cqp, cop, k);
+
+	return k;
+}
+
 /*
  * Process ipsec packets.
  * If packet belong to SA that is subject of inline-crypto,
@@ -201,18 +231,15 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 	struct ipsec_mbuf_metadata *priv;
 	struct rte_ipsec_group *pg;
 	struct rte_ipsec_session *ips;
-	struct cdev_qp *cqp;
-	struct rte_crypto_op *cop[RTE_DIM(trf->ipsec.pkts)];
 	struct rte_ipsec_group grp[RTE_DIM(trf->ipsec.pkts)];
 
 	n = sa_group(trf->ipsec.saptr, trf->ipsec.pkts, grp, trf->ipsec.num);
 
 	for (i = 0; i != n; i++) {
-
 		pg = grp + i;
-		sa = pg->id.ptr;
+		sa = ipsec_mask_saptr(pg->id.ptr);
 
-		ips = ipsec_get_session(sa);
+		ips = ipsec_get_primary_session(sa);
 
 		/* no valid HW session for that SA, try to create one */
 		if (sa == NULL || (ips->crypto.ses == NULL &&
@@ -224,6 +251,7 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 				ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
 
+			/* get SA type */
 			satp = rte_ipsec_sa_type(ips->sa);
 
 			/*
@@ -236,30 +264,33 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 				priv->sa = sa;
 			}
 
-			k = rte_ipsec_pkt_process(ips, pg->m, pg->cnt);
-			copy_to_trf(trf, satp, pg->m, k);
+			/* fallback to cryptodev with RX packets which inline
+			 * processor was unable to process
+			 */
+			if (pg->id.val & IPSEC_SA_OFFLOAD_FALLBACK_FLAG) {
+				/* offload packets to cryptodev */
+				struct rte_ipsec_session *fallback;
+
+				fallback = ipsec_get_fallback_session(sa);
+				if (fallback->crypto.ses == NULL &&
+					fill_ipsec_session(fallback, ctx, sa)
+					!= 0)
+					k = 0;
+				else
+					k = ipsec_prepare_crypto_group(ctx, sa,
+						fallback, pg->m, pg->cnt);
+			} else {
+				/* finish processing of packets successfully
+				 * decrypted by an inline processor
+				 */
+				k = rte_ipsec_pkt_process(ips, pg->m, pg->cnt);
+				copy_to_trf(trf, satp, pg->m, k);
 
+			}
 		/* enqueue packets to crypto dev */
 		} else {
-
-			cqp = &ctx->tbl[sa->cdev_id_qp];
-
-			/* for that app each mbuf has it's own crypto op */
-			for (j = 0; j != pg->cnt; j++) {
-				priv = get_priv(pg->m[j]);
-				cop[j] = &priv->cop;
-				/*
-				 * this is just to satisfy inbound_sa_check()
-				 * should be removed in future.
-				 */
-				priv->sa = sa;
-			}
-
-			/* prepare and enqueue crypto ops */
-			k = rte_ipsec_pkt_crypto_prepare(ips, pg->m, cop,
+			k = ipsec_prepare_crypto_group(ctx, sa, ips, pg->m,
 				pg->cnt);
-			if (k != 0)
-				enqueue_cop_bulk(cqp, cop, k);
 		}
 
 		/* drop packets that cannot be enqueued/processed */
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index a3d8e4545..700d52490 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -31,6 +31,8 @@
 
 #define IP6_FULL_MASK (sizeof(((struct ip_addr *)NULL)->ip.ip6.ip6) * CHAR_BIT)
 
+#define MBUF_NO_SEC_OFFLOAD(m) ((m->ol_flags & PKT_RX_SEC_OFFLOAD) == 0)
+
 struct supported_cipher_algo {
 	const char *keyword;
 	enum rte_crypto_cipher_algorithm algo;
@@ -235,6 +237,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	uint32_t mode_p = 0;
 	uint32_t type_p = 0;
 	uint32_t portid_p = 0;
+	uint32_t fallback_p = 0;
 
 	if (strcmp(tokens[0], "in") == 0) {
 		ri = &nb_sa_in;
@@ -245,6 +248,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 			return;
 
 		rule = &sa_in[*ri];
+		rule->direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS;
 	} else {
 		ri = &nb_sa_out;
 
@@ -254,6 +258,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 			return;
 
 		rule = &sa_out[*ri];
+		rule->direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS;
 	}
 
 	/* spi number */
@@ -263,7 +268,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	if (atoi(tokens[1]) == INVALID_SPI)
 		return;
 	rule->spi = atoi(tokens[1]);
-	ips = ipsec_get_session(rule);
+	ips = ipsec_get_primary_session(rule);
 
 	for (ti = 2; ti < n_tokens; ti++) {
 		if (strcmp(tokens[ti], "mode") == 0) {
@@ -596,6 +601,43 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 			continue;
 		}
 
+		if (strcmp(tokens[ti], "fallback") == 0) {
+			struct rte_ipsec_session *fb;
+
+			APP_CHECK(app_sa_prm.enable, status, "Fallback session "
+				"not allowed for legacy mode.");
+			if (status->status < 0)
+				return;
+			APP_CHECK(rule->direction ==
+				RTE_SECURITY_IPSEC_SA_DIR_INGRESS, status,
+				"Fallback session not allowed for egress "
+				"rule");
+			if (status->status < 0)
+				return;
+			APP_CHECK_PRESENCE(fallback_p, tokens[ti], status);
+			if (status->status < 0)
+				return;
+			INCREMENT_TOKEN_INDEX(ti, n_tokens, status);
+			if (status->status < 0)
+				return;
+			fb = ipsec_get_fallback_session(rule);
+			if (strcmp(tokens[ti], "lookaside-none") == 0) {
+				fb->type = RTE_SECURITY_ACTION_TYPE_NONE;
+			} else if (strcmp(tokens[ti], "lookaside-protocol")
+				== 0) {
+				fb->type =
+				RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL;
+			} else {
+				APP_CHECK(0, status, "unrecognized fallback "
+					"type %s.", tokens[ti]);
+				return;
+			}
+
+			rule->fallback_sessions = 1;
+			fallback_p = 1;
+			continue;
+		}
+
 		/* unrecognizeable input */
 		APP_CHECK(0, status, "unrecognized input \"%s\"",
 			tokens[ti]);
@@ -643,6 +685,7 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 	uint32_t i;
 	uint8_t a, b, c, d;
 	const struct rte_ipsec_session *ips;
+	const struct rte_ipsec_session *fallback_ips;
 
 	printf("\tspi_%s(%3u):", inbound?"in":"out", sa->spi);
 
@@ -699,7 +742,7 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 		break;
 	}
 
-	ips = &sa->ips;
+	ips = &sa->sessions[IPSEC_SESSION_PRIMARY];
 	printf(" type:");
 	switch (ips->type) {
 	case RTE_SECURITY_ACTION_TYPE_NONE:
@@ -715,6 +758,22 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 		printf("lookaside-protocol-offload ");
 		break;
 	}
+
+	fallback_ips = &sa->sessions[IPSEC_SESSION_FALLBACK];
+	if (fallback_ips != NULL && sa->fallback_sessions > 0) {
+		printf("inline fallback:");
+		switch (fallback_ips->type) {
+		case RTE_SECURITY_ACTION_TYPE_NONE:
+			printf("lookaside-none ");
+			break;
+		case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL:
+			printf("lookaside-protocol ");
+			break;
+		default:
+			printf("invalid ");
+			break;
+		}
+	}
 	printf("\n");
 }
 
@@ -896,7 +955,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 		}
 		*sa = entries[i];
 		sa->seq = 0;
-		ips = ipsec_get_session(sa);
+		ips = ipsec_get_primary_session(sa);
 
 		if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
 			ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
@@ -904,9 +963,6 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 				return -EINVAL;
 		}
 
-		sa->direction = (inbound == 1) ?
-				RTE_SECURITY_IPSEC_SA_DIR_INGRESS :
-				RTE_SECURITY_IPSEC_SA_DIR_EGRESS;
 
 		switch (WITHOUT_TRANSPORT_VERSION(sa->flags)) {
 		case IP4_TUNNEL:
@@ -946,7 +1002,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			sa->xforms = &sa_ctx->xf[idx].a;
 
-			ips = ipsec_get_session(sa);
+			ips = ipsec_get_primary_session(sa);
 			if (ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
 				ips->type ==
@@ -1160,9 +1216,15 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	/* init processing session */
-	ips = ipsec_get_session(lsa);
+	/* init primary processing session */
+	ips = ipsec_get_primary_session(lsa);
 	rc = fill_ipsec_session(ips, sa);
+	if (rc != 0)
+		return rc;
+
+	/* init inline fallback processing session */
+	if (lsa->fallback_sessions == 1)
+		rc = fill_ipsec_session(ipsec_get_fallback_session(lsa), sa);
 
 	return rc;
 }
@@ -1318,13 +1380,14 @@ inbound_sa_check(struct sa_ctx *sa_ctx, struct rte_mbuf *m, uint32_t sa_idx)
 
 static inline void
 single_inbound_lookup(struct ipsec_sa *sadb, struct rte_mbuf *pkt,
-		struct ipsec_sa **sa_ret)
+		void **sa_ret)
 {
 	struct rte_esp_hdr *esp;
 	struct ip *ip;
 	uint32_t *src4_addr;
 	uint8_t *src6_addr;
 	struct ipsec_sa *sa;
+	void *result_sa;
 
 	*sa_ret = NULL;
 
@@ -1334,33 +1397,48 @@ single_inbound_lookup(struct ipsec_sa *sadb, struct rte_mbuf *pkt,
 	if (esp->spi == INVALID_SPI)
 		return;
 
-	sa = &sadb[SPI2IDX(rte_be_to_cpu_32(esp->spi))];
+	result_sa = sa = &sadb[SPI2IDX(rte_be_to_cpu_32(esp->spi))];
 	if (rte_be_to_cpu_32(esp->spi) != sa->spi)
 		return;
 
+	/*
+	 * Mark need for inline offload fallback on the LSB of SA pointer.
+	 * Thanks to packet grouping mechanism which ipsec_process is using
+	 * packets marked for fallback processing will form separate group.
+	 *
+	 * Because it is not safe to use SA pointer it is casted to generic
+	 * pointer to prevent from unintentional use. Use ipsec_mask_saptr
+	 * to get valid struct pointer.
+	 */
+	if (MBUF_NO_SEC_OFFLOAD(pkt) && sa->fallback_sessions > 0) {
+		uintptr_t intsa = (uintptr_t)sa;
+		intsa |= IPSEC_SA_OFFLOAD_FALLBACK_FLAG;
+		result_sa = (void *)intsa;
+	}
+
 	switch (WITHOUT_TRANSPORT_VERSION(sa->flags)) {
 	case IP4_TUNNEL:
 		src4_addr = RTE_PTR_ADD(ip, offsetof(struct ip, ip_src));
 		if ((ip->ip_v == IPVERSION) &&
 				(sa->src.ip.ip4 == *src4_addr) &&
 				(sa->dst.ip.ip4 == *(src4_addr + 1)))
-			*sa_ret = sa;
+			*sa_ret = result_sa;
 		break;
 	case IP6_TUNNEL:
 		src6_addr = RTE_PTR_ADD(ip, offsetof(struct ip6_hdr, ip6_src));
 		if ((ip->ip_v == IP6_VERSION) &&
 				!memcmp(&sa->src.ip.ip6.ip6, src6_addr, 16) &&
 				!memcmp(&sa->dst.ip.ip6.ip6, src6_addr + 16, 16))
-			*sa_ret = sa;
+			*sa_ret = result_sa;
 		break;
 	case TRANSPORT:
-		*sa_ret = sa;
+		*sa_ret = result_sa;
 	}
 }
 
 void
 inbound_sa_lookup(struct sa_ctx *sa_ctx, struct rte_mbuf *pkts[],
-		struct ipsec_sa *sa[], uint16_t nb_pkts)
+		void *sa[], uint16_t nb_pkts)
 {
 	uint32_t i;
 
@@ -1370,7 +1448,7 @@ inbound_sa_lookup(struct sa_ctx *sa_ctx, struct rte_mbuf *pkts[],
 
 void
 outbound_sa_lookup(struct sa_ctx *sa_ctx, uint32_t sa_idx[],
-		struct ipsec_sa *sa[], uint16_t nb_pkts)
+		void *sa[], uint16_t nb_pkts)
 {
 	uint32_t i;
 
-- 
2.17.1

[dpdk-dev] [PATCH v4 3/4] examples/ipsec-secgw: add frag TTL cmdline option

[Marcin Smoczynski]

Due to fragment loss on highly saturated links and long fragment
lifetime, ipsec-secgw application quickly runs out of free reassembly
buckets. As a result new fragments are being dropped.

Introduce --frag-ttl option which allow user to lower default fragment
lifitime which solves problem of saturated reassembly buckets with high
bandwidth fragmented traffic.

Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
---
 doc/guides/sample_app_ug/ipsec_secgw.rst |  7 +++++
 examples/ipsec-secgw/ipsec-secgw.c       | 40 ++++++++++++++++++------
 2 files changed, 37 insertions(+), 10 deletions(-)

diff --git a/doc/guides/sample_app_ug/ipsec_secgw.rst b/doc/guides/sample_app_ug/ipsec_secgw.rst
index 21b4b4418..55b65d7bc 100644
--- a/doc/guides/sample_app_ug/ipsec_secgw.rst
+++ b/doc/guides/sample_app_ug/ipsec_secgw.rst
@@ -154,6 +154,13 @@ Where:
     Incoming packets with length bigger then MTU will be discarded.
     Default value: 1500.
 
+*   ``--frag-ttl FRAG_TTL_NS``: fragment lifetime (in nanoseconds).
+    If packet is not reassembled within this time, received fragments
+    will be discarded. Fragment lifetime should be decreased when
+    there is a high fragmented traffic loss in high bandwidth networks.
+    Should be lower for for low number of reassembly buckets.
+    Valid values: from 1 ns to 10 s. Default value: 10000000 (10 s).
+
 *   ``--reassemble NUM``: max number of entries in reassemble fragment table.
     Zero value disables reassembly functionality.
     Default value: 0.
diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index 641ed3767..7b246cc60 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -112,7 +112,7 @@ static uint16_t nb_txd = IPSEC_SECGW_TX_DESC_DEFAULT;
 		0, 0)
 
 #define	FRAG_TBL_BUCKET_ENTRIES	4
-#define	FRAG_TTL_MS		(10 * MS_PER_S)
+#define	MAX_FRAG_TTL_NS		(10LL * NS_PER_S)
 
 #define MTU_TO_FRAMELEN(x)	((x) + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN)
 
@@ -135,6 +135,7 @@ struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS] = {
 #define CMD_LINE_OPT_TX_OFFLOAD		"txoffload"
 #define CMD_LINE_OPT_REASSEMBLE		"reassemble"
 #define CMD_LINE_OPT_MTU		"mtu"
+#define CMD_LINE_OPT_FRAG_TTL		"frag-ttl"
 
 enum {
 	/* long options mapped to a short option */
@@ -150,6 +151,7 @@ enum {
 	CMD_LINE_OPT_TX_OFFLOAD_NUM,
 	CMD_LINE_OPT_REASSEMBLE_NUM,
 	CMD_LINE_OPT_MTU_NUM,
+	CMD_LINE_OPT_FRAG_TTL_NUM,
 };
 
 static const struct option lgopts[] = {
@@ -160,6 +162,7 @@ static const struct option lgopts[] = {
 	{CMD_LINE_OPT_TX_OFFLOAD, 1, 0, CMD_LINE_OPT_TX_OFFLOAD_NUM},
 	{CMD_LINE_OPT_REASSEMBLE, 1, 0, CMD_LINE_OPT_REASSEMBLE_NUM},
 	{CMD_LINE_OPT_MTU, 1, 0, CMD_LINE_OPT_MTU_NUM},
+	{CMD_LINE_OPT_FRAG_TTL, 1, 0, CMD_LINE_OPT_FRAG_TTL_NUM},
 	{NULL, 0, 0, 0}
 };
 
@@ -186,6 +189,7 @@ static uint64_t dev_tx_offload = UINT64_MAX;
 static uint32_t frag_tbl_sz;
 static uint32_t frame_buf_size = RTE_MBUF_DEFAULT_BUF_SIZE;
 static uint32_t mtu_size = RTE_ETHER_MTU;
+static uint64_t frag_ttl_ns = MAX_FRAG_TTL_NS;
 
 /* application wide librte_ipsec/SA parameters */
 struct app_sa_prm app_sa_prm = {.enable = 0};
@@ -1302,6 +1306,9 @@ print_usage(const char *prgname)
 		": MTU value on all ports (default value: 1500)\n"
 		"    outgoing packets with bigger size will be fragmented\n"
 		"    incoming packets with bigger size will be discarded\n"
+		"  --" CMD_LINE_OPT_FRAG_TTL " FRAG_TTL_NS"
+		": fragments lifetime in nanoseconds, default\n"
+		"    and maximum value is 10.000.000.000 ns (10 s)\n"
 		"\n",
 		prgname);
 }
@@ -1338,14 +1345,15 @@ parse_portmask(const char *portmask)
 	return pm;
 }
 
-static int32_t
+static int64_t
 parse_decimal(const char *str)
 {
 	char *end = NULL;
-	unsigned long num;
+	unsigned long long num;
 
-	num = strtoul(str, &end, 10);
-	if ((str[0] == '\0') || (end == NULL) || (*end != '\0'))
+	num = strtoull(str, &end, 10);
+	if ((str[0] == '\0') || (end == NULL) || (*end != '\0')
+		|| num > INT64_MAX || num < 0)
 		return -1;
 
 	return num;
@@ -1419,12 +1427,14 @@ print_app_sa_prm(const struct app_sa_prm *prm)
 	printf("replay window size: %u\n", prm->window_size);
 	printf("ESN: %s\n", (prm->enable_esn == 0) ? "disabled" : "enabled");
 	printf("SA flags: %#" PRIx64 "\n", prm->flags);
+	printf("Frag TTL: %lu ns\n", frag_ttl_ns);
 }
 
 static int32_t
 parse_args(int32_t argc, char **argv)
 {
-	int32_t opt, ret;
+	int opt;
+	int64_t ret;
 	char **argvopt;
 	int32_t option_index;
 	char *prgname = argv[0];
@@ -1503,7 +1513,7 @@ parse_args(int32_t argc, char **argv)
 			break;
 		case CMD_LINE_OPT_SINGLE_SA_NUM:
 			ret = parse_decimal(optarg);
-			if (ret == -1) {
+			if (ret == -1 || ret > UINT32_MAX) {
 				printf("Invalid argument[sa_idx]\n");
 				print_usage(prgname);
 				return -1;
@@ -1546,7 +1556,7 @@ parse_args(int32_t argc, char **argv)
 			break;
 		case CMD_LINE_OPT_REASSEMBLE_NUM:
 			ret = parse_decimal(optarg);
-			if (ret < 0) {
+			if (ret < 0 || ret > UINT32_MAX) {
 				printf("Invalid argument for \'%s\': %s\n",
 					CMD_LINE_OPT_REASSEMBLE, optarg);
 				print_usage(prgname);
@@ -1564,6 +1574,16 @@ parse_args(int32_t argc, char **argv)
 			}
 			mtu_size = ret;
 			break;
+		case CMD_LINE_OPT_FRAG_TTL_NUM:
+			ret = parse_decimal(optarg);
+			if (ret < 0 || ret > MAX_FRAG_TTL_NS) {
+				printf("Invalid argument for \'%s\': %s\n",
+					CMD_LINE_OPT_MTU, optarg);
+				print_usage(prgname);
+				return -1;
+			}
+			frag_ttl_ns = ret;
+			break;
 		default:
 			print_usage(prgname);
 			return -1;
@@ -2324,8 +2344,8 @@ reassemble_lcore_init(struct lcore_conf *lc, uint32_t cid)
 
 	/* create fragment table */
 	sid = rte_lcore_to_socket_id(cid);
-	frag_cycles = (rte_get_tsc_hz() + MS_PER_S - 1) /
-		MS_PER_S * FRAG_TTL_MS;
+	frag_cycles = (rte_get_tsc_hz() + NS_PER_S - 1) /
+		NS_PER_S * frag_ttl_ns;
 
 	lc->frag.tbl = rte_ip_frag_table_create(frag_tbl_sz,
 		FRAG_TBL_BUCKET_ENTRIES, frag_tbl_sz, frag_cycles, sid);
-- 
2.17.1

[dpdk-dev] [PATCH v4 4/4] examples/ipsec-secgw: add offload fallback tests

[Marcin Smoczynski]

Add tests for offload fallback feature; add inbound config modificator
SGW_CFG_XPRM_IN (offload fallback setting can be set only for inbound
SAs). Tests are using cryptodev for outbound SA.

To test fragmentation with QAT set:
MULTI_SEG_TEST="--reassemble=4096 --cryptodev_mask=0x5555"

Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
---
 examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh         | 4 ++--
 .../test/trs_aesgcm_inline_crypto_fallback_defs.sh          | 5 +++++
 examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh         | 6 ++++--
 .../test/tun_aesgcm_inline_crypto_fallback_defs.sh          | 5 +++++
 4 files changed, 16 insertions(+), 4 deletions(-)
 create mode 100644 examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
 create mode 100644 examples/ipsec-secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh

diff --git a/examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh b/examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh
index f6c5bf5a7..17f2f86d2 100644
--- a/examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh
+++ b/examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh
@@ -29,11 +29,11 @@ sp ipv6 out esp bypass pri 1 sport 0:65535 dport 0:65535
 #SA in rules
 sa in 7 aead_algo aes-128-gcm \
 aead_key de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef \
-mode transport ${SGW_CFG_XPRM}
+mode transport ${SGW_CFG_XPRM} ${SGW_CFG_XPRM_IN}
 
 sa in 9 aead_algo aes-128-gcm \
 aead_key de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef \
-mode transport ${SGW_CFG_XPRM}
+mode transport ${SGW_CFG_XPRM} ${SGW_CFG_XPRM_IN}
 
 #SA out rules
 sa out 7 aead_algo aes-128-gcm \
diff --git a/examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh b/examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
new file mode 100644
index 000000000..875a7457d
--- /dev/null
+++ b/examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
@@ -0,0 +1,5 @@
+#! /bin/bash
+
+. ${DIR}/trs_aesgcm_defs.sh
+
+SGW_CFG_XPRM_IN='port_id 0 type inline-crypto-offload fallback lookaside-none'
diff --git a/examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh b/examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh
index 278377967..7490baded 100644
--- a/examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh
+++ b/examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh
@@ -29,11 +29,13 @@ sp ipv6 out esp bypass pri 1 sport 0:65535 dport 0:65535
 #SA in rules
 sa in 7 aead_algo aes-128-gcm \
 aead_key de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef \
-mode ipv4-tunnel src ${REMOTE_IPV4} dst ${LOCAL_IPV4} ${SGW_CFG_XPRM}
+mode ipv4-tunnel src ${REMOTE_IPV4} dst ${LOCAL_IPV4} ${SGW_CFG_XPRM} \
+${SGW_CFG_XPRM_IN}
 
 sa in 9 aead_algo aes-128-gcm \
 aead_key de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef \
-mode ipv6-tunnel src ${REMOTE_IPV6} dst ${LOCAL_IPV6} ${SGW_CFG_XPRM}
+mode ipv6-tunnel src ${REMOTE_IPV6} dst ${LOCAL_IPV6} ${SGW_CFG_XPRM} \
+${SGW_CFG_XPRM_IN}
 
 #SA out rules
 sa out 7 aead_algo aes-128-gcm \
diff --git a/examples/ipsec-secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh b/examples/ipsec-secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh
new file mode 100644
index 000000000..696848432
--- /dev/null
+++ b/examples/ipsec-secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh
@@ -0,0 +1,5 @@
+#! /bin/bash
+
+. ${DIR}/tun_aesgcm_defs.sh
+
+SGW_CFG_XPRM_IN='port_id 0 type inline-crypto-offload fallback lookaside-none'
-- 
2.17.1

[dpdk-dev] [PATCH v5 0/4] add fallback session

[Marcin Smoczynski]

Add fallback session feature allowing to process packets that inline
processor is unable to handle (e.g. fragmented traffic). Processing
takes place in a secondary session defined for SA in a configuration
file.

This feature is limited to ingress IPsec traffic only. IPsec
anti-replay window and ESN are supported in conjunction with fallback
session when following conditions are met:
 * primary session is 'inline-crypto-offload,
 * fallback sessions is 'lookaside-none'.
Due to different processing times of inline and lookaside modes,
fallback session introduces some packet reordering, therefore when
using with IPsec window, its value should be increased.

v4 to v5 changes:
 - fix build errors related to frag TTL command line option and
   parse_decimal method

v3 to v4 changes:
 - add info about packet reordering to the documentation regarding
   fallback session
 - add patch with --frag-ttl command line option which allows to change
   fragment lifetime

v2 to v3 changes:
 - doc and commit log update - explicitly state feature limitations

v1 to v2 changes:
 - disable fallback offload for outbound SAs
 - add test scripts

Marcin Smoczynski (4):
  examples/ipsec-secgw: ipsec_sa structure cleanup
  examples/ipsec-secgw: add fallback session feature
  examples/ipsec-secgw: add frag TTL cmdline option
  examples/ipsec-secgw: add offload fallback tests

 doc/guides/sample_app_ug/ipsec_secgw.rst      |  31 +++-
 examples/ipsec-secgw/esp.c                    |  35 ++--
 examples/ipsec-secgw/ipsec-secgw.c            |  56 ++++--
 examples/ipsec-secgw/ipsec.c                  |  99 ++++++-----
 examples/ipsec-secgw/ipsec.h                  |  61 +++++--
 examples/ipsec-secgw/ipsec_process.c          | 113 +++++++-----
 examples/ipsec-secgw/sa.c                     | 164 +++++++++++++-----
 .../test/trs_aesgcm_common_defs.sh            |   4 +-
 .../trs_aesgcm_inline_crypto_fallback_defs.sh |   5 +
 .../test/tun_aesgcm_common_defs.sh            |   6 +-
 .../tun_aesgcm_inline_crypto_fallback_defs.sh |   5 +
 11 files changed, 402 insertions(+), 177 deletions(-)
 create mode 100644 examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
 create mode 100644 examples/ipsec-secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh

-- 
2.17.1

[dpdk-dev] [PATCH v5 1/4] examples/ipsec-secgw: ipsec_sa structure cleanup

[Marcin Smoczynski]

Cleanup ipsec_sa structure by removing every field that is already in
the rte_ipsec_session structure:
 * cryptodev/security session union
 * action type
 * offload flags
 * security context
References to abovementioned fields are changed to direct references
to matching fields of rte_ipsec_session structure.

Such refactoring is needed to introduce many sessions per SA feature,
e.g. fallback session for inline offload processing.

Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
---
 examples/ipsec-secgw/esp.c           | 35 +++++++----
 examples/ipsec-secgw/ipsec.c         | 91 +++++++++++++++-------------
 examples/ipsec-secgw/ipsec.h         | 27 ++++++---
 examples/ipsec-secgw/ipsec_process.c | 30 ++++-----
 examples/ipsec-secgw/sa.c            | 66 ++++++++++----------
 5 files changed, 137 insertions(+), 112 deletions(-)

diff --git a/examples/ipsec-secgw/esp.c b/examples/ipsec-secgw/esp.c
index d6d7b1256..c1b49da1e 100644
--- a/examples/ipsec-secgw/esp.c
+++ b/examples/ipsec-secgw/esp.c
@@ -30,7 +30,8 @@ esp_inbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 	int32_t payload_len, ip_hdr_len;
 
 	RTE_ASSERT(sa != NULL);
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
+	if (ipsec_get_action_type(sa) ==
+			RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
 		return 0;
 
 	RTE_ASSERT(m != NULL);
@@ -148,13 +149,16 @@ esp_inbound_post(struct rte_mbuf *m, struct ipsec_sa *sa,
 	uint8_t *nexthdr, *pad_len;
 	uint8_t *padding;
 	uint16_t i;
+	struct rte_ipsec_session *ips;
 
 	RTE_ASSERT(m != NULL);
 	RTE_ASSERT(sa != NULL);
 	RTE_ASSERT(cop != NULL);
 
-	if ((sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
-			(sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
+	ips = ipsec_get_session(sa);
+
+	if ((ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
+			(ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
 		if (m->ol_flags & PKT_RX_SEC_OFFLOAD) {
 			if (m->ol_flags & PKT_RX_SEC_OFFLOAD_FAILED)
 				cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
@@ -169,8 +173,8 @@ esp_inbound_post(struct rte_mbuf *m, struct ipsec_sa *sa,
 		return -1;
 	}
 
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO &&
-	    sa->ol_flags & RTE_SECURITY_RX_HW_TRAILER_OFFLOAD) {
+	if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO &&
+		ips->security.ol_flags & RTE_SECURITY_RX_HW_TRAILER_OFFLOAD) {
 		nexthdr = &m->inner_esp_next_proto;
 	} else {
 		nexthdr = rte_pktmbuf_mtod_offset(m, uint8_t*,
@@ -225,10 +229,12 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 	struct rte_crypto_sym_op *sym_cop;
 	int32_t i;
 	uint16_t pad_payload_len, pad_len, ip_hdr_len;
+	struct rte_ipsec_session *ips;
 
 	RTE_ASSERT(m != NULL);
 	RTE_ASSERT(sa != NULL);
 
+	ips = ipsec_get_session(sa);
 	ip_hdr_len = 0;
 
 	ip4 = rte_pktmbuf_mtod(m, struct ip *);
@@ -277,9 +283,10 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 	}
 
 	/* Add trailer padding if it is not constructed by HW */
-	if (sa->type != RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
-	    (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO &&
-	     !(sa->ol_flags & RTE_SECURITY_TX_HW_TRAILER_OFFLOAD))) {
+	if (ips->type != RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+		(ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO &&
+		 !(ips->security.ol_flags &
+			 RTE_SECURITY_TX_HW_TRAILER_OFFLOAD))) {
 		padding = (uint8_t *)rte_pktmbuf_append(m, pad_len +
 							sa->digest_len);
 		if (unlikely(padding == NULL)) {
@@ -344,8 +351,9 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 		}
 	}
 
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
-		if (sa->ol_flags & RTE_SECURITY_TX_HW_TRAILER_OFFLOAD) {
+	if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		if (ips->security.ol_flags &
+				RTE_SECURITY_TX_HW_TRAILER_OFFLOAD) {
 			/* Set the inner esp next protocol for HW trailer */
 			m->inner_esp_next_proto = nlp;
 			m->packet_type |= RTE_PTYPE_TUNNEL_ESP;
@@ -448,11 +456,14 @@ esp_outbound_post(struct rte_mbuf *m,
 		  struct ipsec_sa *sa,
 		  struct rte_crypto_op *cop)
 {
+	enum rte_security_session_action_type type;
 	RTE_ASSERT(m != NULL);
 	RTE_ASSERT(sa != NULL);
 
-	if ((sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
-			(sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
+	type = ipsec_get_action_type(sa);
+
+	if ((type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
+			(type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
 		m->ol_flags |= PKT_TX_SEC_OFFLOAD;
 	} else {
 		RTE_ASSERT(cop != NULL);
diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index dc85adfe5..8c60bd84b 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -40,7 +40,8 @@ set_ipsec_conf(struct ipsec_sa *sa, struct rte_security_ipsec_xform *ipsec)
 }
 
 int
-create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips)
 {
 	struct rte_cryptodev_info cdev_info;
 	unsigned long cdev_id_qp = 0;
@@ -71,9 +72,9 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			ipsec_ctx->tbl[cdev_id_qp].id,
 			ipsec_ctx->tbl[cdev_id_qp].qp);
 
-	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
+	if (ips->type != RTE_SECURITY_ACTION_TYPE_NONE) {
 		struct rte_security_session_conf sess_conf = {
-			.action_type = sa->type,
+			.action_type = ips->type,
 			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 			{.ipsec = {
 				.spi = sa->spi,
@@ -90,7 +91,7 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 
 		};
 
-		if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
+		if (ips->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
 			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
 							rte_cryptodev_get_sec_ctx(
 							ipsec_ctx->tbl[cdev_id_qp].id);
@@ -98,9 +99,9 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			/* Set IPsec parameters in conf */
 			set_ipsec_conf(sa, &(sess_conf.ipsec));
 
-			sa->sec_session = rte_security_session_create(ctx,
+			ips->security.ses = rte_security_session_create(ctx,
 					&sess_conf, ipsec_ctx->session_priv_pool);
-			if (sa->sec_session == NULL) {
+			if (ips->security.ses == NULL) {
 				RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 				return -1;
@@ -110,10 +111,10 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			return -1;
 		}
 	} else {
-		sa->crypto_session = rte_cryptodev_sym_session_create(
+		ips->crypto.ses = rte_cryptodev_sym_session_create(
 				ipsec_ctx->session_pool);
 		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
-				sa->crypto_session, sa->xforms,
+				ips->crypto.ses, sa->xforms,
 				ipsec_ctx->session_priv_pool);
 
 		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
@@ -126,12 +127,13 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 }
 
 int
-create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips)
 {
 	int32_t ret = 0;
 	struct rte_security_ctx *sec_ctx;
 	struct rte_security_session_conf sess_conf = {
-		.action_type = sa->type,
+		.action_type = ips->type,
 		.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 		{.ipsec = {
 			.spi = sa->spi,
@@ -151,7 +153,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on port %u\n",
 		sa->spi, sa->portid);
 
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+	if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
 		struct rte_flow_error err;
 		const struct rte_security_capability *sec_cap;
 		int ret = 0;
@@ -165,9 +167,9 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 			return -1;
 		}
 
-		sa->sec_session = rte_security_session_create(sec_ctx,
+		ips->security.ses = rte_security_session_create(sec_ctx,
 				&sess_conf, skt_ctx->session_pool);
-		if (sa->sec_session == NULL) {
+		if (ips->security.ses == NULL) {
 			RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 			return -1;
@@ -177,7 +179,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 
 		/* iterate until ESP tunnel*/
 		while (sec_cap->action != RTE_SECURITY_ACTION_TYPE_NONE) {
-			if (sec_cap->action == sa->type &&
+			if (sec_cap->action == ips->type &&
 			    sec_cap->protocol ==
 				RTE_SECURITY_PROTOCOL_IPSEC &&
 			    sec_cap->ipsec.mode ==
@@ -193,8 +195,8 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 			return -1;
 		}
 
-		sa->ol_flags = sec_cap->ol_flags;
-		sa->security_ctx = sec_ctx;
+		ips->security.ol_flags = sec_cap->ol_flags;
+		ips->security.ctx = sec_ctx;
 		sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
 
 		if (IS_IP6(sa->flags)) {
@@ -223,7 +225,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 		sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
 
 		sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
-		sa->action[0].conf = sa->sec_session;
+		sa->action[0].conf = ips->security.ses;
 
 		sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
 
@@ -282,8 +284,8 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 			if (ret)
 				goto flow_create_failure;
 		} else if (sa->attr.egress &&
-			   (sa->ol_flags &
-				    RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
+				(ips->security.ol_flags &
+					RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
 			sa->action[1].type =
 					RTE_FLOW_ACTION_TYPE_PASSTHRU;
 			sa->action[2].type =
@@ -299,7 +301,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 				err.message);
 			return -1;
 		}
-	} else if (sa->type ==	RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
+	} else if (ips->type ==	RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
 		const struct rte_security_capability *sec_cap;
 
 		sec_ctx = (struct rte_security_ctx *)
@@ -327,9 +329,9 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 
 		sess_conf.userdata = (void *) sa;
 
-		sa->sec_session = rte_security_session_create(sec_ctx,
+		ips->security.ses = rte_security_session_create(sec_ctx,
 					&sess_conf, skt_ctx->session_pool);
-		if (sa->sec_session == NULL) {
+		if (ips->security.ses == NULL) {
 			RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 			return -1;
@@ -345,7 +347,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 		/* iterate until ESP tunnel*/
 		while (sec_cap->action !=
 				RTE_SECURITY_ACTION_TYPE_NONE) {
-			if (sec_cap->action == sa->type &&
+			if (sec_cap->action == ips->type &&
 			    sec_cap->protocol ==
 				RTE_SECURITY_PROTOCOL_IPSEC &&
 			    sec_cap->ipsec.mode ==
@@ -361,8 +363,8 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 			return -1;
 		}
 
-		sa->ol_flags = sec_cap->ol_flags;
-		sa->security_ctx = sec_ctx;
+		ips->security.ol_flags = sec_cap->ol_flags;
+		ips->security.ctx = sec_ctx;
 	}
 	sa->cdev_id_qp = 0;
 
@@ -409,6 +411,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 	struct ipsec_mbuf_metadata *priv;
 	struct rte_crypto_sym_op *sym_cop;
 	struct ipsec_sa *sa;
+	struct rte_ipsec_session *ips;
 
 	for (i = 0; i < nb_pkts; i++) {
 		if (unlikely(sas[i] == NULL)) {
@@ -422,16 +425,17 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 		priv = get_priv(pkts[i]);
 		sa = sas[i];
 		priv->sa = sa;
+		ips = ipsec_get_session(sa);
 
-		switch (sa->type) {
+		switch (ips->type) {
 		case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL:
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
 
-			if ((unlikely(sa->sec_session == NULL)) &&
-				create_lookaside_session(ipsec_ctx, sa)) {
+			if ((unlikely(ips->security.ses == NULL)) &&
+				create_lookaside_session(ipsec_ctx, sa, ips)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -440,7 +444,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			sym_cop->m_src = pkts[i];
 
 			rte_security_attach_session(&priv->cop,
-					sa->sec_session);
+				ips->security.ses);
 			break;
 		case RTE_SECURITY_ACTION_TYPE_NONE:
 
@@ -449,14 +453,14 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			rte_prefetch0(&priv->sym_cop);
 
-			if ((unlikely(sa->crypto_session == NULL)) &&
-				create_lookaside_session(ipsec_ctx, sa)) {
+			if ((unlikely(ips->crypto.ses == NULL)) &&
+				create_lookaside_session(ipsec_ctx, sa, ips)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
 
 			rte_crypto_op_attach_sym_session(&priv->cop,
-					sa->crypto_session);
+					ips->crypto.ses);
 
 			ret = xform_func(pkts[i], sa, &priv->cop);
 			if (unlikely(ret)) {
@@ -465,21 +469,22 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 			break;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
-			RTE_ASSERT(sa->sec_session != NULL);
+			RTE_ASSERT(ips->security.ses != NULL);
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
-			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
+			if (ips->security.ol_flags &
+				RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
-						sa->security_ctx,
-						sa->sec_session, pkts[i], NULL);
+					ips->security.ctx, ips->security.ses,
+					pkts[i], NULL);
 			continue;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
-			RTE_ASSERT(sa->sec_session != NULL);
+			RTE_ASSERT(ips->security.ses != NULL);
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
 			rte_security_attach_session(&priv->cop,
-					sa->sec_session);
+					ips->security.ses);
 
 			ret = xform_func(pkts[i], sa, &priv->cop);
 			if (unlikely(ret)) {
@@ -488,10 +493,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
-			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
+			if (ips->security.ol_flags &
+				RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
-						sa->security_ctx,
-						sa->sec_session, pkts[i], NULL);
+					ips->security.ctx, ips->security.ses,
+					pkts[i], NULL);
 			continue;
 		}
 
@@ -560,7 +566,8 @@ ipsec_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			RTE_ASSERT(sa != NULL);
 
-			if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {
+			if (ipsec_get_action_type(sa) ==
+				RTE_SECURITY_ACTION_TYPE_NONE) {
 				ret = xform_func(pkt, sa, cops[j]);
 				if (unlikely(ret)) {
 					rte_pktmbuf_free(pkt);
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index 1efa6e488..a4ad81b0e 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -87,10 +87,6 @@ struct ipsec_sa {
 	uint32_t cdev_id_qp;
 	uint64_t seq;
 	uint32_t salt;
-	union {
-		struct rte_cryptodev_sym_session *crypto_session;
-		struct rte_security_session *sec_session;
-	};
 	enum rte_crypto_cipher_algorithm cipher_algo;
 	enum rte_crypto_auth_algorithm auth_algo;
 	enum rte_crypto_aead_algorithm aead_algo;
@@ -114,11 +110,8 @@ struct ipsec_sa {
 		struct rte_crypto_sym_xform *xforms;
 		struct rte_security_ipsec_xform *sec_xform;
 	};
-	enum rte_security_session_action_type type;
 	enum rte_security_ipsec_sa_direction direction;
 	uint16_t portid;
-	struct rte_security_ctx *security_ctx;
-	uint32_t ol_flags;
 
 #define MAX_RTE_FLOW_PATTERN (4)
 #define MAX_RTE_FLOW_ACTIONS (3)
@@ -284,6 +277,20 @@ get_sym_cop(struct rte_crypto_op *cop)
 	return (cop + 1);
 }
 
+static inline struct rte_ipsec_session *
+ipsec_get_session(struct ipsec_sa *sa)
+{
+	return &sa->ips;
+}
+
+static inline enum rte_security_session_action_type
+ipsec_get_action_type(struct ipsec_sa *sa)
+{
+	struct rte_ipsec_session *ips;
+	ips = ipsec_get_session(sa);
+	return ips->type;
+}
+
 int
 inbound_sa_check(struct sa_ctx *sa_ctx, struct rte_mbuf *m, uint32_t sa_idx);
 
@@ -338,9 +345,11 @@ void
 enqueue_cop_burst(struct cdev_qp *cqp);
 
 int
-create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips);
 
 int
-create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa);
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips);
 
 #endif /* __IPSEC_H__ */
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index 868f1a28d..239d81ef6 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -94,22 +94,16 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
 
 	/* setup crypto section */
 	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		if (sa->crypto_session == NULL) {
-			rc = create_lookaside_session(ctx, sa);
-			if (rc != 0)
-				return rc;
-		}
-		ss->crypto.ses = sa->crypto_session;
+		RTE_ASSERT(ss->crypto.ses == NULL);
+		rc = create_lookaside_session(ctx, sa, ss);
+		if (rc != 0)
+			return rc;
 	/* setup session action type */
-	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
-		if (sa->sec_session == NULL) {
-			rc = create_lookaside_session(ctx, sa);
-			if (rc != 0)
-				return rc;
-		}
-		ss->security.ses = sa->sec_session;
-		ss->security.ctx = sa->security_ctx;
-		ss->security.ol_flags = sa->ol_flags;
+	} else if (ss->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
+		RTE_ASSERT(ss->security.ses == NULL);
+		rc = create_lookaside_session(ctx, sa, ss);
+		if (rc != 0)
+			return rc;
 	} else
 		RTE_ASSERT(0);
 
@@ -218,7 +212,7 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 		pg = grp + i;
 		sa = pg->id.ptr;
 
-		ips = &sa->ips;
+		ips = ipsec_get_session(sa);
 
 		/* no valid HW session for that SA, try to create one */
 		if (sa == NULL || (ips->crypto.ses == NULL &&
@@ -226,8 +220,8 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 			k = 0;
 
 		/* process packets inline */
-		else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
-				sa->type ==
+		else if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+				ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
 
 			satp = rte_ipsec_sa_type(ips->sa);
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index c3cf3bd1f..a3d8e4545 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -224,6 +224,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	struct parse_status *status)
 {
 	struct ipsec_sa *rule = NULL;
+	struct rte_ipsec_session *ips;
 	uint32_t ti; /*token index*/
 	uint32_t *ri /*rule index*/;
 	uint32_t cipher_algo_p = 0;
@@ -262,6 +263,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	if (atoi(tokens[1]) == INVALID_SPI)
 		return;
 	rule->spi = atoi(tokens[1]);
+	ips = ipsec_get_session(rule);
 
 	for (ti = 2; ti < n_tokens; ti++) {
 		if (strcmp(tokens[ti], "mode") == 0) {
@@ -558,18 +560,18 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 				return;
 
 			if (strcmp(tokens[ti], "inline-crypto-offload") == 0)
-				rule->type =
+				ips->type =
 					RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO;
 			else if (strcmp(tokens[ti],
 					"inline-protocol-offload") == 0)
-				rule->type =
+				ips->type =
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL;
 			else if (strcmp(tokens[ti],
 					"lookaside-protocol-offload") == 0)
-				rule->type =
+				ips->type =
 				RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL;
 			else if (strcmp(tokens[ti], "no-offload") == 0)
-				rule->type = RTE_SECURITY_ACTION_TYPE_NONE;
+				ips->type = RTE_SECURITY_ACTION_TYPE_NONE;
 			else {
 				APP_CHECK(0, status, "Invalid input \"%s\"",
 						tokens[ti]);
@@ -624,11 +626,11 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	if (status->status < 0)
 		return;
 
-	if ((rule->type != RTE_SECURITY_ACTION_TYPE_NONE) && (portid_p == 0))
+	if ((ips->type != RTE_SECURITY_ACTION_TYPE_NONE) && (portid_p == 0))
 		printf("Missing portid option, falling back to non-offload\n");
 
 	if (!type_p || !portid_p) {
-		rule->type = RTE_SECURITY_ACTION_TYPE_NONE;
+		ips->type = RTE_SECURITY_ACTION_TYPE_NONE;
 		rule->portid = -1;
 	}
 
@@ -640,6 +642,7 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 {
 	uint32_t i;
 	uint8_t a, b, c, d;
+	const struct rte_ipsec_session *ips;
 
 	printf("\tspi_%s(%3u):", inbound?"in":"out", sa->spi);
 
@@ -695,8 +698,10 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 		printf("Transport ");
 		break;
 	}
+
+	ips = &sa->ips;
 	printf(" type:");
-	switch (sa->type) {
+	switch (ips->type) {
 	case RTE_SECURITY_ACTION_TYPE_NONE:
 		printf("no-offload ");
 		break;
@@ -876,6 +881,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 	uint16_t iv_length, aad_length;
 	int inline_status;
 	int32_t rc;
+	struct rte_ipsec_session *ips;
 
 	/* for ESN upper 32 bits of SQN also need to be part of AAD */
 	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
@@ -890,9 +896,10 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 		}
 		*sa = entries[i];
 		sa->seq = 0;
+		ips = ipsec_get_session(sa);
 
-		if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
-			sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
+			ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
 			if (check_eth_dev_caps(sa->portid, inbound))
 				return -EINVAL;
 		}
@@ -907,7 +914,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 			sa->dst.ip.ip4 = rte_cpu_to_be_32(sa->dst.ip.ip4);
 			break;
 		case TRANSPORT:
-			if (sa->type ==
+			if (ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
 				inline_status =
 					sa_add_address_inline_crypto(sa);
@@ -918,6 +925,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 		}
 
 		if (sa->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) {
+			struct rte_ipsec_session *ips;
 			iv_length = 16;
 
 			sa_ctx->xf[idx].a.type = RTE_CRYPTO_SYM_XFORM_AEAD;
@@ -938,11 +946,12 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			sa->xforms = &sa_ctx->xf[idx].a;
 
-			if (sa->type ==
+			ips = ipsec_get_session(sa);
+			if (ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
-				sa->type ==
+				ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
-				rc = create_inline_session(skt_ctx, sa);
+				rc = create_inline_session(skt_ctx, sa, ips);
 				if (rc != 0) {
 					RTE_LOG(ERR, IPSEC_ESP,
 						"create_inline_session() failed\n");
@@ -1100,23 +1109,11 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm, const struct ipsec_sa *ss,
 }
 
 static int
-fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
-	const struct ipsec_sa *lsa)
+fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa)
 {
 	int32_t rc = 0;
 
 	ss->sa = sa;
-	ss->type = lsa->type;
-
-	/* setup crypto section */
-	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		ss->crypto.ses = lsa->crypto_session;
-	/* setup session action type */
-	} else {
-		ss->security.ses = lsa->sec_session;
-		ss->security.ctx = lsa->security_ctx;
-		ss->security.ol_flags = lsa->ol_flags;
-	}
 
 	if (ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
 		ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
@@ -1138,6 +1135,7 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 {
 	int rc;
 	struct rte_ipsec_sa_prm prm;
+	struct rte_ipsec_session *ips;
 	struct rte_ipv4_hdr v4  = {
 		.version_ihl = IPVERSION << 4 |
 			sizeof(v4) / RTE_IPV4_IHL_MULTIPLIER,
@@ -1162,7 +1160,10 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
+	/* init processing session */
+	ips = ipsec_get_session(lsa);
+	rc = fill_ipsec_session(ips, sa);
+
 	return rc;
 }
 
@@ -1386,6 +1387,7 @@ sa_check_offloads(uint16_t port_id, uint64_t *rx_offloads,
 {
 	struct ipsec_sa *rule;
 	uint32_t idx_sa;
+	enum rte_security_session_action_type rule_type;
 
 	*rx_offloads = 0;
 	*tx_offloads = 0;
@@ -1393,8 +1395,9 @@ sa_check_offloads(uint16_t port_id, uint64_t *rx_offloads,
 	/* Check for inbound rules that use offloads and use this port */
 	for (idx_sa = 0; idx_sa < nb_sa_in; idx_sa++) {
 		rule = &sa_in[idx_sa];
-		if ((rule->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
-				rule->type ==
+		rule_type = ipsec_get_action_type(rule);
+		if ((rule_type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+				rule_type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
 				&& rule->portid == port_id)
 			*rx_offloads |= DEV_RX_OFFLOAD_SECURITY;
@@ -1403,8 +1406,9 @@ sa_check_offloads(uint16_t port_id, uint64_t *rx_offloads,
 	/* Check for outbound rules that use offloads and use this port */
 	for (idx_sa = 0; idx_sa < nb_sa_out; idx_sa++) {
 		rule = &sa_out[idx_sa];
-		if ((rule->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
-				rule->type ==
+		rule_type = ipsec_get_action_type(rule);
+		if ((rule_type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+				rule_type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
 				&& rule->portid == port_id)
 			*tx_offloads |= DEV_TX_OFFLOAD_SECURITY;
-- 
2.17.1

Re: [dpdk-dev] [PATCH v5 1/4] examples/ipsec-secgw: ipsec_sa structure cleanup

[Nicolau, Radu]

On 9/27/2019 4:54 PM, Marcin Smoczynski wrote:
> Cleanup ipsec_sa structure by removing every field that is already in
> the rte_ipsec_session structure:
>   * cryptodev/security session union
>   * action type
>   * offload flags
>   * security context
> References to abovementioned fields are changed to direct references
> to matching fields of rte_ipsec_session structure.
>
> Such refactoring is needed to introduce many sessions per SA feature,
> e.g. fallback session for inline offload processing.
>
> Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
> Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
> ---
Acked-by: Radu Nicolau <radu.nicolau@intel.com>

Re: [dpdk-dev] [PATCH v5 1/4] examples/ipsec-secgw: ipsec_sa structure cleanup

[Iremonger, Bernard]

Hi Marcin,

> -----Original Message-----
> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Marcin Smoczynski
> Sent: Friday, September 27, 2019 4:55 PM
> To: anoobj@marvell.com; akhil.goyal@nxp.com; Ananyev, Konstantin
> <konstantin.ananyev@intel.com>
> Cc: dev@dpdk.org; Smoczynski, MarcinX <marcinx.smoczynski@intel.com>
> Subject: [dpdk-dev] [PATCH v5 1/4] examples/ipsec-secgw: ipsec_sa
> structure cleanup

Minor nit.

./devtools/check-git-log.sh -1
Wrong headline format:
        examples/ipsec-secgw: ipsec_sa structure cleanup
 
> Cleanup ipsec_sa structure by removing every field that is already in the
> rte_ipsec_session structure:
>  * cryptodev/security session union
>  * action type
>  * offload flags
>  * security context
> References to abovementioned fields are changed to direct references to
> matching fields of rte_ipsec_session structure.
> 
> Such refactoring is needed to introduce many sessions per SA feature, e.g.
> fallback session for inline offload processing.
> 
> Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
> Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>

Tested-by: Bernard Iremonger <bernard.iremonger@intel.com>

[dpdk-dev] [PATCH v5 2/4] examples/ipsec-secgw: add fallback session feature

[Marcin Smoczynski]

Inline processing is limited to a specified subset of traffic. It is
often unable to handle more complicated situations, such as fragmented
traffic. When using inline processing such traffic is dropped.

Introduce fallback session for inline processing allowing processing
packets that normally would be dropped. A fallback session is
configured by adding 'fallback' keyword with 'lookaside-none' or
'lookaside-protocol' parameter to an SA configuration.

Using IPsec anti-replay window or ESN feature with fallback session is
not yet supported when primary session is of type
'inline-protocol-offload' or fallback session is 'lookaside-protocol'
because SA sequence number is not synchronized between software and
hardware sessions. Fallback sessions are also limited to ingress IPsec
traffic.

Fallback session feature is not available in the legacy mode.

Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
---
 doc/guides/sample_app_ug/ipsec_secgw.rst |  24 ++++-
 examples/ipsec-secgw/esp.c               |   4 +-
 examples/ipsec-secgw/ipsec-secgw.c       |  16 ++--
 examples/ipsec-secgw/ipsec.c             |  10 +--
 examples/ipsec-secgw/ipsec.h             |  40 +++++++--
 examples/ipsec-secgw/ipsec_process.c     |  85 ++++++++++++------
 examples/ipsec-secgw/sa.c                | 110 +++++++++++++++++++----
 7 files changed, 225 insertions(+), 64 deletions(-)

diff --git a/doc/guides/sample_app_ug/ipsec_secgw.rst b/doc/guides/sample_app_ug/ipsec_secgw.rst
index ad2d79e75..21b4b4418 100644
--- a/doc/guides/sample_app_ug/ipsec_secgw.rst
+++ b/doc/guides/sample_app_ug/ipsec_secgw.rst
@@ -401,7 +401,7 @@ The SA rule syntax is shown as follows:
 .. code-block:: console
 
     sa <dir> <spi> <cipher_algo> <cipher_key> <auth_algo> <auth_key>
-    <mode> <src_ip> <dst_ip> <action_type> <port_id>
+    <mode> <src_ip> <dst_ip> <action_type> <port_id> <fallback>
 
 where each options means:
 
@@ -573,6 +573,28 @@ where each options means:
 
    * *port_id X* X is a valid device number in decimal
 
+ ``<fallback>``
+
+ * Action type for ingress IPsec packets that inline processor failed to
+   process. IPsec anti-replay window and ESN is supported with fallback
+   processing session only when primary session is *lookaside-crypto-offload*
+   and fallback session is *lookaside-none*.
+
+   If used in conjunction with IPsec window, its width needs be increased
+   due to different processing times of inline and lookaside modes which
+   results in packet reordering.
+
+ * Optional: Yes.
+
+ * Available options:
+
+   * *lookaside-none*: use automatically chosen cryptodev to process packets
+   * *lookaside-protocol*: lookaside protocol hardware offload
+
+ * Syntax:
+
+   * *fallback lookaside-none*
+   * *fallback lookaside-protocol*
 
 Example SA rules:
 
diff --git a/examples/ipsec-secgw/esp.c b/examples/ipsec-secgw/esp.c
index c1b49da1e..bfa7ff721 100644
--- a/examples/ipsec-secgw/esp.c
+++ b/examples/ipsec-secgw/esp.c
@@ -155,7 +155,7 @@ esp_inbound_post(struct rte_mbuf *m, struct ipsec_sa *sa,
 	RTE_ASSERT(sa != NULL);
 	RTE_ASSERT(cop != NULL);
 
-	ips = ipsec_get_session(sa);
+	ips = ipsec_get_primary_session(sa);
 
 	if ((ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
 			(ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
@@ -234,7 +234,7 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 	RTE_ASSERT(m != NULL);
 	RTE_ASSERT(sa != NULL);
 
-	ips = ipsec_get_session(sa);
+	ips = ipsec_get_primary_session(sa);
 	ip_hdr_len = 0;
 
 	ip4 = rte_pktmbuf_mtod(m, struct ip *);
diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index 0d1fd6af6..641ed3767 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -189,6 +189,7 @@ static uint32_t mtu_size = RTE_ETHER_MTU;
 
 /* application wide librte_ipsec/SA parameters */
 struct app_sa_prm app_sa_prm = {.enable = 0};
+static const char *cfgfile;
 
 struct lcore_rx_queue {
 	uint16_t port_id;
@@ -1462,12 +1463,7 @@ parse_args(int32_t argc, char **argv)
 				print_usage(prgname);
 				return -1;
 			}
-			if (parse_cfg_file(optarg) < 0) {
-				printf("parsing file \"%s\" failed\n",
-					optarg);
-				print_usage(prgname);
-				return -1;
-			}
+			cfgfile = optarg;
 			f_present = 1;
 			break;
 		case 'j':
@@ -2398,6 +2394,14 @@ main(int32_t argc, char **argv)
 	if (ret < 0)
 		rte_exit(EXIT_FAILURE, "Invalid parameters\n");
 
+	/* parse configuration file */
+	if (parse_cfg_file(cfgfile) < 0) {
+		printf("parsing file \"%s\" failed\n",
+			optarg);
+		print_usage(argv[0]);
+		return -1;
+	}
+
 	if ((unprotected_port_mask & enabled_port_mask) !=
 			unprotected_port_mask)
 		rte_exit(EXIT_FAILURE, "Invalid unprotected portmask 0x%x\n",
diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index 8c60bd84b..8b0441028 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -404,7 +404,7 @@ enqueue_cop(struct cdev_qp *cqp, struct rte_crypto_op *cop)
 
 static inline void
 ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
-		struct rte_mbuf *pkts[], struct ipsec_sa *sas[],
+		struct rte_mbuf *pkts[], void *sas[],
 		uint16_t nb_pkts)
 {
 	int32_t ret = 0, i;
@@ -423,9 +423,9 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 		rte_prefetch0(pkts[i]);
 
 		priv = get_priv(pkts[i]);
-		sa = sas[i];
+		sa = ipsec_mask_saptr(sas[i]);
 		priv->sa = sa;
-		ips = ipsec_get_session(sa);
+		ips = ipsec_get_primary_session(sa);
 
 		switch (ips->type) {
 		case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL:
@@ -586,7 +586,7 @@ uint16_t
 ipsec_inbound(struct ipsec_ctx *ctx, struct rte_mbuf *pkts[],
 		uint16_t nb_pkts, uint16_t len)
 {
-	struct ipsec_sa *sas[nb_pkts];
+	void *sas[nb_pkts];
 
 	inbound_sa_lookup(ctx->sa_ctx, pkts, sas, nb_pkts);
 
@@ -606,7 +606,7 @@ uint16_t
 ipsec_outbound(struct ipsec_ctx *ctx, struct rte_mbuf *pkts[],
 		uint32_t sa_idx[], uint16_t nb_pkts, uint16_t len)
 {
-	struct ipsec_sa *sas[nb_pkts];
+	void *sas[nb_pkts];
 
 	outbound_sa_lookup(ctx->sa_ctx, sa_idx, sas, nb_pkts);
 
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index a4ad81b0e..8e075216c 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -81,12 +81,32 @@ struct app_sa_prm {
 
 extern struct app_sa_prm app_sa_prm;
 
+enum {
+	IPSEC_SESSION_PRIMARY = 0,
+	IPSEC_SESSION_FALLBACK = 1,
+	IPSEC_SESSION_MAX
+};
+
+#define IPSEC_SA_OFFLOAD_FALLBACK_FLAG (1)
+
+static inline struct ipsec_sa *
+ipsec_mask_saptr(void *ptr)
+{
+	uintptr_t i = (uintptr_t)ptr;
+	static const uintptr_t mask = IPSEC_SA_OFFLOAD_FALLBACK_FLAG;
+
+	i &= ~mask;
+
+	return (struct ipsec_sa *)i;
+}
+
 struct ipsec_sa {
-	struct rte_ipsec_session ips; /* one session per sa for now */
+	struct rte_ipsec_session sessions[IPSEC_SESSION_MAX];
 	uint32_t spi;
 	uint32_t cdev_id_qp;
 	uint64_t seq;
 	uint32_t salt;
+	uint32_t fallback_sessions;
 	enum rte_crypto_cipher_algorithm cipher_algo;
 	enum rte_crypto_auth_algorithm auth_algo;
 	enum rte_crypto_aead_algorithm aead_algo;
@@ -210,7 +230,7 @@ struct cnt_blk {
 struct traffic_type {
 	const uint8_t *data[MAX_PKT_BURST * 2];
 	struct rte_mbuf *pkts[MAX_PKT_BURST * 2];
-	struct ipsec_sa *saptr[MAX_PKT_BURST * 2];
+	void *saptr[MAX_PKT_BURST * 2];
 	uint32_t res[MAX_PKT_BURST * 2];
 	uint32_t num;
 };
@@ -278,16 +298,22 @@ get_sym_cop(struct rte_crypto_op *cop)
 }
 
 static inline struct rte_ipsec_session *
-ipsec_get_session(struct ipsec_sa *sa)
+ipsec_get_primary_session(struct ipsec_sa *sa)
+{
+	return &sa->sessions[IPSEC_SESSION_PRIMARY];
+}
+
+static inline struct rte_ipsec_session *
+ipsec_get_fallback_session(struct ipsec_sa *sa)
 {
-	return &sa->ips;
+	return &sa->sessions[IPSEC_SESSION_FALLBACK];
 }
 
 static inline enum rte_security_session_action_type
 ipsec_get_action_type(struct ipsec_sa *sa)
 {
 	struct rte_ipsec_session *ips;
-	ips = ipsec_get_session(sa);
+	ips = ipsec_get_primary_session(sa);
 	return ips->type;
 }
 
@@ -296,11 +322,11 @@ inbound_sa_check(struct sa_ctx *sa_ctx, struct rte_mbuf *m, uint32_t sa_idx);
 
 void
 inbound_sa_lookup(struct sa_ctx *sa_ctx, struct rte_mbuf *pkts[],
-		struct ipsec_sa *sa[], uint16_t nb_pkts);
+		void *sa[], uint16_t nb_pkts);
 
 void
 outbound_sa_lookup(struct sa_ctx *sa_ctx, uint32_t sa_idx[],
-		struct ipsec_sa *sa[], uint16_t nb_pkts);
+		void *sa[], uint16_t nb_pkts);
 
 void
 sp4_init(struct socket_ctx *ctx, int32_t socket_id);
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index 239d81ef6..2eb5c8b34 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -18,7 +18,6 @@
 	(((t) & RTE_IPSEC_SATP_IPV_MASK) == RTE_IPSEC_SATP_IPV4)) || \
 	((t) & RTE_IPSEC_SATP_MODE_MASK) == RTE_IPSEC_SATP_MODE_TUNLV4)
 
-
 /* helper routine to free bulk of packets */
 static inline void
 free_pkts(struct rte_mbuf *mb[], uint32_t n)
@@ -118,7 +117,7 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
  * group input packets byt the SA they belong to.
  */
 static uint32_t
-sa_group(struct ipsec_sa *sa_ptr[], struct rte_mbuf *pkts[],
+sa_group(void *sa_ptr[], struct rte_mbuf *pkts[],
 	struct rte_ipsec_group grp[], uint32_t num)
 {
 	uint32_t i, n, spi;
@@ -185,6 +184,37 @@ copy_to_trf(struct ipsec_traffic *trf, uint64_t satp, struct rte_mbuf *mb[],
 	out->num += num;
 }
 
+static uint32_t
+ipsec_prepare_crypto_group(struct ipsec_ctx *ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips, struct rte_mbuf **m,
+		unsigned int cnt)
+{
+	struct cdev_qp *cqp;
+	struct rte_crypto_op *cop[cnt];
+	uint32_t j, k;
+	struct ipsec_mbuf_metadata *priv;
+
+	cqp = &ctx->tbl[sa->cdev_id_qp];
+
+	/* for that app each mbuf has it's own crypto op */
+	for (j = 0; j != cnt; j++) {
+		priv = get_priv(m[j]);
+		cop[j] = &priv->cop;
+		/*
+		 * this is just to satisfy inbound_sa_check()
+		 * should be removed in future.
+		 */
+		priv->sa = sa;
+	}
+
+	/* prepare and enqueue crypto ops */
+	k = rte_ipsec_pkt_crypto_prepare(ips, m, cop, cnt);
+	if (k != 0)
+		enqueue_cop_bulk(cqp, cop, k);
+
+	return k;
+}
+
 /*
  * Process ipsec packets.
  * If packet belong to SA that is subject of inline-crypto,
@@ -201,18 +231,15 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 	struct ipsec_mbuf_metadata *priv;
 	struct rte_ipsec_group *pg;
 	struct rte_ipsec_session *ips;
-	struct cdev_qp *cqp;
-	struct rte_crypto_op *cop[RTE_DIM(trf->ipsec.pkts)];
 	struct rte_ipsec_group grp[RTE_DIM(trf->ipsec.pkts)];
 
 	n = sa_group(trf->ipsec.saptr, trf->ipsec.pkts, grp, trf->ipsec.num);
 
 	for (i = 0; i != n; i++) {
-
 		pg = grp + i;
-		sa = pg->id.ptr;
+		sa = ipsec_mask_saptr(pg->id.ptr);
 
-		ips = ipsec_get_session(sa);
+		ips = ipsec_get_primary_session(sa);
 
 		/* no valid HW session for that SA, try to create one */
 		if (sa == NULL || (ips->crypto.ses == NULL &&
@@ -224,6 +251,7 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 				ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
 
+			/* get SA type */
 			satp = rte_ipsec_sa_type(ips->sa);
 
 			/*
@@ -236,30 +264,33 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 				priv->sa = sa;
 			}
 
-			k = rte_ipsec_pkt_process(ips, pg->m, pg->cnt);
-			copy_to_trf(trf, satp, pg->m, k);
+			/* fallback to cryptodev with RX packets which inline
+			 * processor was unable to process
+			 */
+			if (pg->id.val & IPSEC_SA_OFFLOAD_FALLBACK_FLAG) {
+				/* offload packets to cryptodev */
+				struct rte_ipsec_session *fallback;
+
+				fallback = ipsec_get_fallback_session(sa);
+				if (fallback->crypto.ses == NULL &&
+					fill_ipsec_session(fallback, ctx, sa)
+					!= 0)
+					k = 0;
+				else
+					k = ipsec_prepare_crypto_group(ctx, sa,
+						fallback, pg->m, pg->cnt);
+			} else {
+				/* finish processing of packets successfully
+				 * decrypted by an inline processor
+				 */
+				k = rte_ipsec_pkt_process(ips, pg->m, pg->cnt);
+				copy_to_trf(trf, satp, pg->m, k);
 
+			}
 		/* enqueue packets to crypto dev */
 		} else {
-
-			cqp = &ctx->tbl[sa->cdev_id_qp];
-
-			/* for that app each mbuf has it's own crypto op */
-			for (j = 0; j != pg->cnt; j++) {
-				priv = get_priv(pg->m[j]);
-				cop[j] = &priv->cop;
-				/*
-				 * this is just to satisfy inbound_sa_check()
-				 * should be removed in future.
-				 */
-				priv->sa = sa;
-			}
-
-			/* prepare and enqueue crypto ops */
-			k = rte_ipsec_pkt_crypto_prepare(ips, pg->m, cop,
+			k = ipsec_prepare_crypto_group(ctx, sa, ips, pg->m,
 				pg->cnt);
-			if (k != 0)
-				enqueue_cop_bulk(cqp, cop, k);
 		}
 
 		/* drop packets that cannot be enqueued/processed */
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index a3d8e4545..700d52490 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -31,6 +31,8 @@
 
 #define IP6_FULL_MASK (sizeof(((struct ip_addr *)NULL)->ip.ip6.ip6) * CHAR_BIT)
 
+#define MBUF_NO_SEC_OFFLOAD(m) ((m->ol_flags & PKT_RX_SEC_OFFLOAD) == 0)
+
 struct supported_cipher_algo {
 	const char *keyword;
 	enum rte_crypto_cipher_algorithm algo;
@@ -235,6 +237,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	uint32_t mode_p = 0;
 	uint32_t type_p = 0;
 	uint32_t portid_p = 0;
+	uint32_t fallback_p = 0;
 
 	if (strcmp(tokens[0], "in") == 0) {
 		ri = &nb_sa_in;
@@ -245,6 +248,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 			return;
 
 		rule = &sa_in[*ri];
+		rule->direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS;
 	} else {
 		ri = &nb_sa_out;
 
@@ -254,6 +258,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 			return;
 
 		rule = &sa_out[*ri];
+		rule->direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS;
 	}
 
 	/* spi number */
@@ -263,7 +268,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	if (atoi(tokens[1]) == INVALID_SPI)
 		return;
 	rule->spi = atoi(tokens[1]);
-	ips = ipsec_get_session(rule);
+	ips = ipsec_get_primary_session(rule);
 
 	for (ti = 2; ti < n_tokens; ti++) {
 		if (strcmp(tokens[ti], "mode") == 0) {
@@ -596,6 +601,43 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 			continue;
 		}
 
+		if (strcmp(tokens[ti], "fallback") == 0) {
+			struct rte_ipsec_session *fb;
+
+			APP_CHECK(app_sa_prm.enable, status, "Fallback session "
+				"not allowed for legacy mode.");
+			if (status->status < 0)
+				return;
+			APP_CHECK(rule->direction ==
+				RTE_SECURITY_IPSEC_SA_DIR_INGRESS, status,
+				"Fallback session not allowed for egress "
+				"rule");
+			if (status->status < 0)
+				return;
+			APP_CHECK_PRESENCE(fallback_p, tokens[ti], status);
+			if (status->status < 0)
+				return;
+			INCREMENT_TOKEN_INDEX(ti, n_tokens, status);
+			if (status->status < 0)
+				return;
+			fb = ipsec_get_fallback_session(rule);
+			if (strcmp(tokens[ti], "lookaside-none") == 0) {
+				fb->type = RTE_SECURITY_ACTION_TYPE_NONE;
+			} else if (strcmp(tokens[ti], "lookaside-protocol")
+				== 0) {
+				fb->type =
+				RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL;
+			} else {
+				APP_CHECK(0, status, "unrecognized fallback "
+					"type %s.", tokens[ti]);
+				return;
+			}
+
+			rule->fallback_sessions = 1;
+			fallback_p = 1;
+			continue;
+		}
+
 		/* unrecognizeable input */
 		APP_CHECK(0, status, "unrecognized input \"%s\"",
 			tokens[ti]);
@@ -643,6 +685,7 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 	uint32_t i;
 	uint8_t a, b, c, d;
 	const struct rte_ipsec_session *ips;
+	const struct rte_ipsec_session *fallback_ips;
 
 	printf("\tspi_%s(%3u):", inbound?"in":"out", sa->spi);
 
@@ -699,7 +742,7 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 		break;
 	}
 
-	ips = &sa->ips;
+	ips = &sa->sessions[IPSEC_SESSION_PRIMARY];
 	printf(" type:");
 	switch (ips->type) {
 	case RTE_SECURITY_ACTION_TYPE_NONE:
@@ -715,6 +758,22 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 		printf("lookaside-protocol-offload ");
 		break;
 	}
+
+	fallback_ips = &sa->sessions[IPSEC_SESSION_FALLBACK];
+	if (fallback_ips != NULL && sa->fallback_sessions > 0) {
+		printf("inline fallback:");
+		switch (fallback_ips->type) {
+		case RTE_SECURITY_ACTION_TYPE_NONE:
+			printf("lookaside-none ");
+			break;
+		case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL:
+			printf("lookaside-protocol ");
+			break;
+		default:
+			printf("invalid ");
+			break;
+		}
+	}
 	printf("\n");
 }
 
@@ -896,7 +955,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 		}
 		*sa = entries[i];
 		sa->seq = 0;
-		ips = ipsec_get_session(sa);
+		ips = ipsec_get_primary_session(sa);
 
 		if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
 			ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
@@ -904,9 +963,6 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 				return -EINVAL;
 		}
 
-		sa->direction = (inbound == 1) ?
-				RTE_SECURITY_IPSEC_SA_DIR_INGRESS :
-				RTE_SECURITY_IPSEC_SA_DIR_EGRESS;
 
 		switch (WITHOUT_TRANSPORT_VERSION(sa->flags)) {
 		case IP4_TUNNEL:
@@ -946,7 +1002,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			sa->xforms = &sa_ctx->xf[idx].a;
 
-			ips = ipsec_get_session(sa);
+			ips = ipsec_get_primary_session(sa);
 			if (ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
 				ips->type ==
@@ -1160,9 +1216,15 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	/* init processing session */
-	ips = ipsec_get_session(lsa);
+	/* init primary processing session */
+	ips = ipsec_get_primary_session(lsa);
 	rc = fill_ipsec_session(ips, sa);
+	if (rc != 0)
+		return rc;
+
+	/* init inline fallback processing session */
+	if (lsa->fallback_sessions == 1)
+		rc = fill_ipsec_session(ipsec_get_fallback_session(lsa), sa);
 
 	return rc;
 }
@@ -1318,13 +1380,14 @@ inbound_sa_check(struct sa_ctx *sa_ctx, struct rte_mbuf *m, uint32_t sa_idx)
 
 static inline void
 single_inbound_lookup(struct ipsec_sa *sadb, struct rte_mbuf *pkt,
-		struct ipsec_sa **sa_ret)
+		void **sa_ret)
 {
 	struct rte_esp_hdr *esp;
 	struct ip *ip;
 	uint32_t *src4_addr;
 	uint8_t *src6_addr;
 	struct ipsec_sa *sa;
+	void *result_sa;
 
 	*sa_ret = NULL;
 
@@ -1334,33 +1397,48 @@ single_inbound_lookup(struct ipsec_sa *sadb, struct rte_mbuf *pkt,
 	if (esp->spi == INVALID_SPI)
 		return;
 
-	sa = &sadb[SPI2IDX(rte_be_to_cpu_32(esp->spi))];
+	result_sa = sa = &sadb[SPI2IDX(rte_be_to_cpu_32(esp->spi))];
 	if (rte_be_to_cpu_32(esp->spi) != sa->spi)
 		return;
 
+	/*
+	 * Mark need for inline offload fallback on the LSB of SA pointer.
+	 * Thanks to packet grouping mechanism which ipsec_process is using
+	 * packets marked for fallback processing will form separate group.
+	 *
+	 * Because it is not safe to use SA pointer it is casted to generic
+	 * pointer to prevent from unintentional use. Use ipsec_mask_saptr
+	 * to get valid struct pointer.
+	 */
+	if (MBUF_NO_SEC_OFFLOAD(pkt) && sa->fallback_sessions > 0) {
+		uintptr_t intsa = (uintptr_t)sa;
+		intsa |= IPSEC_SA_OFFLOAD_FALLBACK_FLAG;
+		result_sa = (void *)intsa;
+	}
+
 	switch (WITHOUT_TRANSPORT_VERSION(sa->flags)) {
 	case IP4_TUNNEL:
 		src4_addr = RTE_PTR_ADD(ip, offsetof(struct ip, ip_src));
 		if ((ip->ip_v == IPVERSION) &&
 				(sa->src.ip.ip4 == *src4_addr) &&
 				(sa->dst.ip.ip4 == *(src4_addr + 1)))
-			*sa_ret = sa;
+			*sa_ret = result_sa;
 		break;
 	case IP6_TUNNEL:
 		src6_addr = RTE_PTR_ADD(ip, offsetof(struct ip6_hdr, ip6_src));
 		if ((ip->ip_v == IP6_VERSION) &&
 				!memcmp(&sa->src.ip.ip6.ip6, src6_addr, 16) &&
 				!memcmp(&sa->dst.ip.ip6.ip6, src6_addr + 16, 16))
-			*sa_ret = sa;
+			*sa_ret = result_sa;
 		break;
 	case TRANSPORT:
-		*sa_ret = sa;
+		*sa_ret = result_sa;
 	}
 }
 
 void
 inbound_sa_lookup(struct sa_ctx *sa_ctx, struct rte_mbuf *pkts[],
-		struct ipsec_sa *sa[], uint16_t nb_pkts)
+		void *sa[], uint16_t nb_pkts)
 {
 	uint32_t i;
 
@@ -1370,7 +1448,7 @@ inbound_sa_lookup(struct sa_ctx *sa_ctx, struct rte_mbuf *pkts[],
 
 void
 outbound_sa_lookup(struct sa_ctx *sa_ctx, uint32_t sa_idx[],
-		struct ipsec_sa *sa[], uint16_t nb_pkts)
+		void *sa[], uint16_t nb_pkts)
 {
 	uint32_t i;
 
-- 
2.17.1

Re: [dpdk-dev] [PATCH v5 2/4] examples/ipsec-secgw: add fallback session feature

[Ananyev, Konstantin]

Hi Marcin,

> 
> diff --git a/doc/guides/sample_app_ug/ipsec_secgw.rst b/doc/guides/sample_app_ug/ipsec_secgw.rst
> index ad2d79e75..21b4b4418 100644
> --- a/doc/guides/sample_app_ug/ipsec_secgw.rst
> +++ b/doc/guides/sample_app_ug/ipsec_secgw.rst
> @@ -401,7 +401,7 @@ The SA rule syntax is shown as follows:
>  .. code-block:: console
> 
>      sa <dir> <spi> <cipher_algo> <cipher_key> <auth_algo> <auth_key>
> -    <mode> <src_ip> <dst_ip> <action_type> <port_id>
> +    <mode> <src_ip> <dst_ip> <action_type> <port_id> <fallback>
> 
>  where each options means:
> 
> @@ -573,6 +573,28 @@ where each options means:
> 
>     * *port_id X* X is a valid device number in decimal
> 
> + ``<fallback>``
> +
> + * Action type for ingress IPsec packets that inline processor failed to
> +   process. IPsec anti-replay window and ESN is supported with fallback
> +   processing session only when primary session is *lookaside-crypto-offload*

You probably meant here 'inline-crypto-offload'?

> +   and fallback session is *lookaside-none*.

Re: [dpdk-dev] [PATCH v5 2/4] examples/ipsec-secgw: add fallback session feature

[Smoczynski, MarcinX]

> -----Original Message-----
> From: Ananyev, Konstantin
> Sent: Sunday, September 29, 2019 7:04 PM
> To: Smoczynski, MarcinX <marcinx.smoczynski@intel.com>;
> anoobj@marvell.com; akhil.goyal@nxp.com
> Cc: dev@dpdk.org
> Subject: RE: [PATCH v5 2/4] examples/ipsec-secgw: add fallback session
> feature
> 
> Hi Marcin,
> 
> >
> > diff --git a/doc/guides/sample_app_ug/ipsec_secgw.rst
> b/doc/guides/sample_app_ug/ipsec_secgw.rst
> > index ad2d79e75..21b4b4418 100644
> > --- a/doc/guides/sample_app_ug/ipsec_secgw.rst
> > +++ b/doc/guides/sample_app_ug/ipsec_secgw.rst
> > @@ -401,7 +401,7 @@ The SA rule syntax is shown as follows:
> >  .. code-block:: console
> >
> >      sa <dir> <spi> <cipher_algo> <cipher_key> <auth_algo> <auth_key>
> > -    <mode> <src_ip> <dst_ip> <action_type> <port_id>
> > +    <mode> <src_ip> <dst_ip> <action_type> <port_id> <fallback>
> >
> >  where each options means:
> >
> > @@ -573,6 +573,28 @@ where each options means:
> >
> >     * *port_id X* X is a valid device number in decimal
> >
> > + ``<fallback>``
> > +
> > + * Action type for ingress IPsec packets that inline processor failed to
> > +   process. IPsec anti-replay window and ESN is supported with fallback
> > +   processing session only when primary session is *lookaside-crypto-
> offload*
> 
> You probably meant here 'inline-crypto-offload'?
> 
> > +   and fallback session is *lookaside-none*.

Yes, thanks.
Marcin.

Re: [dpdk-dev] [PATCH v5 2/4] examples/ipsec-secgw: add fallback session feature

[Iremonger, Bernard]

> -----Original Message-----
> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Marcin Smoczynski
> Sent: Friday, September 27, 2019 4:55 PM
> To: anoobj@marvell.com; akhil.goyal@nxp.com; Ananyev, Konstantin
> <konstantin.ananyev@intel.com>
> Cc: dev@dpdk.org; Smoczynski, MarcinX <marcinx.smoczynski@intel.com>
> Subject: [dpdk-dev] [PATCH v5 2/4] examples/ipsec-secgw: add fallback
> session feature
> 
> Inline processing is limited to a specified subset of traffic. It is often unable to
> handle more complicated situations, such as fragmented traffic. When using
> inline processing such traffic is dropped.
> 
> Introduce fallback session for inline processing allowing processing packets
> that normally would be dropped. A fallback session is configured by adding
> 'fallback' keyword with 'lookaside-none' or 'lookaside-protocol' parameter to
> an SA configuration.
> 
> Using IPsec anti-replay window or ESN feature with fallback session is not yet
> supported when primary session is of type 'inline-protocol-offload' or
> fallback session is 'lookaside-protocol'
> because SA sequence number is not synchronized between software and
> hardware sessions. Fallback sessions are also limited to ingress IPsec traffic.
> 
> Fallback session feature is not available in the legacy mode.
> 
> Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
> Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>

Tested-by: Bernard Iremonger <bernard.iremonger@intel.com>

[dpdk-dev] [PATCH v5 3/4] examples/ipsec-secgw: add frag TTL cmdline option

[Marcin Smoczynski]

Due to fragment loss on highly saturated links and long fragment
lifetime, ipsec-secgw application quickly runs out of free reassembly
buckets. As a result new fragments are being dropped.

Introduce --frag-ttl option which allow user to lower default fragment
lifitime which solves problem of saturated reassembly buckets with high
bandwidth fragmented traffic.

Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
---
 doc/guides/sample_app_ug/ipsec_secgw.rst |  7 +++++
 examples/ipsec-secgw/ipsec-secgw.c       | 40 ++++++++++++++++++------
 2 files changed, 37 insertions(+), 10 deletions(-)

diff --git a/doc/guides/sample_app_ug/ipsec_secgw.rst b/doc/guides/sample_app_ug/ipsec_secgw.rst
index 21b4b4418..55b65d7bc 100644
--- a/doc/guides/sample_app_ug/ipsec_secgw.rst
+++ b/doc/guides/sample_app_ug/ipsec_secgw.rst
@@ -154,6 +154,13 @@ Where:
     Incoming packets with length bigger then MTU will be discarded.
     Default value: 1500.
 
+*   ``--frag-ttl FRAG_TTL_NS``: fragment lifetime (in nanoseconds).
+    If packet is not reassembled within this time, received fragments
+    will be discarded. Fragment lifetime should be decreased when
+    there is a high fragmented traffic loss in high bandwidth networks.
+    Should be lower for for low number of reassembly buckets.
+    Valid values: from 1 ns to 10 s. Default value: 10000000 (10 s).
+
 *   ``--reassemble NUM``: max number of entries in reassemble fragment table.
     Zero value disables reassembly functionality.
     Default value: 0.
diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index 641ed3767..1d415ace8 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -112,7 +112,7 @@ static uint16_t nb_txd = IPSEC_SECGW_TX_DESC_DEFAULT;
 		0, 0)
 
 #define	FRAG_TBL_BUCKET_ENTRIES	4
-#define	FRAG_TTL_MS		(10 * MS_PER_S)
+#define	MAX_FRAG_TTL_NS		(10LL * NS_PER_S)
 
 #define MTU_TO_FRAMELEN(x)	((x) + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN)
 
@@ -135,6 +135,7 @@ struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS] = {
 #define CMD_LINE_OPT_TX_OFFLOAD		"txoffload"
 #define CMD_LINE_OPT_REASSEMBLE		"reassemble"
 #define CMD_LINE_OPT_MTU		"mtu"
+#define CMD_LINE_OPT_FRAG_TTL		"frag-ttl"
 
 enum {
 	/* long options mapped to a short option */
@@ -150,6 +151,7 @@ enum {
 	CMD_LINE_OPT_TX_OFFLOAD_NUM,
 	CMD_LINE_OPT_REASSEMBLE_NUM,
 	CMD_LINE_OPT_MTU_NUM,
+	CMD_LINE_OPT_FRAG_TTL_NUM,
 };
 
 static const struct option lgopts[] = {
@@ -160,6 +162,7 @@ static const struct option lgopts[] = {
 	{CMD_LINE_OPT_TX_OFFLOAD, 1, 0, CMD_LINE_OPT_TX_OFFLOAD_NUM},
 	{CMD_LINE_OPT_REASSEMBLE, 1, 0, CMD_LINE_OPT_REASSEMBLE_NUM},
 	{CMD_LINE_OPT_MTU, 1, 0, CMD_LINE_OPT_MTU_NUM},
+	{CMD_LINE_OPT_FRAG_TTL, 1, 0, CMD_LINE_OPT_FRAG_TTL_NUM},
 	{NULL, 0, 0, 0}
 };
 
@@ -186,6 +189,7 @@ static uint64_t dev_tx_offload = UINT64_MAX;
 static uint32_t frag_tbl_sz;
 static uint32_t frame_buf_size = RTE_MBUF_DEFAULT_BUF_SIZE;
 static uint32_t mtu_size = RTE_ETHER_MTU;
+static uint64_t frag_ttl_ns = MAX_FRAG_TTL_NS;
 
 /* application wide librte_ipsec/SA parameters */
 struct app_sa_prm app_sa_prm = {.enable = 0};
@@ -1302,6 +1306,9 @@ print_usage(const char *prgname)
 		": MTU value on all ports (default value: 1500)\n"
 		"    outgoing packets with bigger size will be fragmented\n"
 		"    incoming packets with bigger size will be discarded\n"
+		"  --" CMD_LINE_OPT_FRAG_TTL " FRAG_TTL_NS"
+		": fragments lifetime in nanoseconds, default\n"
+		"    and maximum value is 10.000.000.000 ns (10 s)\n"
 		"\n",
 		prgname);
 }
@@ -1338,14 +1345,15 @@ parse_portmask(const char *portmask)
 	return pm;
 }
 
-static int32_t
+static int64_t
 parse_decimal(const char *str)
 {
 	char *end = NULL;
-	unsigned long num;
+	uint64_t num;
 
-	num = strtoul(str, &end, 10);
-	if ((str[0] == '\0') || (end == NULL) || (*end != '\0'))
+	num = strtoull(str, &end, 10);
+	if ((str[0] == '\0') || (end == NULL) || (*end != '\0')
+		|| num > INT64_MAX)
 		return -1;
 
 	return num;
@@ -1419,12 +1427,14 @@ print_app_sa_prm(const struct app_sa_prm *prm)
 	printf("replay window size: %u\n", prm->window_size);
 	printf("ESN: %s\n", (prm->enable_esn == 0) ? "disabled" : "enabled");
 	printf("SA flags: %#" PRIx64 "\n", prm->flags);
+	printf("Frag TTL: %" PRIu64 " ns\n", frag_ttl_ns);
 }
 
 static int32_t
 parse_args(int32_t argc, char **argv)
 {
-	int32_t opt, ret;
+	int opt;
+	int64_t ret;
 	char **argvopt;
 	int32_t option_index;
 	char *prgname = argv[0];
@@ -1503,7 +1513,7 @@ parse_args(int32_t argc, char **argv)
 			break;
 		case CMD_LINE_OPT_SINGLE_SA_NUM:
 			ret = parse_decimal(optarg);
-			if (ret == -1) {
+			if (ret == -1 || ret > UINT32_MAX) {
 				printf("Invalid argument[sa_idx]\n");
 				print_usage(prgname);
 				return -1;
@@ -1546,7 +1556,7 @@ parse_args(int32_t argc, char **argv)
 			break;
 		case CMD_LINE_OPT_REASSEMBLE_NUM:
 			ret = parse_decimal(optarg);
-			if (ret < 0) {
+			if (ret < 0 || ret > UINT32_MAX) {
 				printf("Invalid argument for \'%s\': %s\n",
 					CMD_LINE_OPT_REASSEMBLE, optarg);
 				print_usage(prgname);
@@ -1564,6 +1574,16 @@ parse_args(int32_t argc, char **argv)
 			}
 			mtu_size = ret;
 			break;
+		case CMD_LINE_OPT_FRAG_TTL_NUM:
+			ret = parse_decimal(optarg);
+			if (ret < 0 || ret > MAX_FRAG_TTL_NS) {
+				printf("Invalid argument for \'%s\': %s\n",
+					CMD_LINE_OPT_MTU, optarg);
+				print_usage(prgname);
+				return -1;
+			}
+			frag_ttl_ns = ret;
+			break;
 		default:
 			print_usage(prgname);
 			return -1;
@@ -2324,8 +2344,8 @@ reassemble_lcore_init(struct lcore_conf *lc, uint32_t cid)
 
 	/* create fragment table */
 	sid = rte_lcore_to_socket_id(cid);
-	frag_cycles = (rte_get_tsc_hz() + MS_PER_S - 1) /
-		MS_PER_S * FRAG_TTL_MS;
+	frag_cycles = (rte_get_tsc_hz() + NS_PER_S - 1) /
+		NS_PER_S * frag_ttl_ns;
 
 	lc->frag.tbl = rte_ip_frag_table_create(frag_tbl_sz,
 		FRAG_TBL_BUCKET_ENTRIES, frag_tbl_sz, frag_cycles, sid);
-- 
2.17.1

Re: [dpdk-dev] [PATCH v5 3/4] examples/ipsec-secgw: add frag TTL cmdline option

[Ananyev, Konstantin]

> 
> Due to fragment loss on highly saturated links and long fragment
> lifetime, ipsec-secgw application quickly runs out of free reassembly
> buckets. As a result new fragments are being dropped.
> 
> Introduce --frag-ttl option which allow user to lower default fragment
> lifitime which solves problem of saturated reassembly buckets with high
> bandwidth fragmented traffic.
> 
> Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
> ---

Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
Tested-by: Konstantin Ananyev <konstantin.ananyev@intel.com>

> 2.17.1

[dpdk-dev] [PATCH v5 4/4] examples/ipsec-secgw: add offload fallback tests

[Marcin Smoczynski]

Add tests for offload fallback feature; add inbound config modificator
SGW_CFG_XPRM_IN (offload fallback setting can be set only for inbound
SAs). Tests are using cryptodev for outbound SA.

To test fragmentation with QAT set:
MULTI_SEG_TEST="--reassemble=4096 --cryptodev_mask=0x5555"

Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
---
 examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh         | 4 ++--
 .../test/trs_aesgcm_inline_crypto_fallback_defs.sh          | 5 +++++
 examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh         | 6 ++++--
 .../test/tun_aesgcm_inline_crypto_fallback_defs.sh          | 5 +++++
 4 files changed, 16 insertions(+), 4 deletions(-)
 create mode 100644 examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
 create mode 100644 examples/ipsec-secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh

diff --git a/examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh b/examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh
index f6c5bf5a7..17f2f86d2 100644
--- a/examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh
+++ b/examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh
@@ -29,11 +29,11 @@ sp ipv6 out esp bypass pri 1 sport 0:65535 dport 0:65535
 #SA in rules
 sa in 7 aead_algo aes-128-gcm \
 aead_key de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef \
-mode transport ${SGW_CFG_XPRM}
+mode transport ${SGW_CFG_XPRM} ${SGW_CFG_XPRM_IN}
 
 sa in 9 aead_algo aes-128-gcm \
 aead_key de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef \
-mode transport ${SGW_CFG_XPRM}
+mode transport ${SGW_CFG_XPRM} ${SGW_CFG_XPRM_IN}
 
 #SA out rules
 sa out 7 aead_algo aes-128-gcm \
diff --git a/examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh b/examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
new file mode 100644
index 000000000..875a7457d
--- /dev/null
+++ b/examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
@@ -0,0 +1,5 @@
+#! /bin/bash
+
+. ${DIR}/trs_aesgcm_defs.sh
+
+SGW_CFG_XPRM_IN='port_id 0 type inline-crypto-offload fallback lookaside-none'
diff --git a/examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh b/examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh
index 278377967..7490baded 100644
--- a/examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh
+++ b/examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh
@@ -29,11 +29,13 @@ sp ipv6 out esp bypass pri 1 sport 0:65535 dport 0:65535
 #SA in rules
 sa in 7 aead_algo aes-128-gcm \
 aead_key de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef \
-mode ipv4-tunnel src ${REMOTE_IPV4} dst ${LOCAL_IPV4} ${SGW_CFG_XPRM}
+mode ipv4-tunnel src ${REMOTE_IPV4} dst ${LOCAL_IPV4} ${SGW_CFG_XPRM} \
+${SGW_CFG_XPRM_IN}
 
 sa in 9 aead_algo aes-128-gcm \
 aead_key de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef \
-mode ipv6-tunnel src ${REMOTE_IPV6} dst ${LOCAL_IPV6} ${SGW_CFG_XPRM}
+mode ipv6-tunnel src ${REMOTE_IPV6} dst ${LOCAL_IPV6} ${SGW_CFG_XPRM} \
+${SGW_CFG_XPRM_IN}
 
 #SA out rules
 sa out 7 aead_algo aes-128-gcm \
diff --git a/examples/ipsec-secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh b/examples/ipsec-secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh
new file mode 100644
index 000000000..696848432
--- /dev/null
+++ b/examples/ipsec-secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh
@@ -0,0 +1,5 @@
+#! /bin/bash
+
+. ${DIR}/tun_aesgcm_defs.sh
+
+SGW_CFG_XPRM_IN='port_id 0 type inline-crypto-offload fallback lookaside-none'
-- 
2.17.1

Re: [dpdk-dev] [PATCH v5 4/4] examples/ipsec-secgw: add offload fallback tests

[Iremonger, Bernard]

Hi Marcin,

> -----Original Message-----
> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Marcin Smoczynski
> Sent: Friday, September 27, 2019 4:55 PM
> To: anoobj@marvell.com; akhil.goyal@nxp.com; Ananyev, Konstantin
> <konstantin.ananyev@intel.com>
> Cc: dev@dpdk.org; Smoczynski, MarcinX <marcinx.smoczynski@intel.com>
> Subject: [dpdk-dev] [PATCH v5 4/4] examples/ipsec-secgw: add offload
> fallback tests
> 
> Add tests for offload fallback feature; add inbound config modificator
> SGW_CFG_XPRM_IN (offload fallback setting can be set only for inbound
> SAs). Tests are using cryptodev for outbound SA.
> 
> To test fragmentation with QAT set:
> MULTI_SEG_TEST="--reassemble=4096 --cryptodev_mask=0x5555"

The environment variable MULTI_SEG_TEST should be documented in the ipsec-secgw guide, section
49.6 Test directory, along side the other environment variables.

 
> Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
> Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>

Tested-by:  Bernard Iremonger <bernard.iremonger@intel.com>

> ---
>  examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh         | 4 ++--
>  .../test/trs_aesgcm_inline_crypto_fallback_defs.sh          | 5 +++++
>  examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh         | 6 ++++--
>  .../test/tun_aesgcm_inline_crypto_fallback_defs.sh          | 5 +++++
>  4 files changed, 16 insertions(+), 4 deletions(-)  create mode 100644
> examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
>  create mode 100644 examples/ipsec-
> secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh
> 
> diff --git a/examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh
> b/examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh
> index f6c5bf5a7..17f2f86d2 100644
> --- a/examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh
> +++ b/examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh
> @@ -29,11 +29,11 @@ sp ipv6 out esp bypass pri 1 sport 0:65535 dport
> 0:65535  #SA in rules  sa in 7 aead_algo aes-128-gcm \  aead_key
> de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef \ -mode
> transport ${SGW_CFG_XPRM}
> +mode transport ${SGW_CFG_XPRM} ${SGW_CFG_XPRM_IN}
> 
>  sa in 9 aead_algo aes-128-gcm \
>  aead_key de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef \ -
> mode transport ${SGW_CFG_XPRM}
> +mode transport ${SGW_CFG_XPRM} ${SGW_CFG_XPRM_IN}
> 
>  #SA out rules
>  sa out 7 aead_algo aes-128-gcm \
> diff --git a/examples/ipsec-
> secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh b/examples/ipsec-
> secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
> new file mode 100644
> index 000000000..875a7457d
> --- /dev/null
> +++ b/examples/ipsec-
> secgw/test/trs_aesgcm_inline_crypto_fallback_defs.s
> +++ h
> @@ -0,0 +1,5 @@
> +#! /bin/bash
> +
> +. ${DIR}/trs_aesgcm_defs.sh
> +
> +SGW_CFG_XPRM_IN='port_id 0 type inline-crypto-offload fallback
> lookaside-none'
> diff --git a/examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh
> b/examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh
> index 278377967..7490baded 100644
> --- a/examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh
> +++ b/examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh
> @@ -29,11 +29,13 @@ sp ipv6 out esp bypass pri 1 sport 0:65535 dport
> 0:65535  #SA in rules  sa in 7 aead_algo aes-128-gcm \  aead_key
> de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef \ -mode ipv4-
> tunnel src ${REMOTE_IPV4} dst ${LOCAL_IPV4} ${SGW_CFG_XPRM}
> +mode ipv4-tunnel src ${REMOTE_IPV4} dst ${LOCAL_IPV4}
> ${SGW_CFG_XPRM} \
> +${SGW_CFG_XPRM_IN}
> 
>  sa in 9 aead_algo aes-128-gcm \
>  aead_key de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef \ -
> mode ipv6-tunnel src ${REMOTE_IPV6} dst ${LOCAL_IPV6}
> ${SGW_CFG_XPRM}
> +mode ipv6-tunnel src ${REMOTE_IPV6} dst ${LOCAL_IPV6}
> ${SGW_CFG_XPRM} \
> +${SGW_CFG_XPRM_IN}
> 
>  #SA out rules
>  sa out 7 aead_algo aes-128-gcm \
> diff --git a/examples/ipsec-
> secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh b/examples/ipsec-
> secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh
> new file mode 100644
> index 000000000..696848432
> --- /dev/null
> +++ b/examples/ipsec-
> secgw/test/tun_aesgcm_inline_crypto_fallback_defs.s
> +++ h
> @@ -0,0 +1,5 @@
> +#! /bin/bash
> +
> +. ${DIR}/tun_aesgcm_defs.sh
> +
> +SGW_CFG_XPRM_IN='port_id 0 type inline-crypto-offload fallback
> lookaside-none'
> --
> 2.17.1

Regards,

Bernard.

[dpdk-dev] [PATCH v6 0/4] add fallback session

[Marcin Smoczynski]

Add fallback session feature allowing to process packets that inline
processor is unable to handle (e.g. fragmented traffic). Processing
takes place in a secondary session defined for SA in a configuration
file.

This feature is limited to ingress IPsec traffic only. IPsec
anti-replay window and ESN are supported in conjunction with fallback
session when following conditions are met:
 * primary session is 'inline-crypto-offload,
 * fallback sessions is 'lookaside-none'.
Due to different processing times of inline and lookaside modes,
fallback session introduces some packet reordering, therefore when
using with IPsec window, its value should be increased.

v5 to v6 changes:
 - add sanity check: fail to parse SA configuration if fallback session
   is configured but primary session is not inline crypto
 - update documentation for ipsec-secgw tests (MULTI_SEG_TESTS var
   described)
 - add release notes
 - minor commit log changes

v4 to v5 changes:
 - fix build errors related to frag TTL command line option and
   parse_decimal method

v3 to v4 changes:
 - add info about packet reordering to the documentation regarding
   fallback session
 - add patch with --frag-ttl command line option which allows to change
   fragment lifetime

v2 to v3 changes:
 - doc and commit log update - explicitly state feature limitations

v1 to v2 changes:
 - disable fallback offload for outbound SAs
 - add test scripts

Marcin Smoczynski (4):
  examples/ipsec-secgw: sa structure cleanup
  examples/ipsec-secgw: add fallback session feature
  examples/ipsec-secgw: add frag TTL cmdline option
  examples/ipsec-secgw: add offload fallback tests

 doc/guides/rel_notes/release_19_11.rst        |   8 +
 doc/guides/sample_app_ug/ipsec_secgw.rst      |  36 +++-
 examples/ipsec-secgw/esp.c                    |  35 ++--
 examples/ipsec-secgw/ipsec-secgw.c            |  56 ++++--
 examples/ipsec-secgw/ipsec.c                  |  99 +++++-----
 examples/ipsec-secgw/ipsec.h                  |  61 +++++--
 examples/ipsec-secgw/ipsec_process.c          | 113 +++++++-----
 examples/ipsec-secgw/sa.c                     | 170 +++++++++++++-----
 .../test/trs_aesgcm_common_defs.sh            |   4 +-
 .../trs_aesgcm_inline_crypto_fallback_defs.sh |   5 +
 .../test/tun_aesgcm_common_defs.sh            |   6 +-
 .../tun_aesgcm_inline_crypto_fallback_defs.sh |   5 +
 12 files changed, 421 insertions(+), 177 deletions(-)
 create mode 100644 examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
 create mode 100644 examples/ipsec-secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh

--
2.17.1

[dpdk-dev] [PATCH v6 1/4] examples/ipsec-secgw: sa structure cleanup

[Marcin Smoczynski]

Cleanup ipsec_sa structure by removing every field that is already in
the rte_ipsec_session structure:
 * cryptodev/security session union
 * action type
 * offload flags
 * security context
References to abovementioned fields are changed to direct references
to matching fields of rte_ipsec_session structure.

Such refactoring is needed to introduce many sessions per SA feature,
e.g. fallback session for inline offload processing.

Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
Acked-by: Radu Nicolau <radu.nicolau@intel.com>
Tested-by: Bernard Iremonger <bernard.iremonger@intel.com>
Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
---
 examples/ipsec-secgw/esp.c           | 35 +++++++----
 examples/ipsec-secgw/ipsec.c         | 91 +++++++++++++++-------------
 examples/ipsec-secgw/ipsec.h         | 27 ++++++---
 examples/ipsec-secgw/ipsec_process.c | 30 ++++-----
 examples/ipsec-secgw/sa.c            | 66 ++++++++++----------
 5 files changed, 137 insertions(+), 112 deletions(-)

diff --git a/examples/ipsec-secgw/esp.c b/examples/ipsec-secgw/esp.c
index d6d7b1256..c1b49da1e 100644
--- a/examples/ipsec-secgw/esp.c
+++ b/examples/ipsec-secgw/esp.c
@@ -30,7 +30,8 @@ esp_inbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 	int32_t payload_len, ip_hdr_len;
 
 	RTE_ASSERT(sa != NULL);
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
+	if (ipsec_get_action_type(sa) ==
+			RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
 		return 0;
 
 	RTE_ASSERT(m != NULL);
@@ -148,13 +149,16 @@ esp_inbound_post(struct rte_mbuf *m, struct ipsec_sa *sa,
 	uint8_t *nexthdr, *pad_len;
 	uint8_t *padding;
 	uint16_t i;
+	struct rte_ipsec_session *ips;
 
 	RTE_ASSERT(m != NULL);
 	RTE_ASSERT(sa != NULL);
 	RTE_ASSERT(cop != NULL);
 
-	if ((sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
-			(sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
+	ips = ipsec_get_session(sa);
+
+	if ((ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
+			(ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
 		if (m->ol_flags & PKT_RX_SEC_OFFLOAD) {
 			if (m->ol_flags & PKT_RX_SEC_OFFLOAD_FAILED)
 				cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
@@ -169,8 +173,8 @@ esp_inbound_post(struct rte_mbuf *m, struct ipsec_sa *sa,
 		return -1;
 	}
 
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO &&
-	    sa->ol_flags & RTE_SECURITY_RX_HW_TRAILER_OFFLOAD) {
+	if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO &&
+		ips->security.ol_flags & RTE_SECURITY_RX_HW_TRAILER_OFFLOAD) {
 		nexthdr = &m->inner_esp_next_proto;
 	} else {
 		nexthdr = rte_pktmbuf_mtod_offset(m, uint8_t*,
@@ -225,10 +229,12 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 	struct rte_crypto_sym_op *sym_cop;
 	int32_t i;
 	uint16_t pad_payload_len, pad_len, ip_hdr_len;
+	struct rte_ipsec_session *ips;
 
 	RTE_ASSERT(m != NULL);
 	RTE_ASSERT(sa != NULL);
 
+	ips = ipsec_get_session(sa);
 	ip_hdr_len = 0;
 
 	ip4 = rte_pktmbuf_mtod(m, struct ip *);
@@ -277,9 +283,10 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 	}
 
 	/* Add trailer padding if it is not constructed by HW */
-	if (sa->type != RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
-	    (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO &&
-	     !(sa->ol_flags & RTE_SECURITY_TX_HW_TRAILER_OFFLOAD))) {
+	if (ips->type != RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+		(ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO &&
+		 !(ips->security.ol_flags &
+			 RTE_SECURITY_TX_HW_TRAILER_OFFLOAD))) {
 		padding = (uint8_t *)rte_pktmbuf_append(m, pad_len +
 							sa->digest_len);
 		if (unlikely(padding == NULL)) {
@@ -344,8 +351,9 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 		}
 	}
 
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
-		if (sa->ol_flags & RTE_SECURITY_TX_HW_TRAILER_OFFLOAD) {
+	if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		if (ips->security.ol_flags &
+				RTE_SECURITY_TX_HW_TRAILER_OFFLOAD) {
 			/* Set the inner esp next protocol for HW trailer */
 			m->inner_esp_next_proto = nlp;
 			m->packet_type |= RTE_PTYPE_TUNNEL_ESP;
@@ -448,11 +456,14 @@ esp_outbound_post(struct rte_mbuf *m,
 		  struct ipsec_sa *sa,
 		  struct rte_crypto_op *cop)
 {
+	enum rte_security_session_action_type type;
 	RTE_ASSERT(m != NULL);
 	RTE_ASSERT(sa != NULL);
 
-	if ((sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
-			(sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
+	type = ipsec_get_action_type(sa);
+
+	if ((type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
+			(type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
 		m->ol_flags |= PKT_TX_SEC_OFFLOAD;
 	} else {
 		RTE_ASSERT(cop != NULL);
diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index dc85adfe5..8c60bd84b 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -40,7 +40,8 @@ set_ipsec_conf(struct ipsec_sa *sa, struct rte_security_ipsec_xform *ipsec)
 }
 
 int
-create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips)
 {
 	struct rte_cryptodev_info cdev_info;
 	unsigned long cdev_id_qp = 0;
@@ -71,9 +72,9 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			ipsec_ctx->tbl[cdev_id_qp].id,
 			ipsec_ctx->tbl[cdev_id_qp].qp);
 
-	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
+	if (ips->type != RTE_SECURITY_ACTION_TYPE_NONE) {
 		struct rte_security_session_conf sess_conf = {
-			.action_type = sa->type,
+			.action_type = ips->type,
 			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 			{.ipsec = {
 				.spi = sa->spi,
@@ -90,7 +91,7 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 
 		};
 
-		if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
+		if (ips->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
 			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
 							rte_cryptodev_get_sec_ctx(
 							ipsec_ctx->tbl[cdev_id_qp].id);
@@ -98,9 +99,9 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			/* Set IPsec parameters in conf */
 			set_ipsec_conf(sa, &(sess_conf.ipsec));
 
-			sa->sec_session = rte_security_session_create(ctx,
+			ips->security.ses = rte_security_session_create(ctx,
 					&sess_conf, ipsec_ctx->session_priv_pool);
-			if (sa->sec_session == NULL) {
+			if (ips->security.ses == NULL) {
 				RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 				return -1;
@@ -110,10 +111,10 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			return -1;
 		}
 	} else {
-		sa->crypto_session = rte_cryptodev_sym_session_create(
+		ips->crypto.ses = rte_cryptodev_sym_session_create(
 				ipsec_ctx->session_pool);
 		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
-				sa->crypto_session, sa->xforms,
+				ips->crypto.ses, sa->xforms,
 				ipsec_ctx->session_priv_pool);
 
 		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
@@ -126,12 +127,13 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 }
 
 int
-create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips)
 {
 	int32_t ret = 0;
 	struct rte_security_ctx *sec_ctx;
 	struct rte_security_session_conf sess_conf = {
-		.action_type = sa->type,
+		.action_type = ips->type,
 		.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 		{.ipsec = {
 			.spi = sa->spi,
@@ -151,7 +153,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on port %u\n",
 		sa->spi, sa->portid);
 
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+	if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
 		struct rte_flow_error err;
 		const struct rte_security_capability *sec_cap;
 		int ret = 0;
@@ -165,9 +167,9 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 			return -1;
 		}
 
-		sa->sec_session = rte_security_session_create(sec_ctx,
+		ips->security.ses = rte_security_session_create(sec_ctx,
 				&sess_conf, skt_ctx->session_pool);
-		if (sa->sec_session == NULL) {
+		if (ips->security.ses == NULL) {
 			RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 			return -1;
@@ -177,7 +179,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 
 		/* iterate until ESP tunnel*/
 		while (sec_cap->action != RTE_SECURITY_ACTION_TYPE_NONE) {
-			if (sec_cap->action == sa->type &&
+			if (sec_cap->action == ips->type &&
 			    sec_cap->protocol ==
 				RTE_SECURITY_PROTOCOL_IPSEC &&
 			    sec_cap->ipsec.mode ==
@@ -193,8 +195,8 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 			return -1;
 		}
 
-		sa->ol_flags = sec_cap->ol_flags;
-		sa->security_ctx = sec_ctx;
+		ips->security.ol_flags = sec_cap->ol_flags;
+		ips->security.ctx = sec_ctx;
 		sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
 
 		if (IS_IP6(sa->flags)) {
@@ -223,7 +225,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 		sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
 
 		sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
-		sa->action[0].conf = sa->sec_session;
+		sa->action[0].conf = ips->security.ses;
 
 		sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
 
@@ -282,8 +284,8 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 			if (ret)
 				goto flow_create_failure;
 		} else if (sa->attr.egress &&
-			   (sa->ol_flags &
-				    RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
+				(ips->security.ol_flags &
+					RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
 			sa->action[1].type =
 					RTE_FLOW_ACTION_TYPE_PASSTHRU;
 			sa->action[2].type =
@@ -299,7 +301,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 				err.message);
 			return -1;
 		}
-	} else if (sa->type ==	RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
+	} else if (ips->type ==	RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
 		const struct rte_security_capability *sec_cap;
 
 		sec_ctx = (struct rte_security_ctx *)
@@ -327,9 +329,9 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 
 		sess_conf.userdata = (void *) sa;
 
-		sa->sec_session = rte_security_session_create(sec_ctx,
+		ips->security.ses = rte_security_session_create(sec_ctx,
 					&sess_conf, skt_ctx->session_pool);
-		if (sa->sec_session == NULL) {
+		if (ips->security.ses == NULL) {
 			RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 			return -1;
@@ -345,7 +347,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 		/* iterate until ESP tunnel*/
 		while (sec_cap->action !=
 				RTE_SECURITY_ACTION_TYPE_NONE) {
-			if (sec_cap->action == sa->type &&
+			if (sec_cap->action == ips->type &&
 			    sec_cap->protocol ==
 				RTE_SECURITY_PROTOCOL_IPSEC &&
 			    sec_cap->ipsec.mode ==
@@ -361,8 +363,8 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 			return -1;
 		}
 
-		sa->ol_flags = sec_cap->ol_flags;
-		sa->security_ctx = sec_ctx;
+		ips->security.ol_flags = sec_cap->ol_flags;
+		ips->security.ctx = sec_ctx;
 	}
 	sa->cdev_id_qp = 0;
 
@@ -409,6 +411,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 	struct ipsec_mbuf_metadata *priv;
 	struct rte_crypto_sym_op *sym_cop;
 	struct ipsec_sa *sa;
+	struct rte_ipsec_session *ips;
 
 	for (i = 0; i < nb_pkts; i++) {
 		if (unlikely(sas[i] == NULL)) {
@@ -422,16 +425,17 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 		priv = get_priv(pkts[i]);
 		sa = sas[i];
 		priv->sa = sa;
+		ips = ipsec_get_session(sa);
 
-		switch (sa->type) {
+		switch (ips->type) {
 		case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL:
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
 
-			if ((unlikely(sa->sec_session == NULL)) &&
-				create_lookaside_session(ipsec_ctx, sa)) {
+			if ((unlikely(ips->security.ses == NULL)) &&
+				create_lookaside_session(ipsec_ctx, sa, ips)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -440,7 +444,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			sym_cop->m_src = pkts[i];
 
 			rte_security_attach_session(&priv->cop,
-					sa->sec_session);
+				ips->security.ses);
 			break;
 		case RTE_SECURITY_ACTION_TYPE_NONE:
 
@@ -449,14 +453,14 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			rte_prefetch0(&priv->sym_cop);
 
-			if ((unlikely(sa->crypto_session == NULL)) &&
-				create_lookaside_session(ipsec_ctx, sa)) {
+			if ((unlikely(ips->crypto.ses == NULL)) &&
+				create_lookaside_session(ipsec_ctx, sa, ips)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
 
 			rte_crypto_op_attach_sym_session(&priv->cop,
-					sa->crypto_session);
+					ips->crypto.ses);
 
 			ret = xform_func(pkts[i], sa, &priv->cop);
 			if (unlikely(ret)) {
@@ -465,21 +469,22 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 			break;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
-			RTE_ASSERT(sa->sec_session != NULL);
+			RTE_ASSERT(ips->security.ses != NULL);
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
-			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
+			if (ips->security.ol_flags &
+				RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
-						sa->security_ctx,
-						sa->sec_session, pkts[i], NULL);
+					ips->security.ctx, ips->security.ses,
+					pkts[i], NULL);
 			continue;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
-			RTE_ASSERT(sa->sec_session != NULL);
+			RTE_ASSERT(ips->security.ses != NULL);
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
 			rte_security_attach_session(&priv->cop,
-					sa->sec_session);
+					ips->security.ses);
 
 			ret = xform_func(pkts[i], sa, &priv->cop);
 			if (unlikely(ret)) {
@@ -488,10 +493,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
-			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
+			if (ips->security.ol_flags &
+				RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
-						sa->security_ctx,
-						sa->sec_session, pkts[i], NULL);
+					ips->security.ctx, ips->security.ses,
+					pkts[i], NULL);
 			continue;
 		}
 
@@ -560,7 +566,8 @@ ipsec_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			RTE_ASSERT(sa != NULL);
 
-			if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {
+			if (ipsec_get_action_type(sa) ==
+				RTE_SECURITY_ACTION_TYPE_NONE) {
 				ret = xform_func(pkt, sa, cops[j]);
 				if (unlikely(ret)) {
 					rte_pktmbuf_free(pkt);
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index 1efa6e488..a4ad81b0e 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -87,10 +87,6 @@ struct ipsec_sa {
 	uint32_t cdev_id_qp;
 	uint64_t seq;
 	uint32_t salt;
-	union {
-		struct rte_cryptodev_sym_session *crypto_session;
-		struct rte_security_session *sec_session;
-	};
 	enum rte_crypto_cipher_algorithm cipher_algo;
 	enum rte_crypto_auth_algorithm auth_algo;
 	enum rte_crypto_aead_algorithm aead_algo;
@@ -114,11 +110,8 @@ struct ipsec_sa {
 		struct rte_crypto_sym_xform *xforms;
 		struct rte_security_ipsec_xform *sec_xform;
 	};
-	enum rte_security_session_action_type type;
 	enum rte_security_ipsec_sa_direction direction;
 	uint16_t portid;
-	struct rte_security_ctx *security_ctx;
-	uint32_t ol_flags;
 
 #define MAX_RTE_FLOW_PATTERN (4)
 #define MAX_RTE_FLOW_ACTIONS (3)
@@ -284,6 +277,20 @@ get_sym_cop(struct rte_crypto_op *cop)
 	return (cop + 1);
 }
 
+static inline struct rte_ipsec_session *
+ipsec_get_session(struct ipsec_sa *sa)
+{
+	return &sa->ips;
+}
+
+static inline enum rte_security_session_action_type
+ipsec_get_action_type(struct ipsec_sa *sa)
+{
+	struct rte_ipsec_session *ips;
+	ips = ipsec_get_session(sa);
+	return ips->type;
+}
+
 int
 inbound_sa_check(struct sa_ctx *sa_ctx, struct rte_mbuf *m, uint32_t sa_idx);
 
@@ -338,9 +345,11 @@ void
 enqueue_cop_burst(struct cdev_qp *cqp);
 
 int
-create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips);
 
 int
-create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa);
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips);
 
 #endif /* __IPSEC_H__ */
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index 868f1a28d..239d81ef6 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -94,22 +94,16 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
 
 	/* setup crypto section */
 	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		if (sa->crypto_session == NULL) {
-			rc = create_lookaside_session(ctx, sa);
-			if (rc != 0)
-				return rc;
-		}
-		ss->crypto.ses = sa->crypto_session;
+		RTE_ASSERT(ss->crypto.ses == NULL);
+		rc = create_lookaside_session(ctx, sa, ss);
+		if (rc != 0)
+			return rc;
 	/* setup session action type */
-	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
-		if (sa->sec_session == NULL) {
-			rc = create_lookaside_session(ctx, sa);
-			if (rc != 0)
-				return rc;
-		}
-		ss->security.ses = sa->sec_session;
-		ss->security.ctx = sa->security_ctx;
-		ss->security.ol_flags = sa->ol_flags;
+	} else if (ss->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
+		RTE_ASSERT(ss->security.ses == NULL);
+		rc = create_lookaside_session(ctx, sa, ss);
+		if (rc != 0)
+			return rc;
 	} else
 		RTE_ASSERT(0);
 
@@ -218,7 +212,7 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 		pg = grp + i;
 		sa = pg->id.ptr;
 
-		ips = &sa->ips;
+		ips = ipsec_get_session(sa);
 
 		/* no valid HW session for that SA, try to create one */
 		if (sa == NULL || (ips->crypto.ses == NULL &&
@@ -226,8 +220,8 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 			k = 0;
 
 		/* process packets inline */
-		else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
-				sa->type ==
+		else if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+				ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
 
 			satp = rte_ipsec_sa_type(ips->sa);
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index c3cf3bd1f..a3d8e4545 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -224,6 +224,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	struct parse_status *status)
 {
 	struct ipsec_sa *rule = NULL;
+	struct rte_ipsec_session *ips;
 	uint32_t ti; /*token index*/
 	uint32_t *ri /*rule index*/;
 	uint32_t cipher_algo_p = 0;
@@ -262,6 +263,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	if (atoi(tokens[1]) == INVALID_SPI)
 		return;
 	rule->spi = atoi(tokens[1]);
+	ips = ipsec_get_session(rule);
 
 	for (ti = 2; ti < n_tokens; ti++) {
 		if (strcmp(tokens[ti], "mode") == 0) {
@@ -558,18 +560,18 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 				return;
 
 			if (strcmp(tokens[ti], "inline-crypto-offload") == 0)
-				rule->type =
+				ips->type =
 					RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO;
 			else if (strcmp(tokens[ti],
 					"inline-protocol-offload") == 0)
-				rule->type =
+				ips->type =
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL;
 			else if (strcmp(tokens[ti],
 					"lookaside-protocol-offload") == 0)
-				rule->type =
+				ips->type =
 				RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL;
 			else if (strcmp(tokens[ti], "no-offload") == 0)
-				rule->type = RTE_SECURITY_ACTION_TYPE_NONE;
+				ips->type = RTE_SECURITY_ACTION_TYPE_NONE;
 			else {
 				APP_CHECK(0, status, "Invalid input \"%s\"",
 						tokens[ti]);
@@ -624,11 +626,11 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	if (status->status < 0)
 		return;
 
-	if ((rule->type != RTE_SECURITY_ACTION_TYPE_NONE) && (portid_p == 0))
+	if ((ips->type != RTE_SECURITY_ACTION_TYPE_NONE) && (portid_p == 0))
 		printf("Missing portid option, falling back to non-offload\n");
 
 	if (!type_p || !portid_p) {
-		rule->type = RTE_SECURITY_ACTION_TYPE_NONE;
+		ips->type = RTE_SECURITY_ACTION_TYPE_NONE;
 		rule->portid = -1;
 	}
 
@@ -640,6 +642,7 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 {
 	uint32_t i;
 	uint8_t a, b, c, d;
+	const struct rte_ipsec_session *ips;
 
 	printf("\tspi_%s(%3u):", inbound?"in":"out", sa->spi);
 
@@ -695,8 +698,10 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 		printf("Transport ");
 		break;
 	}
+
+	ips = &sa->ips;
 	printf(" type:");
-	switch (sa->type) {
+	switch (ips->type) {
 	case RTE_SECURITY_ACTION_TYPE_NONE:
 		printf("no-offload ");
 		break;
@@ -876,6 +881,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 	uint16_t iv_length, aad_length;
 	int inline_status;
 	int32_t rc;
+	struct rte_ipsec_session *ips;
 
 	/* for ESN upper 32 bits of SQN also need to be part of AAD */
 	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
@@ -890,9 +896,10 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 		}
 		*sa = entries[i];
 		sa->seq = 0;
+		ips = ipsec_get_session(sa);
 
-		if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
-			sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
+			ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
 			if (check_eth_dev_caps(sa->portid, inbound))
 				return -EINVAL;
 		}
@@ -907,7 +914,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 			sa->dst.ip.ip4 = rte_cpu_to_be_32(sa->dst.ip.ip4);
 			break;
 		case TRANSPORT:
-			if (sa->type ==
+			if (ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
 				inline_status =
 					sa_add_address_inline_crypto(sa);
@@ -918,6 +925,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 		}
 
 		if (sa->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) {
+			struct rte_ipsec_session *ips;
 			iv_length = 16;
 
 			sa_ctx->xf[idx].a.type = RTE_CRYPTO_SYM_XFORM_AEAD;
@@ -938,11 +946,12 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			sa->xforms = &sa_ctx->xf[idx].a;
 
-			if (sa->type ==
+			ips = ipsec_get_session(sa);
+			if (ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
-				sa->type ==
+				ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
-				rc = create_inline_session(skt_ctx, sa);
+				rc = create_inline_session(skt_ctx, sa, ips);
 				if (rc != 0) {
 					RTE_LOG(ERR, IPSEC_ESP,
 						"create_inline_session() failed\n");
@@ -1100,23 +1109,11 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm, const struct ipsec_sa *ss,
 }
 
 static int
-fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
-	const struct ipsec_sa *lsa)
+fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa)
 {
 	int32_t rc = 0;
 
 	ss->sa = sa;
-	ss->type = lsa->type;
-
-	/* setup crypto section */
-	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		ss->crypto.ses = lsa->crypto_session;
-	/* setup session action type */
-	} else {
-		ss->security.ses = lsa->sec_session;
-		ss->security.ctx = lsa->security_ctx;
-		ss->security.ol_flags = lsa->ol_flags;
-	}
 
 	if (ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
 		ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
@@ -1138,6 +1135,7 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 {
 	int rc;
 	struct rte_ipsec_sa_prm prm;
+	struct rte_ipsec_session *ips;
 	struct rte_ipv4_hdr v4  = {
 		.version_ihl = IPVERSION << 4 |
 			sizeof(v4) / RTE_IPV4_IHL_MULTIPLIER,
@@ -1162,7 +1160,10 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
+	/* init processing session */
+	ips = ipsec_get_session(lsa);
+	rc = fill_ipsec_session(ips, sa);
+
 	return rc;
 }
 
@@ -1386,6 +1387,7 @@ sa_check_offloads(uint16_t port_id, uint64_t *rx_offloads,
 {
 	struct ipsec_sa *rule;
 	uint32_t idx_sa;
+	enum rte_security_session_action_type rule_type;
 
 	*rx_offloads = 0;
 	*tx_offloads = 0;
@@ -1393,8 +1395,9 @@ sa_check_offloads(uint16_t port_id, uint64_t *rx_offloads,
 	/* Check for inbound rules that use offloads and use this port */
 	for (idx_sa = 0; idx_sa < nb_sa_in; idx_sa++) {
 		rule = &sa_in[idx_sa];
-		if ((rule->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
-				rule->type ==
+		rule_type = ipsec_get_action_type(rule);
+		if ((rule_type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+				rule_type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
 				&& rule->portid == port_id)
 			*rx_offloads |= DEV_RX_OFFLOAD_SECURITY;
@@ -1403,8 +1406,9 @@ sa_check_offloads(uint16_t port_id, uint64_t *rx_offloads,
 	/* Check for outbound rules that use offloads and use this port */
 	for (idx_sa = 0; idx_sa < nb_sa_out; idx_sa++) {
 		rule = &sa_out[idx_sa];
-		if ((rule->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
-				rule->type ==
+		rule_type = ipsec_get_action_type(rule);
+		if ((rule_type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+				rule_type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
 				&& rule->portid == port_id)
 			*tx_offloads |= DEV_TX_OFFLOAD_SECURITY;
-- 
2.17.1

[dpdk-dev] [PATCH v6 2/4] examples/ipsec-secgw: add fallback session feature

[Marcin Smoczynski]

Inline processing is limited to a specified subset of traffic. It is
often unable to handle more complicated situations, such as fragmented
traffic. When using inline processing such traffic is dropped.

Introduce fallback session for inline processing allowing processing
packets that normally would be dropped. A fallback session is
configured by adding 'fallback' keyword with 'lookaside-none' or
'lookaside-protocol' parameter to an SA configuration.

Using IPsec anti-replay window or ESN feature with fallback session is
not yet supported when primary session is of type
'inline-protocol-offload' or fallback session is 'lookaside-protocol'
because SA sequence number is not synchronized between software and
hardware sessions. Fallback sessions are also limited to ingress IPsec
traffic.

Fallback session feature is not available in the legacy mode.

Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
Tested-by: Bernard Iremonger <bernard.iremonger@intel.com>
Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
---
 doc/guides/rel_notes/release_19_11.rst   |   8 ++
 doc/guides/sample_app_ug/ipsec_secgw.rst |  24 ++++-
 examples/ipsec-secgw/esp.c               |   4 +-
 examples/ipsec-secgw/ipsec-secgw.c       |  16 ++--
 examples/ipsec-secgw/ipsec.c             |  10 +-
 examples/ipsec-secgw/ipsec.h             |  40 ++++++--
 examples/ipsec-secgw/ipsec_process.c     |  85 +++++++++++------
 examples/ipsec-secgw/sa.c                | 116 +++++++++++++++++++----
 8 files changed, 239 insertions(+), 64 deletions(-)

diff --git a/doc/guides/rel_notes/release_19_11.rst b/doc/guides/rel_notes/release_19_11.rst
index 27cfbd9e3..f2ed040e9 100644
--- a/doc/guides/rel_notes/release_19_11.rst
+++ b/doc/guides/rel_notes/release_19_11.rst
@@ -56,6 +56,14 @@ New Features
      Also, make sure to start the actual text at the margin.
      =========================================================
 
+* **Updated the IPSec library and IPsec Security Gateway application.**
+
+  Added the following features to ``librte_ipsec`` and the ``ipsec-secgw``
+  sample application:
+
+  * Support fragmented packets in inline crypto processing mode with fallback
+    lookaside session.
+
 
 Removed Items
 -------------
diff --git a/doc/guides/sample_app_ug/ipsec_secgw.rst b/doc/guides/sample_app_ug/ipsec_secgw.rst
index ad2d79e75..45478e2a5 100644
--- a/doc/guides/sample_app_ug/ipsec_secgw.rst
+++ b/doc/guides/sample_app_ug/ipsec_secgw.rst
@@ -401,7 +401,7 @@ The SA rule syntax is shown as follows:
 .. code-block:: console
 
     sa <dir> <spi> <cipher_algo> <cipher_key> <auth_algo> <auth_key>
-    <mode> <src_ip> <dst_ip> <action_type> <port_id>
+    <mode> <src_ip> <dst_ip> <action_type> <port_id> <fallback>
 
 where each options means:
 
@@ -573,6 +573,28 @@ where each options means:
 
    * *port_id X* X is a valid device number in decimal
 
+ ``<fallback>``
+
+ * Action type for ingress IPsec packets that inline processor failed to
+   process. IPsec anti-replay window and ESN is supported with fallback
+   processing session only when primary session is *inline-crypto-offload*
+   and fallback session is *lookaside-none*.
+
+   If used in conjunction with IPsec window, its width needs be increased
+   due to different processing times of inline and lookaside modes which
+   results in packet reordering.
+
+ * Optional: Yes.
+
+ * Available options:
+
+   * *lookaside-none*: use automatically chosen cryptodev to process packets
+   * *lookaside-protocol*: lookaside protocol hardware offload
+
+ * Syntax:
+
+   * *fallback lookaside-none*
+   * *fallback lookaside-protocol*
 
 Example SA rules:
 
diff --git a/examples/ipsec-secgw/esp.c b/examples/ipsec-secgw/esp.c
index c1b49da1e..bfa7ff721 100644
--- a/examples/ipsec-secgw/esp.c
+++ b/examples/ipsec-secgw/esp.c
@@ -155,7 +155,7 @@ esp_inbound_post(struct rte_mbuf *m, struct ipsec_sa *sa,
 	RTE_ASSERT(sa != NULL);
 	RTE_ASSERT(cop != NULL);
 
-	ips = ipsec_get_session(sa);
+	ips = ipsec_get_primary_session(sa);
 
 	if ((ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
 			(ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
@@ -234,7 +234,7 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 	RTE_ASSERT(m != NULL);
 	RTE_ASSERT(sa != NULL);
 
-	ips = ipsec_get_session(sa);
+	ips = ipsec_get_primary_session(sa);
 	ip_hdr_len = 0;
 
 	ip4 = rte_pktmbuf_mtod(m, struct ip *);
diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index 0d1fd6af6..641ed3767 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -189,6 +189,7 @@ static uint32_t mtu_size = RTE_ETHER_MTU;
 
 /* application wide librte_ipsec/SA parameters */
 struct app_sa_prm app_sa_prm = {.enable = 0};
+static const char *cfgfile;
 
 struct lcore_rx_queue {
 	uint16_t port_id;
@@ -1462,12 +1463,7 @@ parse_args(int32_t argc, char **argv)
 				print_usage(prgname);
 				return -1;
 			}
-			if (parse_cfg_file(optarg) < 0) {
-				printf("parsing file \"%s\" failed\n",
-					optarg);
-				print_usage(prgname);
-				return -1;
-			}
+			cfgfile = optarg;
 			f_present = 1;
 			break;
 		case 'j':
@@ -2398,6 +2394,14 @@ main(int32_t argc, char **argv)
 	if (ret < 0)
 		rte_exit(EXIT_FAILURE, "Invalid parameters\n");
 
+	/* parse configuration file */
+	if (parse_cfg_file(cfgfile) < 0) {
+		printf("parsing file \"%s\" failed\n",
+			optarg);
+		print_usage(argv[0]);
+		return -1;
+	}
+
 	if ((unprotected_port_mask & enabled_port_mask) !=
 			unprotected_port_mask)
 		rte_exit(EXIT_FAILURE, "Invalid unprotected portmask 0x%x\n",
diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index 8c60bd84b..8b0441028 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -404,7 +404,7 @@ enqueue_cop(struct cdev_qp *cqp, struct rte_crypto_op *cop)
 
 static inline void
 ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
-		struct rte_mbuf *pkts[], struct ipsec_sa *sas[],
+		struct rte_mbuf *pkts[], void *sas[],
 		uint16_t nb_pkts)
 {
 	int32_t ret = 0, i;
@@ -423,9 +423,9 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 		rte_prefetch0(pkts[i]);
 
 		priv = get_priv(pkts[i]);
-		sa = sas[i];
+		sa = ipsec_mask_saptr(sas[i]);
 		priv->sa = sa;
-		ips = ipsec_get_session(sa);
+		ips = ipsec_get_primary_session(sa);
 
 		switch (ips->type) {
 		case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL:
@@ -586,7 +586,7 @@ uint16_t
 ipsec_inbound(struct ipsec_ctx *ctx, struct rte_mbuf *pkts[],
 		uint16_t nb_pkts, uint16_t len)
 {
-	struct ipsec_sa *sas[nb_pkts];
+	void *sas[nb_pkts];
 
 	inbound_sa_lookup(ctx->sa_ctx, pkts, sas, nb_pkts);
 
@@ -606,7 +606,7 @@ uint16_t
 ipsec_outbound(struct ipsec_ctx *ctx, struct rte_mbuf *pkts[],
 		uint32_t sa_idx[], uint16_t nb_pkts, uint16_t len)
 {
-	struct ipsec_sa *sas[nb_pkts];
+	void *sas[nb_pkts];
 
 	outbound_sa_lookup(ctx->sa_ctx, sa_idx, sas, nb_pkts);
 
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index a4ad81b0e..8e075216c 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -81,12 +81,32 @@ struct app_sa_prm {
 
 extern struct app_sa_prm app_sa_prm;
 
+enum {
+	IPSEC_SESSION_PRIMARY = 0,
+	IPSEC_SESSION_FALLBACK = 1,
+	IPSEC_SESSION_MAX
+};
+
+#define IPSEC_SA_OFFLOAD_FALLBACK_FLAG (1)
+
+static inline struct ipsec_sa *
+ipsec_mask_saptr(void *ptr)
+{
+	uintptr_t i = (uintptr_t)ptr;
+	static const uintptr_t mask = IPSEC_SA_OFFLOAD_FALLBACK_FLAG;
+
+	i &= ~mask;
+
+	return (struct ipsec_sa *)i;
+}
+
 struct ipsec_sa {
-	struct rte_ipsec_session ips; /* one session per sa for now */
+	struct rte_ipsec_session sessions[IPSEC_SESSION_MAX];
 	uint32_t spi;
 	uint32_t cdev_id_qp;
 	uint64_t seq;
 	uint32_t salt;
+	uint32_t fallback_sessions;
 	enum rte_crypto_cipher_algorithm cipher_algo;
 	enum rte_crypto_auth_algorithm auth_algo;
 	enum rte_crypto_aead_algorithm aead_algo;
@@ -210,7 +230,7 @@ struct cnt_blk {
 struct traffic_type {
 	const uint8_t *data[MAX_PKT_BURST * 2];
 	struct rte_mbuf *pkts[MAX_PKT_BURST * 2];
-	struct ipsec_sa *saptr[MAX_PKT_BURST * 2];
+	void *saptr[MAX_PKT_BURST * 2];
 	uint32_t res[MAX_PKT_BURST * 2];
 	uint32_t num;
 };
@@ -278,16 +298,22 @@ get_sym_cop(struct rte_crypto_op *cop)
 }
 
 static inline struct rte_ipsec_session *
-ipsec_get_session(struct ipsec_sa *sa)
+ipsec_get_primary_session(struct ipsec_sa *sa)
+{
+	return &sa->sessions[IPSEC_SESSION_PRIMARY];
+}
+
+static inline struct rte_ipsec_session *
+ipsec_get_fallback_session(struct ipsec_sa *sa)
 {
-	return &sa->ips;
+	return &sa->sessions[IPSEC_SESSION_FALLBACK];
 }
 
 static inline enum rte_security_session_action_type
 ipsec_get_action_type(struct ipsec_sa *sa)
 {
 	struct rte_ipsec_session *ips;
-	ips = ipsec_get_session(sa);
+	ips = ipsec_get_primary_session(sa);
 	return ips->type;
 }
 
@@ -296,11 +322,11 @@ inbound_sa_check(struct sa_ctx *sa_ctx, struct rte_mbuf *m, uint32_t sa_idx);
 
 void
 inbound_sa_lookup(struct sa_ctx *sa_ctx, struct rte_mbuf *pkts[],
-		struct ipsec_sa *sa[], uint16_t nb_pkts);
+		void *sa[], uint16_t nb_pkts);
 
 void
 outbound_sa_lookup(struct sa_ctx *sa_ctx, uint32_t sa_idx[],
-		struct ipsec_sa *sa[], uint16_t nb_pkts);
+		void *sa[], uint16_t nb_pkts);
 
 void
 sp4_init(struct socket_ctx *ctx, int32_t socket_id);
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index 239d81ef6..2eb5c8b34 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -18,7 +18,6 @@
 	(((t) & RTE_IPSEC_SATP_IPV_MASK) == RTE_IPSEC_SATP_IPV4)) || \
 	((t) & RTE_IPSEC_SATP_MODE_MASK) == RTE_IPSEC_SATP_MODE_TUNLV4)
 
-
 /* helper routine to free bulk of packets */
 static inline void
 free_pkts(struct rte_mbuf *mb[], uint32_t n)
@@ -118,7 +117,7 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
  * group input packets byt the SA they belong to.
  */
 static uint32_t
-sa_group(struct ipsec_sa *sa_ptr[], struct rte_mbuf *pkts[],
+sa_group(void *sa_ptr[], struct rte_mbuf *pkts[],
 	struct rte_ipsec_group grp[], uint32_t num)
 {
 	uint32_t i, n, spi;
@@ -185,6 +184,37 @@ copy_to_trf(struct ipsec_traffic *trf, uint64_t satp, struct rte_mbuf *mb[],
 	out->num += num;
 }
 
+static uint32_t
+ipsec_prepare_crypto_group(struct ipsec_ctx *ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips, struct rte_mbuf **m,
+		unsigned int cnt)
+{
+	struct cdev_qp *cqp;
+	struct rte_crypto_op *cop[cnt];
+	uint32_t j, k;
+	struct ipsec_mbuf_metadata *priv;
+
+	cqp = &ctx->tbl[sa->cdev_id_qp];
+
+	/* for that app each mbuf has it's own crypto op */
+	for (j = 0; j != cnt; j++) {
+		priv = get_priv(m[j]);
+		cop[j] = &priv->cop;
+		/*
+		 * this is just to satisfy inbound_sa_check()
+		 * should be removed in future.
+		 */
+		priv->sa = sa;
+	}
+
+	/* prepare and enqueue crypto ops */
+	k = rte_ipsec_pkt_crypto_prepare(ips, m, cop, cnt);
+	if (k != 0)
+		enqueue_cop_bulk(cqp, cop, k);
+
+	return k;
+}
+
 /*
  * Process ipsec packets.
  * If packet belong to SA that is subject of inline-crypto,
@@ -201,18 +231,15 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 	struct ipsec_mbuf_metadata *priv;
 	struct rte_ipsec_group *pg;
 	struct rte_ipsec_session *ips;
-	struct cdev_qp *cqp;
-	struct rte_crypto_op *cop[RTE_DIM(trf->ipsec.pkts)];
 	struct rte_ipsec_group grp[RTE_DIM(trf->ipsec.pkts)];
 
 	n = sa_group(trf->ipsec.saptr, trf->ipsec.pkts, grp, trf->ipsec.num);
 
 	for (i = 0; i != n; i++) {
-
 		pg = grp + i;
-		sa = pg->id.ptr;
+		sa = ipsec_mask_saptr(pg->id.ptr);
 
-		ips = ipsec_get_session(sa);
+		ips = ipsec_get_primary_session(sa);
 
 		/* no valid HW session for that SA, try to create one */
 		if (sa == NULL || (ips->crypto.ses == NULL &&
@@ -224,6 +251,7 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 				ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
 
+			/* get SA type */
 			satp = rte_ipsec_sa_type(ips->sa);
 
 			/*
@@ -236,30 +264,33 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 				priv->sa = sa;
 			}
 
-			k = rte_ipsec_pkt_process(ips, pg->m, pg->cnt);
-			copy_to_trf(trf, satp, pg->m, k);
+			/* fallback to cryptodev with RX packets which inline
+			 * processor was unable to process
+			 */
+			if (pg->id.val & IPSEC_SA_OFFLOAD_FALLBACK_FLAG) {
+				/* offload packets to cryptodev */
+				struct rte_ipsec_session *fallback;
+
+				fallback = ipsec_get_fallback_session(sa);
+				if (fallback->crypto.ses == NULL &&
+					fill_ipsec_session(fallback, ctx, sa)
+					!= 0)
+					k = 0;
+				else
+					k = ipsec_prepare_crypto_group(ctx, sa,
+						fallback, pg->m, pg->cnt);
+			} else {
+				/* finish processing of packets successfully
+				 * decrypted by an inline processor
+				 */
+				k = rte_ipsec_pkt_process(ips, pg->m, pg->cnt);
+				copy_to_trf(trf, satp, pg->m, k);
 
+			}
 		/* enqueue packets to crypto dev */
 		} else {
-
-			cqp = &ctx->tbl[sa->cdev_id_qp];
-
-			/* for that app each mbuf has it's own crypto op */
-			for (j = 0; j != pg->cnt; j++) {
-				priv = get_priv(pg->m[j]);
-				cop[j] = &priv->cop;
-				/*
-				 * this is just to satisfy inbound_sa_check()
-				 * should be removed in future.
-				 */
-				priv->sa = sa;
-			}
-
-			/* prepare and enqueue crypto ops */
-			k = rte_ipsec_pkt_crypto_prepare(ips, pg->m, cop,
+			k = ipsec_prepare_crypto_group(ctx, sa, ips, pg->m,
 				pg->cnt);
-			if (k != 0)
-				enqueue_cop_bulk(cqp, cop, k);
 		}
 
 		/* drop packets that cannot be enqueued/processed */
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index a3d8e4545..92681f822 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -31,6 +31,8 @@
 
 #define IP6_FULL_MASK (sizeof(((struct ip_addr *)NULL)->ip.ip6.ip6) * CHAR_BIT)
 
+#define MBUF_NO_SEC_OFFLOAD(m) ((m->ol_flags & PKT_RX_SEC_OFFLOAD) == 0)
+
 struct supported_cipher_algo {
 	const char *keyword;
 	enum rte_crypto_cipher_algorithm algo;
@@ -235,6 +237,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	uint32_t mode_p = 0;
 	uint32_t type_p = 0;
 	uint32_t portid_p = 0;
+	uint32_t fallback_p = 0;
 
 	if (strcmp(tokens[0], "in") == 0) {
 		ri = &nb_sa_in;
@@ -245,6 +248,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 			return;
 
 		rule = &sa_in[*ri];
+		rule->direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS;
 	} else {
 		ri = &nb_sa_out;
 
@@ -254,6 +258,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 			return;
 
 		rule = &sa_out[*ri];
+		rule->direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS;
 	}
 
 	/* spi number */
@@ -263,7 +268,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	if (atoi(tokens[1]) == INVALID_SPI)
 		return;
 	rule->spi = atoi(tokens[1]);
-	ips = ipsec_get_session(rule);
+	ips = ipsec_get_primary_session(rule);
 
 	for (ti = 2; ti < n_tokens; ti++) {
 		if (strcmp(tokens[ti], "mode") == 0) {
@@ -596,6 +601,49 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 			continue;
 		}
 
+		if (strcmp(tokens[ti], "fallback") == 0) {
+			struct rte_ipsec_session *fb;
+
+			APP_CHECK(app_sa_prm.enable, status, "Fallback session "
+				"not allowed for legacy mode.");
+			if (status->status < 0)
+				return;
+			APP_CHECK(ips->type ==
+				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO, status,
+				"Fallback session allowed if primary session "
+				"is of type inline-crypto-offload only.");
+			if (status->status < 0)
+				return;
+			APP_CHECK(rule->direction ==
+				RTE_SECURITY_IPSEC_SA_DIR_INGRESS, status,
+				"Fallback session not allowed for egress "
+				"rule");
+			if (status->status < 0)
+				return;
+			APP_CHECK_PRESENCE(fallback_p, tokens[ti], status);
+			if (status->status < 0)
+				return;
+			INCREMENT_TOKEN_INDEX(ti, n_tokens, status);
+			if (status->status < 0)
+				return;
+			fb = ipsec_get_fallback_session(rule);
+			if (strcmp(tokens[ti], "lookaside-none") == 0) {
+				fb->type = RTE_SECURITY_ACTION_TYPE_NONE;
+			} else if (strcmp(tokens[ti], "lookaside-protocol")
+				== 0) {
+				fb->type =
+				RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL;
+			} else {
+				APP_CHECK(0, status, "unrecognized fallback "
+					"type %s.", tokens[ti]);
+				return;
+			}
+
+			rule->fallback_sessions = 1;
+			fallback_p = 1;
+			continue;
+		}
+
 		/* unrecognizeable input */
 		APP_CHECK(0, status, "unrecognized input \"%s\"",
 			tokens[ti]);
@@ -643,6 +691,7 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 	uint32_t i;
 	uint8_t a, b, c, d;
 	const struct rte_ipsec_session *ips;
+	const struct rte_ipsec_session *fallback_ips;
 
 	printf("\tspi_%s(%3u):", inbound?"in":"out", sa->spi);
 
@@ -699,7 +748,7 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 		break;
 	}
 
-	ips = &sa->ips;
+	ips = &sa->sessions[IPSEC_SESSION_PRIMARY];
 	printf(" type:");
 	switch (ips->type) {
 	case RTE_SECURITY_ACTION_TYPE_NONE:
@@ -715,6 +764,22 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 		printf("lookaside-protocol-offload ");
 		break;
 	}
+
+	fallback_ips = &sa->sessions[IPSEC_SESSION_FALLBACK];
+	if (fallback_ips != NULL && sa->fallback_sessions > 0) {
+		printf("inline fallback:");
+		switch (fallback_ips->type) {
+		case RTE_SECURITY_ACTION_TYPE_NONE:
+			printf("lookaside-none ");
+			break;
+		case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL:
+			printf("lookaside-protocol ");
+			break;
+		default:
+			printf("invalid ");
+			break;
+		}
+	}
 	printf("\n");
 }
 
@@ -896,7 +961,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 		}
 		*sa = entries[i];
 		sa->seq = 0;
-		ips = ipsec_get_session(sa);
+		ips = ipsec_get_primary_session(sa);
 
 		if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
 			ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
@@ -904,9 +969,6 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 				return -EINVAL;
 		}
 
-		sa->direction = (inbound == 1) ?
-				RTE_SECURITY_IPSEC_SA_DIR_INGRESS :
-				RTE_SECURITY_IPSEC_SA_DIR_EGRESS;
 
 		switch (WITHOUT_TRANSPORT_VERSION(sa->flags)) {
 		case IP4_TUNNEL:
@@ -946,7 +1008,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			sa->xforms = &sa_ctx->xf[idx].a;
 
-			ips = ipsec_get_session(sa);
+			ips = ipsec_get_primary_session(sa);
 			if (ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
 				ips->type ==
@@ -1160,9 +1222,15 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	/* init processing session */
-	ips = ipsec_get_session(lsa);
+	/* init primary processing session */
+	ips = ipsec_get_primary_session(lsa);
 	rc = fill_ipsec_session(ips, sa);
+	if (rc != 0)
+		return rc;
+
+	/* init inline fallback processing session */
+	if (lsa->fallback_sessions == 1)
+		rc = fill_ipsec_session(ipsec_get_fallback_session(lsa), sa);
 
 	return rc;
 }
@@ -1318,13 +1386,14 @@ inbound_sa_check(struct sa_ctx *sa_ctx, struct rte_mbuf *m, uint32_t sa_idx)
 
 static inline void
 single_inbound_lookup(struct ipsec_sa *sadb, struct rte_mbuf *pkt,
-		struct ipsec_sa **sa_ret)
+		void **sa_ret)
 {
 	struct rte_esp_hdr *esp;
 	struct ip *ip;
 	uint32_t *src4_addr;
 	uint8_t *src6_addr;
 	struct ipsec_sa *sa;
+	void *result_sa;
 
 	*sa_ret = NULL;
 
@@ -1334,33 +1403,48 @@ single_inbound_lookup(struct ipsec_sa *sadb, struct rte_mbuf *pkt,
 	if (esp->spi == INVALID_SPI)
 		return;
 
-	sa = &sadb[SPI2IDX(rte_be_to_cpu_32(esp->spi))];
+	result_sa = sa = &sadb[SPI2IDX(rte_be_to_cpu_32(esp->spi))];
 	if (rte_be_to_cpu_32(esp->spi) != sa->spi)
 		return;
 
+	/*
+	 * Mark need for inline offload fallback on the LSB of SA pointer.
+	 * Thanks to packet grouping mechanism which ipsec_process is using
+	 * packets marked for fallback processing will form separate group.
+	 *
+	 * Because it is not safe to use SA pointer it is casted to generic
+	 * pointer to prevent from unintentional use. Use ipsec_mask_saptr
+	 * to get valid struct pointer.
+	 */
+	if (MBUF_NO_SEC_OFFLOAD(pkt) && sa->fallback_sessions > 0) {
+		uintptr_t intsa = (uintptr_t)sa;
+		intsa |= IPSEC_SA_OFFLOAD_FALLBACK_FLAG;
+		result_sa = (void *)intsa;
+	}
+
 	switch (WITHOUT_TRANSPORT_VERSION(sa->flags)) {
 	case IP4_TUNNEL:
 		src4_addr = RTE_PTR_ADD(ip, offsetof(struct ip, ip_src));
 		if ((ip->ip_v == IPVERSION) &&
 				(sa->src.ip.ip4 == *src4_addr) &&
 				(sa->dst.ip.ip4 == *(src4_addr + 1)))
-			*sa_ret = sa;
+			*sa_ret = result_sa;
 		break;
 	case IP6_TUNNEL:
 		src6_addr = RTE_PTR_ADD(ip, offsetof(struct ip6_hdr, ip6_src));
 		if ((ip->ip_v == IP6_VERSION) &&
 				!memcmp(&sa->src.ip.ip6.ip6, src6_addr, 16) &&
 				!memcmp(&sa->dst.ip.ip6.ip6, src6_addr + 16, 16))
-			*sa_ret = sa;
+			*sa_ret = result_sa;
 		break;
 	case TRANSPORT:
-		*sa_ret = sa;
+		*sa_ret = result_sa;
 	}
 }
 
 void
 inbound_sa_lookup(struct sa_ctx *sa_ctx, struct rte_mbuf *pkts[],
-		struct ipsec_sa *sa[], uint16_t nb_pkts)
+		void *sa[], uint16_t nb_pkts)
 {
 	uint32_t i;
 
@@ -1370,7 +1454,7 @@ inbound_sa_lookup(struct sa_ctx *sa_ctx, struct rte_mbuf *pkts[],
 
 void
 outbound_sa_lookup(struct sa_ctx *sa_ctx, uint32_t sa_idx[],
-		struct ipsec_sa *sa[], uint16_t nb_pkts)
+		void *sa[], uint16_t nb_pkts)
 {
 	uint32_t i;
 
-- 
2.17.1

Re: [dpdk-dev] [PATCH v6 2/4] examples/ipsec-secgw: add fallback session feature

[Akhil Goyal]

Hi All,
> 
> Inline processing is limited to a specified subset of traffic. It is
> often unable to handle more complicated situations, such as fragmented
> traffic. When using inline processing such traffic is dropped.
> 
> Introduce fallback session for inline processing allowing processing
> packets that normally would be dropped. A fallback session is
> configured by adding 'fallback' keyword with 'lookaside-none' or
> 'lookaside-protocol' parameter to an SA configuration.
> 
> Using IPsec anti-replay window or ESN feature with fallback session is
> not yet supported when primary session is of type
> 'inline-protocol-offload' or fallback session is 'lookaside-protocol'
> because SA sequence number is not synchronized between software and
> hardware sessions. Fallback sessions are also limited to ingress IPsec
> traffic.
> 
> Fallback session feature is not available in the legacy mode.
> 
I started looking this patch, but some initial thoughts looking at the patch description.

When you say a fallback session will be a lookaside none or lookaside protocol,
the packet will be processed asynchronously and might as well reorder.

The best possible solution for this would be the synchronous API which are in talks
in another patchset or use a SW PMD(eg. Openssl etc.) session and wait till you get the packet dequeued.
So effectively async APIs will be used to behave synchronously.
You can not use hardware PMD session as it will perform very badly for fallback packets
Because you have to wait till the packet is not getting dequeued back.

Having said that, you won't find a device or a scenario where you can use
Inline crypto as primary and lookaside proto as fallback.
It can only be like inline crypto as primary and lookaside none as fallback.

BTW, I am ok with Patch 1/4 and 3/4. If no objections from the community, I can pick those.

-Akhil

> Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
> Tested-by: Bernard Iremonger <bernard.iremonger@intel.com>
> Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
> ---

Re: [dpdk-dev] [PATCH v6 2/4] examples/ipsec-secgw: add fallback session feature

[Ananyev, Konstantin]

Hi Akhil,

> >
> > Inline processing is limited to a specified subset of traffic. It is
> > often unable to handle more complicated situations, such as fragmented
> > traffic. When using inline processing such traffic is dropped.
> >
> > Introduce fallback session for inline processing allowing processing
> > packets that normally would be dropped. A fallback session is
> > configured by adding 'fallback' keyword with 'lookaside-none' or
> > 'lookaside-protocol' parameter to an SA configuration.
> >
> > Using IPsec anti-replay window or ESN feature with fallback session is
> > not yet supported when primary session is of type
> > 'inline-protocol-offload' or fallback session is 'lookaside-protocol'
> > because SA sequence number is not synchronized between software and
> > hardware sessions. Fallback sessions are also limited to ingress IPsec
> > traffic.
> >
> > Fallback session feature is not available in the legacy mode.
> >
> I started looking this patch, but some initial thoughts looking at the patch description.
> 
> When you say a fallback session will be a lookaside none or lookaside protocol,
> the packet will be processed asynchronously and might as well reorder.

Yes, we documented it as one of limitations.
Though as I already mentioned for some use-cases some reordering it is acceptable.

 > The best possible solution for this would be the synchronous API which are in talks

Agree, that would be a way to avoid reordering, but it is not there yet.

> in another patchset or use a SW PMD(eg. Openssl etc.) session and wait till you get the packet dequeued.
> So effectively async APIs will be used to behave synchronously.
> You can not use hardware PMD session as it will perform very badly for fallback packets
> Because you have to wait till the packet is not getting dequeued back.

We don't plan to support that model because of great performance penalty you mentioned.

> 
> Having said that, you won't find a device or a scenario where you can use
> Inline crypto as primary and lookaside proto as fallback.
> It can only be like inline crypto as primary and lookaside none as fallback.

Yes, correct.
I thought that we already removed lookaside-proto from supported types.
If we didn't - will certainly do that. 

> 
> BTW, I am ok with Patch 1/4 and 3/4. If no objections from the community, I can pick those.

Great to hear.
What obstacles do you see with others two?
Konstantin

> 
> -Akhil
> 
> > Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
> > Tested-by: Bernard Iremonger <bernard.iremonger@intel.com>
> > Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
> > ---

Re: [dpdk-dev] [PATCH v6 2/4] examples/ipsec-secgw: add fallback session feature

[Akhil Goyal]

Hi Konstantin,
> 
> Hi Akhil,
> 
> > >
> > > Inline processing is limited to a specified subset of traffic. It is
> > > often unable to handle more complicated situations, such as fragmented
> > > traffic. When using inline processing such traffic is dropped.
> > >
> > > Introduce fallback session for inline processing allowing processing
> > > packets that normally would be dropped. A fallback session is
> > > configured by adding 'fallback' keyword with 'lookaside-none' or
> > > 'lookaside-protocol' parameter to an SA configuration.
> > >
> > > Using IPsec anti-replay window or ESN feature with fallback session is
> > > not yet supported when primary session is of type
> > > 'inline-protocol-offload' or fallback session is 'lookaside-protocol'
> > > because SA sequence number is not synchronized between software and
> > > hardware sessions. Fallback sessions are also limited to ingress IPsec
> > > traffic.
> > >
> > > Fallback session feature is not available in the legacy mode.
> > >
> > I started looking this patch, but some initial thoughts looking at the patch
> description.
> >
> > When you say a fallback session will be a lookaside none or lookaside protocol,
> > the packet will be processed asynchronously and might as well reorder.
> 
> Yes, we documented it as one of limitations.
> Though as I already mentioned for some use-cases some reordering it is
> acceptable.

Which usecases allow reordering. I think most stacks have replay window of less than 256/128 frames.
> 
>  > The best possible solution for this would be the synchronous API which are in
> talks
> 
> Agree, that would be a way to avoid reordering, but it is not there yet.
> 
> > in another patchset or use a SW PMD(eg. Openssl etc.) session and wait till you
> get the packet dequeued.
> > So effectively async APIs will be used to behave synchronously.
> > You can not use hardware PMD session as it will perform very badly for
> fallback packets
> > Because you have to wait till the packet is not getting dequeued back.
> 
> We don't plan to support that model because of great performance penalty you
> mentioned.

So what is currently supported with this patchset.
- cpu crypto is not there yet.
- SW PMD you are not supporting that model.

> 
> >
> > Having said that, you won't find a device or a scenario where you can use
> > Inline crypto as primary and lookaside proto as fallback.
> > It can only be like inline crypto as primary and lookaside none as fallback.
> 
> Yes, correct.
> I thought that we already removed lookaside-proto from supported types.
> If we didn't - will certainly do that.
> 
> >
> > BTW, I am ok with Patch 1/4 and 3/4. If no objections from the community, I
> can pick those.
> 
> Great to hear.
> What obstacles do you see with others two?
I believe there are some discussion going on between you and Anoob.


> Konstantin
> 
> >
> > -Akhil
> >
> > > Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
> > > Tested-by: Bernard Iremonger <bernard.iremonger@intel.com>
> > > Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
> > > ---

Re: [dpdk-dev] [PATCH v6 2/4] examples/ipsec-secgw: add fallback session feature

[Ananyev, Konstantin]

> > > > Inline processing is limited to a specified subset of traffic. It is
> > > > often unable to handle more complicated situations, such as fragmented
> > > > traffic. When using inline processing such traffic is dropped.
> > > >
> > > > Introduce fallback session for inline processing allowing processing
> > > > packets that normally would be dropped. A fallback session is
> > > > configured by adding 'fallback' keyword with 'lookaside-none' or
> > > > 'lookaside-protocol' parameter to an SA configuration.
> > > >
> > > > Using IPsec anti-replay window or ESN feature with fallback session is
> > > > not yet supported when primary session is of type
> > > > 'inline-protocol-offload' or fallback session is 'lookaside-protocol'
> > > > because SA sequence number is not synchronized between software and
> > > > hardware sessions. Fallback sessions are also limited to ingress IPsec
> > > > traffic.
> > > >
> > > > Fallback session feature is not available in the legacy mode.
> > > >
> > > I started looking this patch, but some initial thoughts looking at the patch
> > description.
> > >
> > > When you say a fallback session will be a lookaside none or lookaside protocol,
> > > the packet will be processed asynchronously and might as well reorder.
> >
> > Yes, we documented it as one of limitations.
> > Though as I already mentioned for some use-cases some reordering it is
> > acceptable.
> 
> Which usecases allow reordering.

AFAIK, VoIP, video-streaming, netflow/ipfix, some P2P protocols -  all of them tolerate packet reordering.
Actually modern TCP implementations are also quite robust to packet reorderings
(till some extent of course).

> I think most stacks have replay window of less than 256/128 frames.

From our measurements, on IA boxes, ~256 seems good enough for many cases. 

> >
> >  > The best possible solution for this would be the synchronous API which are in
> > talks
> >
> > Agree, that would be a way to avoid reordering, but it is not there yet.
> >
> > > in another patchset or use a SW PMD(eg. Openssl etc.) session and wait till you
> > get the packet dequeued.
> > > So effectively async APIs will be used to behave synchronously.
> > > You can not use hardware PMD session as it will perform very badly for
> > fallback packets
> > > Because you have to wait till the packet is not getting dequeued back.
> >
> > We don't plan to support that model because of great performance penalty you
> > mentioned.
> 
> So what is currently supported with this patchset.
> - cpu crypto is not there yet.
> - SW PMD you are not supporting that model.

ixbge(inline-crypto)+qat(lksd-crypto)

> 
> >
> > >
> > > Having said that, you won't find a device or a scenario where you can use
> > > Inline crypto as primary and lookaside proto as fallback.
> > > It can only be like inline crypto as primary and lookaside none as fallback.
> >
> > Yes, correct.
> > I thought that we already removed lookaside-proto from supported types.
> > If we didn't - will certainly do that.
> >
> > >
> > > BTW, I am ok with Patch 1/4 and 3/4. If no objections from the community, I
> > can pick those.
> >
> > Great to hear.
> > What obstacles do you see with others two?
> I believe there are some discussion going on between you and Anoob.

Seems too vague...
Anything concrete?
Konstantin

[dpdk-dev] [PATCH v6 3/4] examples/ipsec-secgw: add frag TTL cmdline option

[Marcin Smoczynski]

Due to fragment loss on highly saturated links and long fragment
lifetime, ipsec-secgw application quickly runs out of free reassembly
buckets. As a result new fragments are being dropped.

Introduce --frag-ttl option which allow user to lower default fragment
lifitime which solves problem of saturated reassembly buckets with high
bandwidth fragmented traffic.

Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
Tested-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
---
 doc/guides/sample_app_ug/ipsec_secgw.rst |  7 +++++
 examples/ipsec-secgw/ipsec-secgw.c       | 40 ++++++++++++++++++------
 2 files changed, 37 insertions(+), 10 deletions(-)

diff --git a/doc/guides/sample_app_ug/ipsec_secgw.rst b/doc/guides/sample_app_ug/ipsec_secgw.rst
index 45478e2a5..279ad4126 100644
--- a/doc/guides/sample_app_ug/ipsec_secgw.rst
+++ b/doc/guides/sample_app_ug/ipsec_secgw.rst
@@ -154,6 +154,13 @@ Where:
     Incoming packets with length bigger then MTU will be discarded.
     Default value: 1500.
 
+*   ``--frag-ttl FRAG_TTL_NS``: fragment lifetime (in nanoseconds).
+    If packet is not reassembled within this time, received fragments
+    will be discarded. Fragment lifetime should be decreased when
+    there is a high fragmented traffic loss in high bandwidth networks.
+    Should be lower for for low number of reassembly buckets.
+    Valid values: from 1 ns to 10 s. Default value: 10000000 (10 s).
+
 *   ``--reassemble NUM``: max number of entries in reassemble fragment table.
     Zero value disables reassembly functionality.
     Default value: 0.
diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index 641ed3767..1d415ace8 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -112,7 +112,7 @@ static uint16_t nb_txd = IPSEC_SECGW_TX_DESC_DEFAULT;
 		0, 0)
 
 #define	FRAG_TBL_BUCKET_ENTRIES	4
-#define	FRAG_TTL_MS		(10 * MS_PER_S)
+#define	MAX_FRAG_TTL_NS		(10LL * NS_PER_S)
 
 #define MTU_TO_FRAMELEN(x)	((x) + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN)
 
@@ -135,6 +135,7 @@ struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS] = {
 #define CMD_LINE_OPT_TX_OFFLOAD		"txoffload"
 #define CMD_LINE_OPT_REASSEMBLE		"reassemble"
 #define CMD_LINE_OPT_MTU		"mtu"
+#define CMD_LINE_OPT_FRAG_TTL		"frag-ttl"
 
 enum {
 	/* long options mapped to a short option */
@@ -150,6 +151,7 @@ enum {
 	CMD_LINE_OPT_TX_OFFLOAD_NUM,
 	CMD_LINE_OPT_REASSEMBLE_NUM,
 	CMD_LINE_OPT_MTU_NUM,
+	CMD_LINE_OPT_FRAG_TTL_NUM,
 };
 
 static const struct option lgopts[] = {
@@ -160,6 +162,7 @@ static const struct option lgopts[] = {
 	{CMD_LINE_OPT_TX_OFFLOAD, 1, 0, CMD_LINE_OPT_TX_OFFLOAD_NUM},
 	{CMD_LINE_OPT_REASSEMBLE, 1, 0, CMD_LINE_OPT_REASSEMBLE_NUM},
 	{CMD_LINE_OPT_MTU, 1, 0, CMD_LINE_OPT_MTU_NUM},
+	{CMD_LINE_OPT_FRAG_TTL, 1, 0, CMD_LINE_OPT_FRAG_TTL_NUM},
 	{NULL, 0, 0, 0}
 };
 
@@ -186,6 +189,7 @@ static uint64_t dev_tx_offload = UINT64_MAX;
 static uint32_t frag_tbl_sz;
 static uint32_t frame_buf_size = RTE_MBUF_DEFAULT_BUF_SIZE;
 static uint32_t mtu_size = RTE_ETHER_MTU;
+static uint64_t frag_ttl_ns = MAX_FRAG_TTL_NS;
 
 /* application wide librte_ipsec/SA parameters */
 struct app_sa_prm app_sa_prm = {.enable = 0};
@@ -1302,6 +1306,9 @@ print_usage(const char *prgname)
 		": MTU value on all ports (default value: 1500)\n"
 		"    outgoing packets with bigger size will be fragmented\n"
 		"    incoming packets with bigger size will be discarded\n"
+		"  --" CMD_LINE_OPT_FRAG_TTL " FRAG_TTL_NS"
+		": fragments lifetime in nanoseconds, default\n"
+		"    and maximum value is 10.000.000.000 ns (10 s)\n"
 		"\n",
 		prgname);
 }
@@ -1338,14 +1345,15 @@ parse_portmask(const char *portmask)
 	return pm;
 }
 
-static int32_t
+static int64_t
 parse_decimal(const char *str)
 {
 	char *end = NULL;
-	unsigned long num;
+	uint64_t num;
 
-	num = strtoul(str, &end, 10);
-	if ((str[0] == '\0') || (end == NULL) || (*end != '\0'))
+	num = strtoull(str, &end, 10);
+	if ((str[0] == '\0') || (end == NULL) || (*end != '\0')
+		|| num > INT64_MAX)
 		return -1;
 
 	return num;
@@ -1419,12 +1427,14 @@ print_app_sa_prm(const struct app_sa_prm *prm)
 	printf("replay window size: %u\n", prm->window_size);
 	printf("ESN: %s\n", (prm->enable_esn == 0) ? "disabled" : "enabled");
 	printf("SA flags: %#" PRIx64 "\n", prm->flags);
+	printf("Frag TTL: %" PRIu64 " ns\n", frag_ttl_ns);
 }
 
 static int32_t
 parse_args(int32_t argc, char **argv)
 {
-	int32_t opt, ret;
+	int opt;
+	int64_t ret;
 	char **argvopt;
 	int32_t option_index;
 	char *prgname = argv[0];
@@ -1503,7 +1513,7 @@ parse_args(int32_t argc, char **argv)
 			break;
 		case CMD_LINE_OPT_SINGLE_SA_NUM:
 			ret = parse_decimal(optarg);
-			if (ret == -1) {
+			if (ret == -1 || ret > UINT32_MAX) {
 				printf("Invalid argument[sa_idx]\n");
 				print_usage(prgname);
 				return -1;
@@ -1546,7 +1556,7 @@ parse_args(int32_t argc, char **argv)
 			break;
 		case CMD_LINE_OPT_REASSEMBLE_NUM:
 			ret = parse_decimal(optarg);
-			if (ret < 0) {
+			if (ret < 0 || ret > UINT32_MAX) {
 				printf("Invalid argument for \'%s\': %s\n",
 					CMD_LINE_OPT_REASSEMBLE, optarg);
 				print_usage(prgname);
@@ -1564,6 +1574,16 @@ parse_args(int32_t argc, char **argv)
 			}
 			mtu_size = ret;
 			break;
+		case CMD_LINE_OPT_FRAG_TTL_NUM:
+			ret = parse_decimal(optarg);
+			if (ret < 0 || ret > MAX_FRAG_TTL_NS) {
+				printf("Invalid argument for \'%s\': %s\n",
+					CMD_LINE_OPT_MTU, optarg);
+				print_usage(prgname);
+				return -1;
+			}
+			frag_ttl_ns = ret;
+			break;
 		default:
 			print_usage(prgname);
 			return -1;
@@ -2324,8 +2344,8 @@ reassemble_lcore_init(struct lcore_conf *lc, uint32_t cid)
 
 	/* create fragment table */
 	sid = rte_lcore_to_socket_id(cid);
-	frag_cycles = (rte_get_tsc_hz() + MS_PER_S - 1) /
-		MS_PER_S * FRAG_TTL_MS;
+	frag_cycles = (rte_get_tsc_hz() + NS_PER_S - 1) /
+		NS_PER_S * frag_ttl_ns;
 
 	lc->frag.tbl = rte_ip_frag_table_create(frag_tbl_sz,
 		FRAG_TBL_BUCKET_ENTRIES, frag_tbl_sz, frag_cycles, sid);
-- 
2.17.1

[dpdk-dev] [PATCH v6 4/4] examples/ipsec-secgw: add offload fallback tests

[Marcin Smoczynski]

Add tests for offload fallback feature; add inbound config modificator
SGW_CFG_XPRM_IN (offload fallback setting can be set only for inbound
SAs). Tests are using cryptodev for outbound SA.

To test fragmentation with QAT set:
MULTI_SEG_TEST="--reassemble=4096 --cryptodev_mask=0x5555"

Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
Tested-by: Bernard Iremonger <bernard.iremonger@intel.com>
Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
---
 doc/guides/sample_app_ug/ipsec_secgw.rst                    | 5 +++++
 examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh         | 4 ++--
 .../test/trs_aesgcm_inline_crypto_fallback_defs.sh          | 5 +++++
 examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh         | 6 ++++--
 .../test/tun_aesgcm_inline_crypto_fallback_defs.sh          | 5 +++++
 5 files changed, 21 insertions(+), 4 deletions(-)
 create mode 100644 examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
 create mode 100644 examples/ipsec-secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh

diff --git a/doc/guides/sample_app_ug/ipsec_secgw.rst b/doc/guides/sample_app_ug/ipsec_secgw.rst
index 279ad4126..0dc81fbb4 100644
--- a/doc/guides/sample_app_ug/ipsec_secgw.rst
+++ b/doc/guides/sample_app_ug/ipsec_secgw.rst
@@ -755,6 +755,11 @@ Also the user can optionally setup:
 *   ``CRYPTO_DEV``: crypto device to be used ('-w <pci-id>'). If none specified
     appropriate vdevs will be created by the script
 
+*   ``MULTI_SEG_TEST``: ipsec-secgw option to enable reassembly support and
+    specify size of reassembly table (e.g.
+    ``MULTI_SEG_TEST='--reassemble 128'``). This option must be set for
+    fallback session tests.
+
 Note that most of the tests require the appropriate crypto PMD/device to be
 available.
 
diff --git a/examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh b/examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh
index f6c5bf5a7..17f2f86d2 100644
--- a/examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh
+++ b/examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh
@@ -29,11 +29,11 @@ sp ipv6 out esp bypass pri 1 sport 0:65535 dport 0:65535
 #SA in rules
 sa in 7 aead_algo aes-128-gcm \
 aead_key de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef \
-mode transport ${SGW_CFG_XPRM}
+mode transport ${SGW_CFG_XPRM} ${SGW_CFG_XPRM_IN}
 
 sa in 9 aead_algo aes-128-gcm \
 aead_key de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef \
-mode transport ${SGW_CFG_XPRM}
+mode transport ${SGW_CFG_XPRM} ${SGW_CFG_XPRM_IN}
 
 #SA out rules
 sa out 7 aead_algo aes-128-gcm \
diff --git a/examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh b/examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
new file mode 100644
index 000000000..875a7457d
--- /dev/null
+++ b/examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
@@ -0,0 +1,5 @@
+#! /bin/bash
+
+. ${DIR}/trs_aesgcm_defs.sh
+
+SGW_CFG_XPRM_IN='port_id 0 type inline-crypto-offload fallback lookaside-none'
diff --git a/examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh b/examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh
index 278377967..7490baded 100644
--- a/examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh
+++ b/examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh
@@ -29,11 +29,13 @@ sp ipv6 out esp bypass pri 1 sport 0:65535 dport 0:65535
 #SA in rules
 sa in 7 aead_algo aes-128-gcm \
 aead_key de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef \
-mode ipv4-tunnel src ${REMOTE_IPV4} dst ${LOCAL_IPV4} ${SGW_CFG_XPRM}
+mode ipv4-tunnel src ${REMOTE_IPV4} dst ${LOCAL_IPV4} ${SGW_CFG_XPRM} \
+${SGW_CFG_XPRM_IN}
 
 sa in 9 aead_algo aes-128-gcm \
 aead_key de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef \
-mode ipv6-tunnel src ${REMOTE_IPV6} dst ${LOCAL_IPV6} ${SGW_CFG_XPRM}
+mode ipv6-tunnel src ${REMOTE_IPV6} dst ${LOCAL_IPV6} ${SGW_CFG_XPRM} \
+${SGW_CFG_XPRM_IN}
 
 #SA out rules
 sa out 7 aead_algo aes-128-gcm \
diff --git a/examples/ipsec-secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh b/examples/ipsec-secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh
new file mode 100644
index 000000000..696848432
--- /dev/null
+++ b/examples/ipsec-secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh
@@ -0,0 +1,5 @@
+#! /bin/bash
+
+. ${DIR}/tun_aesgcm_defs.sh
+
+SGW_CFG_XPRM_IN='port_id 0 type inline-crypto-offload fallback lookaside-none'
-- 
2.17.1

[dpdk-dev] [PATCH v7 0/4] add fallback session

[Marcin Smoczynski]

Add fallback session feature allowing to process packets that inline
processor is unable to handle (e.g. fragmented traffic). Processing
takes place in a secondary session defined for SA in a configuration
file.

This feature is limited to ingress IPsec traffic only. IPsec
anti-replay window and ESN are supported in conjunction with fallback
session when following conditions are met:
 * primary session is 'inline-crypto-offload,
 * fallback sessions is 'lookaside-none'.
Due to different processing times of inline and lookaside modes,
fallback session introduces some packet reordering, therefore when
using with IPsec window, its value should be increased.

v6 to v7 changes:
 - remove partial support for "*-protocol" fall-back session types
 - rebase on master (31b798a)

v5 to v6 changes:
 - add sanity check: fail to parse SA configuration if fallback session
   is configured but primary session is not inline crypto
 - update documentation for ipsec-secgw tests (MULTI_SEG_TESTS var
   described)
 - add release notes
 - minor commit log changes

v4 to v5 changes:
 - fix build errors related to frag TTL command line option and
   parse_decimal method

v3 to v4 changes:
 - add info about packet reordering to the documentation regarding
   fallback session
 - add patch with --frag-ttl command line option which allows to change
   fragment lifetime

v2 to v3 changes:
 - doc and commit log update - explicitly state feature limitations

v1 to v2 changes:
 - disable fallback offload for outbound SAs
 - add test scripts

Marcin Smoczynski (4):
  examples/ipsec-secgw: sa structure cleanup
  examples/ipsec-secgw: add fallback session feature
  examples/ipsec-secgw: add frag TTL cmdline option
  examples/ipsec-secgw: add offload fallback tests

 doc/guides/rel_notes/release_19_11.rst        |   8 +
 doc/guides/sample_app_ug/ipsec_secgw.rst      |  34 +++-
 examples/ipsec-secgw/esp.c                    |  35 ++--
 examples/ipsec-secgw/ipsec-secgw.c            |  56 ++++--
 examples/ipsec-secgw/ipsec.c                  | 101 +++++------
 examples/ipsec-secgw/ipsec.h                  |  61 +++++--
 examples/ipsec-secgw/ipsec_process.c          | 113 ++++++++-----
 examples/ipsec-secgw/sa.c                     | 159 +++++++++++++-----
 .../test/trs_aesgcm_common_defs.sh            |   4 +-
 .../trs_aesgcm_inline_crypto_fallback_defs.sh |   5 +
 .../test/tun_aesgcm_common_defs.sh            |   6 +-
 .../tun_aesgcm_inline_crypto_fallback_defs.sh |   5 +
 12 files changed, 409 insertions(+), 178 deletions(-)
 create mode 100644 examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
 create mode 100644 examples/ipsec-secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh

--
2.17.1

[dpdk-dev] [PATCH v7 1/4] examples/ipsec-secgw: sa structure cleanup

[Marcin Smoczynski]

Cleanup ipsec_sa structure by removing every field that is already in
the rte_ipsec_session structure:
 * cryptodev/security session union
 * action type
 * offload flags
 * security context
References to abovementioned fields are changed to direct references
to matching fields of rte_ipsec_session structure.

Such refactoring is needed to introduce many sessions per SA feature,
e.g. fallback session for inline offload processing.

Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
Acked-by: Radu Nicolau <radu.nicolau@intel.com>
Tested-by: Bernard Iremonger <bernard.iremonger@intel.com>
Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
---
 examples/ipsec-secgw/esp.c           | 35 +++++++----
 examples/ipsec-secgw/ipsec.c         | 93 +++++++++++++++-------------
 examples/ipsec-secgw/ipsec.h         | 27 +++++---
 examples/ipsec-secgw/ipsec_process.c | 30 ++++-----
 examples/ipsec-secgw/sa.c            | 66 ++++++++++----------
 5 files changed, 138 insertions(+), 113 deletions(-)

diff --git a/examples/ipsec-secgw/esp.c b/examples/ipsec-secgw/esp.c
index d6d7b1256..c1b49da1e 100644
--- a/examples/ipsec-secgw/esp.c
+++ b/examples/ipsec-secgw/esp.c
@@ -30,7 +30,8 @@ esp_inbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 	int32_t payload_len, ip_hdr_len;
 
 	RTE_ASSERT(sa != NULL);
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
+	if (ipsec_get_action_type(sa) ==
+			RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
 		return 0;
 
 	RTE_ASSERT(m != NULL);
@@ -148,13 +149,16 @@ esp_inbound_post(struct rte_mbuf *m, struct ipsec_sa *sa,
 	uint8_t *nexthdr, *pad_len;
 	uint8_t *padding;
 	uint16_t i;
+	struct rte_ipsec_session *ips;
 
 	RTE_ASSERT(m != NULL);
 	RTE_ASSERT(sa != NULL);
 	RTE_ASSERT(cop != NULL);
 
-	if ((sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
-			(sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
+	ips = ipsec_get_session(sa);
+
+	if ((ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
+			(ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
 		if (m->ol_flags & PKT_RX_SEC_OFFLOAD) {
 			if (m->ol_flags & PKT_RX_SEC_OFFLOAD_FAILED)
 				cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
@@ -169,8 +173,8 @@ esp_inbound_post(struct rte_mbuf *m, struct ipsec_sa *sa,
 		return -1;
 	}
 
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO &&
-	    sa->ol_flags & RTE_SECURITY_RX_HW_TRAILER_OFFLOAD) {
+	if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO &&
+		ips->security.ol_flags & RTE_SECURITY_RX_HW_TRAILER_OFFLOAD) {
 		nexthdr = &m->inner_esp_next_proto;
 	} else {
 		nexthdr = rte_pktmbuf_mtod_offset(m, uint8_t*,
@@ -225,10 +229,12 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 	struct rte_crypto_sym_op *sym_cop;
 	int32_t i;
 	uint16_t pad_payload_len, pad_len, ip_hdr_len;
+	struct rte_ipsec_session *ips;
 
 	RTE_ASSERT(m != NULL);
 	RTE_ASSERT(sa != NULL);
 
+	ips = ipsec_get_session(sa);
 	ip_hdr_len = 0;
 
 	ip4 = rte_pktmbuf_mtod(m, struct ip *);
@@ -277,9 +283,10 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 	}
 
 	/* Add trailer padding if it is not constructed by HW */
-	if (sa->type != RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
-	    (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO &&
-	     !(sa->ol_flags & RTE_SECURITY_TX_HW_TRAILER_OFFLOAD))) {
+	if (ips->type != RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+		(ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO &&
+		 !(ips->security.ol_flags &
+			 RTE_SECURITY_TX_HW_TRAILER_OFFLOAD))) {
 		padding = (uint8_t *)rte_pktmbuf_append(m, pad_len +
 							sa->digest_len);
 		if (unlikely(padding == NULL)) {
@@ -344,8 +351,9 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 		}
 	}
 
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
-		if (sa->ol_flags & RTE_SECURITY_TX_HW_TRAILER_OFFLOAD) {
+	if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		if (ips->security.ol_flags &
+				RTE_SECURITY_TX_HW_TRAILER_OFFLOAD) {
 			/* Set the inner esp next protocol for HW trailer */
 			m->inner_esp_next_proto = nlp;
 			m->packet_type |= RTE_PTYPE_TUNNEL_ESP;
@@ -448,11 +456,14 @@ esp_outbound_post(struct rte_mbuf *m,
 		  struct ipsec_sa *sa,
 		  struct rte_crypto_op *cop)
 {
+	enum rte_security_session_action_type type;
 	RTE_ASSERT(m != NULL);
 	RTE_ASSERT(sa != NULL);
 
-	if ((sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
-			(sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
+	type = ipsec_get_action_type(sa);
+
+	if ((type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
+			(type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
 		m->ol_flags |= PKT_TX_SEC_OFFLOAD;
 	} else {
 		RTE_ASSERT(cop != NULL);
diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index 51fb22e8a..5b63a13fd 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -52,7 +52,8 @@ set_ipsec_conf(struct ipsec_sa *sa, struct rte_security_ipsec_xform *ipsec)
 }
 
 int
-create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips)
 {
 	struct rte_cryptodev_info cdev_info;
 	unsigned long cdev_id_qp = 0;
@@ -83,9 +84,9 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			ipsec_ctx->tbl[cdev_id_qp].id,
 			ipsec_ctx->tbl[cdev_id_qp].qp);
 
-	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
+	if (ips->type != RTE_SECURITY_ACTION_TYPE_NONE) {
 		struct rte_security_session_conf sess_conf = {
-			.action_type = sa->type,
+			.action_type = ips->type,
 			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 			{.ipsec = {
 				.spi = sa->spi,
@@ -102,7 +103,7 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 
 		};
 
-		if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
+		if (ips->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
 			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
 							rte_cryptodev_get_sec_ctx(
 							ipsec_ctx->tbl[cdev_id_qp].id);
@@ -110,9 +111,9 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			/* Set IPsec parameters in conf */
 			set_ipsec_conf(sa, &(sess_conf.ipsec));
 
-			sa->sec_session = rte_security_session_create(ctx,
+			ips->security.ses = rte_security_session_create(ctx,
 					&sess_conf, ipsec_ctx->session_priv_pool);
-			if (sa->sec_session == NULL) {
+			if (ips->security.ses == NULL) {
 				RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 				return -1;
@@ -122,10 +123,10 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			return -1;
 		}
 	} else {
-		sa->crypto_session = rte_cryptodev_sym_session_create(
+		ips->crypto.ses = rte_cryptodev_sym_session_create(
 				ipsec_ctx->session_pool);
 		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
-				sa->crypto_session, sa->xforms,
+				ips->crypto.ses, sa->xforms,
 				ipsec_ctx->session_priv_pool);
 
 		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
@@ -138,12 +139,13 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 }
 
 int
-create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips)
 {
 	int32_t ret = 0;
 	struct rte_security_ctx *sec_ctx;
 	struct rte_security_session_conf sess_conf = {
-		.action_type = sa->type,
+		.action_type = ips->type,
 		.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 		{.ipsec = {
 			.spi = sa->spi,
@@ -163,7 +165,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on port %u\n",
 		sa->spi, sa->portid);
 
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+	if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
 		struct rte_flow_error err;
 		const struct rte_security_capability *sec_cap;
 		int ret = 0;
@@ -177,9 +179,9 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 			return -1;
 		}
 
-		sa->sec_session = rte_security_session_create(sec_ctx,
+		ips->security.ses = rte_security_session_create(sec_ctx,
 				&sess_conf, skt_ctx->session_pool);
-		if (sa->sec_session == NULL) {
+		if (ips->security.ses == NULL) {
 			RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 			return -1;
@@ -189,7 +191,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 
 		/* iterate until ESP tunnel*/
 		while (sec_cap->action != RTE_SECURITY_ACTION_TYPE_NONE) {
-			if (sec_cap->action == sa->type &&
+			if (sec_cap->action == ips->type &&
 			    sec_cap->protocol ==
 				RTE_SECURITY_PROTOCOL_IPSEC &&
 			    sec_cap->ipsec.mode ==
@@ -205,8 +207,8 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 			return -1;
 		}
 
-		sa->ol_flags = sec_cap->ol_flags;
-		sa->security_ctx = sec_ctx;
+		ips->security.ol_flags = sec_cap->ol_flags;
+		ips->security.ctx = sec_ctx;
 		sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
 
 		if (IS_IP6(sa->flags)) {
@@ -235,7 +237,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 		sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
 
 		sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
-		sa->action[0].conf = sa->sec_session;
+		sa->action[0].conf = ips->security.ses;
 
 		sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
 
@@ -308,8 +310,8 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 			if (ret)
 				goto flow_create_failure;
 		} else if (sa->attr.egress &&
-			   (sa->ol_flags &
-				    RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
+				(ips->security.ol_flags &
+					RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
 			sa->action[1].type =
 					RTE_FLOW_ACTION_TYPE_PASSTHRU;
 			sa->action[2].type =
@@ -325,7 +327,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 				err.message);
 			return -1;
 		}
-	} else if (sa->type ==	RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
+	} else if (ips->type ==	RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
 		const struct rte_security_capability *sec_cap;
 
 		sec_ctx = (struct rte_security_ctx *)
@@ -353,9 +355,9 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 
 		sess_conf.userdata = (void *) sa;
 
-		sa->sec_session = rte_security_session_create(sec_ctx,
+		ips->security.ses = rte_security_session_create(sec_ctx,
 					&sess_conf, skt_ctx->session_pool);
-		if (sa->sec_session == NULL) {
+		if (ips->security.ses == NULL) {
 			RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 			return -1;
@@ -371,7 +373,7 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 		/* iterate until ESP tunnel*/
 		while (sec_cap->action !=
 				RTE_SECURITY_ACTION_TYPE_NONE) {
-			if (sec_cap->action == sa->type &&
+			if (sec_cap->action == ips->type &&
 			    sec_cap->protocol ==
 				RTE_SECURITY_PROTOCOL_IPSEC &&
 			    sec_cap->ipsec.mode ==
@@ -387,8 +389,8 @@ create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
 			return -1;
 		}
 
-		sa->ol_flags = sec_cap->ol_flags;
-		sa->security_ctx = sec_ctx;
+		ips->security.ol_flags = sec_cap->ol_flags;
+		ips->security.ctx = sec_ctx;
 	}
 	sa->cdev_id_qp = 0;
 
@@ -435,6 +437,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 	struct ipsec_mbuf_metadata *priv;
 	struct rte_crypto_sym_op *sym_cop;
 	struct ipsec_sa *sa;
+	struct rte_ipsec_session *ips;
 
 	for (i = 0; i < nb_pkts; i++) {
 		if (unlikely(sas[i] == NULL)) {
@@ -448,16 +451,17 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 		priv = get_priv(pkts[i]);
 		sa = sas[i];
 		priv->sa = sa;
+		ips = ipsec_get_session(sa);
 
-		switch (sa->type) {
+		switch (ips->type) {
 		case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL:
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
 
-			if ((unlikely(sa->sec_session == NULL)) &&
-				create_lookaside_session(ipsec_ctx, sa)) {
+			if ((unlikely(ips->security.ses == NULL)) &&
+				create_lookaside_session(ipsec_ctx, sa, ips)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -466,7 +470,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			sym_cop->m_src = pkts[i];
 
 			rte_security_attach_session(&priv->cop,
-					sa->sec_session);
+				ips->security.ses);
 			break;
 		case RTE_SECURITY_ACTION_TYPE_NONE:
 
@@ -475,14 +479,14 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			rte_prefetch0(&priv->sym_cop);
 
-			if ((unlikely(sa->crypto_session == NULL)) &&
-				create_lookaside_session(ipsec_ctx, sa)) {
+			if ((unlikely(ips->crypto.ses == NULL)) &&
+				create_lookaside_session(ipsec_ctx, sa, ips)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
 
 			rte_crypto_op_attach_sym_session(&priv->cop,
-					sa->crypto_session);
+					ips->crypto.ses);
 
 			ret = xform_func(pkts[i], sa, &priv->cop);
 			if (unlikely(ret)) {
@@ -491,21 +495,22 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 			break;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
-			RTE_ASSERT(sa->sec_session != NULL);
+			RTE_ASSERT(ips->security.ses != NULL);
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
-			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
+			if (ips->security.ol_flags &
+				RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
-						sa->security_ctx,
-						sa->sec_session, pkts[i], NULL);
+					ips->security.ctx, ips->security.ses,
+					pkts[i], NULL);
 			continue;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
-			RTE_ASSERT(sa->sec_session != NULL);
+			RTE_ASSERT(ips->security.ses != NULL);
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
 			rte_security_attach_session(&priv->cop,
-					sa->sec_session);
+					ips->security.ses);
 
 			ret = xform_func(pkts[i], sa, &priv->cop);
 			if (unlikely(ret)) {
@@ -514,10 +519,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
-			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
+			if (ips->security.ol_flags &
+				RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
-						sa->security_ctx,
-						sa->sec_session, pkts[i], NULL);
+					ips->security.ctx, ips->security.ses,
+					pkts[i], NULL);
 			continue;
 		}
 
@@ -586,13 +592,14 @@ ipsec_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			RTE_ASSERT(sa != NULL);
 
-			if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {
+			if (ipsec_get_action_type(sa) ==
+				RTE_SECURITY_ACTION_TYPE_NONE) {
 				ret = xform_func(pkt, sa, cops[j]);
 				if (unlikely(ret)) {
 					rte_pktmbuf_free(pkt);
 					continue;
 				}
-			} else if (sa->type ==
+			} else if (ipsec_get_action_type(sa) ==
 				RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
 				if (cops[j]->status) {
 					rte_pktmbuf_free(pkt);
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index 1efa6e488..a4ad81b0e 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -87,10 +87,6 @@ struct ipsec_sa {
 	uint32_t cdev_id_qp;
 	uint64_t seq;
 	uint32_t salt;
-	union {
-		struct rte_cryptodev_sym_session *crypto_session;
-		struct rte_security_session *sec_session;
-	};
 	enum rte_crypto_cipher_algorithm cipher_algo;
 	enum rte_crypto_auth_algorithm auth_algo;
 	enum rte_crypto_aead_algorithm aead_algo;
@@ -114,11 +110,8 @@ struct ipsec_sa {
 		struct rte_crypto_sym_xform *xforms;
 		struct rte_security_ipsec_xform *sec_xform;
 	};
-	enum rte_security_session_action_type type;
 	enum rte_security_ipsec_sa_direction direction;
 	uint16_t portid;
-	struct rte_security_ctx *security_ctx;
-	uint32_t ol_flags;
 
 #define MAX_RTE_FLOW_PATTERN (4)
 #define MAX_RTE_FLOW_ACTIONS (3)
@@ -284,6 +277,20 @@ get_sym_cop(struct rte_crypto_op *cop)
 	return (cop + 1);
 }
 
+static inline struct rte_ipsec_session *
+ipsec_get_session(struct ipsec_sa *sa)
+{
+	return &sa->ips;
+}
+
+static inline enum rte_security_session_action_type
+ipsec_get_action_type(struct ipsec_sa *sa)
+{
+	struct rte_ipsec_session *ips;
+	ips = ipsec_get_session(sa);
+	return ips->type;
+}
+
 int
 inbound_sa_check(struct sa_ctx *sa_ctx, struct rte_mbuf *m, uint32_t sa_idx);
 
@@ -338,9 +345,11 @@ void
 enqueue_cop_burst(struct cdev_qp *cqp);
 
 int
-create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips);
 
 int
-create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa);
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips);
 
 #endif /* __IPSEC_H__ */
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index 868f1a28d..239d81ef6 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -94,22 +94,16 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
 
 	/* setup crypto section */
 	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		if (sa->crypto_session == NULL) {
-			rc = create_lookaside_session(ctx, sa);
-			if (rc != 0)
-				return rc;
-		}
-		ss->crypto.ses = sa->crypto_session;
+		RTE_ASSERT(ss->crypto.ses == NULL);
+		rc = create_lookaside_session(ctx, sa, ss);
+		if (rc != 0)
+			return rc;
 	/* setup session action type */
-	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
-		if (sa->sec_session == NULL) {
-			rc = create_lookaside_session(ctx, sa);
-			if (rc != 0)
-				return rc;
-		}
-		ss->security.ses = sa->sec_session;
-		ss->security.ctx = sa->security_ctx;
-		ss->security.ol_flags = sa->ol_flags;
+	} else if (ss->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
+		RTE_ASSERT(ss->security.ses == NULL);
+		rc = create_lookaside_session(ctx, sa, ss);
+		if (rc != 0)
+			return rc;
 	} else
 		RTE_ASSERT(0);
 
@@ -218,7 +212,7 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 		pg = grp + i;
 		sa = pg->id.ptr;
 
-		ips = &sa->ips;
+		ips = ipsec_get_session(sa);
 
 		/* no valid HW session for that SA, try to create one */
 		if (sa == NULL || (ips->crypto.ses == NULL &&
@@ -226,8 +220,8 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 			k = 0;
 
 		/* process packets inline */
-		else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
-				sa->type ==
+		else if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+				ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
 
 			satp = rte_ipsec_sa_type(ips->sa);
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index 14ee94731..52752e15f 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -224,6 +224,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	struct parse_status *status)
 {
 	struct ipsec_sa *rule = NULL;
+	struct rte_ipsec_session *ips;
 	uint32_t ti; /*token index*/
 	uint32_t *ri /*rule index*/;
 	uint32_t cipher_algo_p = 0;
@@ -262,6 +263,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	if (atoi(tokens[1]) == INVALID_SPI)
 		return;
 	rule->spi = atoi(tokens[1]);
+	ips = ipsec_get_session(rule);
 
 	for (ti = 2; ti < n_tokens; ti++) {
 		if (strcmp(tokens[ti], "mode") == 0) {
@@ -558,18 +560,18 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 				return;
 
 			if (strcmp(tokens[ti], "inline-crypto-offload") == 0)
-				rule->type =
+				ips->type =
 					RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO;
 			else if (strcmp(tokens[ti],
 					"inline-protocol-offload") == 0)
-				rule->type =
+				ips->type =
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL;
 			else if (strcmp(tokens[ti],
 					"lookaside-protocol-offload") == 0)
-				rule->type =
+				ips->type =
 				RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL;
 			else if (strcmp(tokens[ti], "no-offload") == 0)
-				rule->type = RTE_SECURITY_ACTION_TYPE_NONE;
+				ips->type = RTE_SECURITY_ACTION_TYPE_NONE;
 			else {
 				APP_CHECK(0, status, "Invalid input \"%s\"",
 						tokens[ti]);
@@ -624,11 +626,11 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	if (status->status < 0)
 		return;
 
-	if ((rule->type != RTE_SECURITY_ACTION_TYPE_NONE) && (portid_p == 0))
+	if ((ips->type != RTE_SECURITY_ACTION_TYPE_NONE) && (portid_p == 0))
 		printf("Missing portid option, falling back to non-offload\n");
 
 	if (!type_p || !portid_p) {
-		rule->type = RTE_SECURITY_ACTION_TYPE_NONE;
+		ips->type = RTE_SECURITY_ACTION_TYPE_NONE;
 		rule->portid = -1;
 	}
 
@@ -640,6 +642,7 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 {
 	uint32_t i;
 	uint8_t a, b, c, d;
+	const struct rte_ipsec_session *ips;
 
 	printf("\tspi_%s(%3u):", inbound?"in":"out", sa->spi);
 
@@ -695,8 +698,10 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 		printf("Transport ");
 		break;
 	}
+
+	ips = &sa->ips;
 	printf(" type:");
-	switch (sa->type) {
+	switch (ips->type) {
 	case RTE_SECURITY_ACTION_TYPE_NONE:
 		printf("no-offload ");
 		break;
@@ -884,6 +889,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 	uint16_t iv_length, aad_length;
 	int inline_status;
 	int32_t rc;
+	struct rte_ipsec_session *ips;
 
 	/* for ESN upper 32 bits of SQN also need to be part of AAD */
 	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
@@ -898,9 +904,10 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 		}
 		*sa = entries[i];
 		sa->seq = 0;
+		ips = ipsec_get_session(sa);
 
-		if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
-			sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
+			ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
 			if (check_eth_dev_caps(sa->portid, inbound))
 				return -EINVAL;
 		}
@@ -915,7 +922,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 			sa->dst.ip.ip4 = rte_cpu_to_be_32(sa->dst.ip.ip4);
 			break;
 		case TRANSPORT:
-			if (sa->type ==
+			if (ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
 				inline_status =
 					sa_add_address_inline_crypto(sa);
@@ -926,6 +933,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 		}
 
 		if (sa->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) {
+			struct rte_ipsec_session *ips;
 			iv_length = 16;
 
 			sa_ctx->xf[idx].a.type = RTE_CRYPTO_SYM_XFORM_AEAD;
@@ -946,11 +954,12 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			sa->xforms = &sa_ctx->xf[idx].a;
 
-			if (sa->type ==
+			ips = ipsec_get_session(sa);
+			if (ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
-				sa->type ==
+				ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
-				rc = create_inline_session(skt_ctx, sa);
+				rc = create_inline_session(skt_ctx, sa, ips);
 				if (rc != 0) {
 					RTE_LOG(ERR, IPSEC_ESP,
 						"create_inline_session() failed\n");
@@ -1108,23 +1117,11 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm, const struct ipsec_sa *ss,
 }
 
 static int
-fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
-	const struct ipsec_sa *lsa)
+fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa)
 {
 	int32_t rc = 0;
 
 	ss->sa = sa;
-	ss->type = lsa->type;
-
-	/* setup crypto section */
-	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		ss->crypto.ses = lsa->crypto_session;
-	/* setup session action type */
-	} else {
-		ss->security.ses = lsa->sec_session;
-		ss->security.ctx = lsa->security_ctx;
-		ss->security.ol_flags = lsa->ol_flags;
-	}
 
 	if (ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
 		ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
@@ -1146,6 +1143,7 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 {
 	int rc;
 	struct rte_ipsec_sa_prm prm;
+	struct rte_ipsec_session *ips;
 	struct rte_ipv4_hdr v4  = {
 		.version_ihl = IPVERSION << 4 |
 			sizeof(v4) / RTE_IPV4_IHL_MULTIPLIER,
@@ -1170,7 +1168,10 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
+	/* init processing session */
+	ips = ipsec_get_session(lsa);
+	rc = fill_ipsec_session(ips, sa);
+
 	return rc;
 }
 
@@ -1394,6 +1395,7 @@ sa_check_offloads(uint16_t port_id, uint64_t *rx_offloads,
 {
 	struct ipsec_sa *rule;
 	uint32_t idx_sa;
+	enum rte_security_session_action_type rule_type;
 
 	*rx_offloads = 0;
 	*tx_offloads = 0;
@@ -1401,8 +1403,9 @@ sa_check_offloads(uint16_t port_id, uint64_t *rx_offloads,
 	/* Check for inbound rules that use offloads and use this port */
 	for (idx_sa = 0; idx_sa < nb_sa_in; idx_sa++) {
 		rule = &sa_in[idx_sa];
-		if ((rule->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
-				rule->type ==
+		rule_type = ipsec_get_action_type(rule);
+		if ((rule_type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+				rule_type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
 				&& rule->portid == port_id)
 			*rx_offloads |= DEV_RX_OFFLOAD_SECURITY;
@@ -1411,8 +1414,9 @@ sa_check_offloads(uint16_t port_id, uint64_t *rx_offloads,
 	/* Check for outbound rules that use offloads and use this port */
 	for (idx_sa = 0; idx_sa < nb_sa_out; idx_sa++) {
 		rule = &sa_out[idx_sa];
-		if ((rule->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
-				rule->type ==
+		rule_type = ipsec_get_action_type(rule);
+		if ((rule_type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+				rule_type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
 				&& rule->portid == port_id)
 			*tx_offloads |= DEV_TX_OFFLOAD_SECURITY;
-- 
2.17.1

[dpdk-dev] [PATCH v7 2/4] examples/ipsec-secgw: add fallback session feature

[Marcin Smoczynski]

Inline processing is limited to a specified subset of traffic. It is
often unable to handle more complicated situations, such as fragmented
traffic. When using inline processing such traffic is dropped.

Introduce fallback session for inline crypto processing allowing
handling packets that normally would be dropped. A fallback session is
configured by adding 'fallback' keyword with 'lookaside-none' parameter
to an SA configuration. Only 'inline-crypto-offload" as a primary
session and 'lookaside-none' as a fall-back session combination is
supported by this patch.

Fallback session feature is not available in the legacy mode.

Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
Tested-by: Bernard Iremonger <bernard.iremonger@intel.com>
Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
---
 doc/guides/rel_notes/release_19_11.rst   |   8 ++
 doc/guides/sample_app_ug/ipsec_secgw.rst |  22 ++++-
 examples/ipsec-secgw/esp.c               |   4 +-
 examples/ipsec-secgw/ipsec-secgw.c       |  16 ++--
 examples/ipsec-secgw/ipsec.c             |  10 +--
 examples/ipsec-secgw/ipsec.h             |  40 +++++++--
 examples/ipsec-secgw/ipsec_process.c     |  85 ++++++++++++------
 examples/ipsec-secgw/sa.c                | 105 +++++++++++++++++++----
 8 files changed, 226 insertions(+), 64 deletions(-)

diff --git a/doc/guides/rel_notes/release_19_11.rst b/doc/guides/rel_notes/release_19_11.rst
index 85953b962..7e1205be7 100644
--- a/doc/guides/rel_notes/release_19_11.rst
+++ b/doc/guides/rel_notes/release_19_11.rst
@@ -115,6 +115,14 @@ New Features
   Added eBPF JIT support for arm64 architecture to improve the eBPF program
   performance.
 
+* **Updated the IPSec library and IPsec Security Gateway application.**
+
+  Added the following features to ``librte_ipsec`` and the ``ipsec-secgw``
+  sample application:
+
+  * Support fragmented packets in inline crypto processing mode with fallback
+    ``lookaside-none`` session.
+
 
 Removed Items
 -------------
diff --git a/doc/guides/sample_app_ug/ipsec_secgw.rst b/doc/guides/sample_app_ug/ipsec_secgw.rst
index ad2d79e75..cb6fadb35 100644
--- a/doc/guides/sample_app_ug/ipsec_secgw.rst
+++ b/doc/guides/sample_app_ug/ipsec_secgw.rst
@@ -401,7 +401,7 @@ The SA rule syntax is shown as follows:
 .. code-block:: console
 
     sa <dir> <spi> <cipher_algo> <cipher_key> <auth_algo> <auth_key>
-    <mode> <src_ip> <dst_ip> <action_type> <port_id>
+    <mode> <src_ip> <dst_ip> <action_type> <port_id> <fallback>
 
 where each options means:
 
@@ -573,6 +573,26 @@ where each options means:
 
    * *port_id X* X is a valid device number in decimal
 
+ ``<fallback>``
+
+ * Action type for ingress IPsec packets that inline processor failed to
+   process. Only a combination of *inline-crypto-offload* as a primary
+   session and *lookaside-none* as a fall-back session is supported at the
+   moment.
+
+   If used in conjunction with IPsec window, its width needs be increased
+   due to different processing times of inline and lookaside modes which
+   results in packet reordering.
+
+ * Optional: Yes.
+
+ * Available options:
+
+   * *lookaside-none*: use automatically chosen cryptodev to process packets
+
+ * Syntax:
+
+   * *fallback lookaside-none*
 
 Example SA rules:
 
diff --git a/examples/ipsec-secgw/esp.c b/examples/ipsec-secgw/esp.c
index c1b49da1e..bfa7ff721 100644
--- a/examples/ipsec-secgw/esp.c
+++ b/examples/ipsec-secgw/esp.c
@@ -155,7 +155,7 @@ esp_inbound_post(struct rte_mbuf *m, struct ipsec_sa *sa,
 	RTE_ASSERT(sa != NULL);
 	RTE_ASSERT(cop != NULL);
 
-	ips = ipsec_get_session(sa);
+	ips = ipsec_get_primary_session(sa);
 
 	if ((ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) ||
 			(ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)) {
@@ -234,7 +234,7 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 	RTE_ASSERT(m != NULL);
 	RTE_ASSERT(sa != NULL);
 
-	ips = ipsec_get_session(sa);
+	ips = ipsec_get_primary_session(sa);
 	ip_hdr_len = 0;
 
 	ip4 = rte_pktmbuf_mtod(m, struct ip *);
diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index f52ca5297..904ee65d9 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -189,6 +189,7 @@ static uint32_t mtu_size = RTE_ETHER_MTU;
 
 /* application wide librte_ipsec/SA parameters */
 struct app_sa_prm app_sa_prm = {.enable = 0};
+static const char *cfgfile;
 
 struct lcore_rx_queue {
 	uint16_t port_id;
@@ -1465,12 +1466,7 @@ parse_args(int32_t argc, char **argv)
 				print_usage(prgname);
 				return -1;
 			}
-			if (parse_cfg_file(optarg) < 0) {
-				printf("parsing file \"%s\" failed\n",
-					optarg);
-				print_usage(prgname);
-				return -1;
-			}
+			cfgfile = optarg;
 			f_present = 1;
 			break;
 		case 'j':
@@ -2418,6 +2414,14 @@ main(int32_t argc, char **argv)
 	if (ret < 0)
 		rte_exit(EXIT_FAILURE, "Invalid parameters\n");
 
+	/* parse configuration file */
+	if (parse_cfg_file(cfgfile) < 0) {
+		printf("parsing file \"%s\" failed\n",
+			optarg);
+		print_usage(argv[0]);
+		return -1;
+	}
+
 	if ((unprotected_port_mask & enabled_port_mask) !=
 			unprotected_port_mask)
 		rte_exit(EXIT_FAILURE, "Invalid unprotected portmask 0x%x\n",
diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index 5b63a13fd..d7761e966 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -430,7 +430,7 @@ enqueue_cop(struct cdev_qp *cqp, struct rte_crypto_op *cop)
 
 static inline void
 ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
-		struct rte_mbuf *pkts[], struct ipsec_sa *sas[],
+		struct rte_mbuf *pkts[], void *sas[],
 		uint16_t nb_pkts)
 {
 	int32_t ret = 0, i;
@@ -449,9 +449,9 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 		rte_prefetch0(pkts[i]);
 
 		priv = get_priv(pkts[i]);
-		sa = sas[i];
+		sa = ipsec_mask_saptr(sas[i]);
 		priv->sa = sa;
-		ips = ipsec_get_session(sa);
+		ips = ipsec_get_primary_session(sa);
 
 		switch (ips->type) {
 		case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL:
@@ -618,7 +618,7 @@ uint16_t
 ipsec_inbound(struct ipsec_ctx *ctx, struct rte_mbuf *pkts[],
 		uint16_t nb_pkts, uint16_t len)
 {
-	struct ipsec_sa *sas[nb_pkts];
+	void *sas[nb_pkts];
 
 	inbound_sa_lookup(ctx->sa_ctx, pkts, sas, nb_pkts);
 
@@ -638,7 +638,7 @@ uint16_t
 ipsec_outbound(struct ipsec_ctx *ctx, struct rte_mbuf *pkts[],
 		uint32_t sa_idx[], uint16_t nb_pkts, uint16_t len)
 {
-	struct ipsec_sa *sas[nb_pkts];
+	void *sas[nb_pkts];
 
 	outbound_sa_lookup(ctx->sa_ctx, sa_idx, sas, nb_pkts);
 
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index a4ad81b0e..8e075216c 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -81,12 +81,32 @@ struct app_sa_prm {
 
 extern struct app_sa_prm app_sa_prm;
 
+enum {
+	IPSEC_SESSION_PRIMARY = 0,
+	IPSEC_SESSION_FALLBACK = 1,
+	IPSEC_SESSION_MAX
+};
+
+#define IPSEC_SA_OFFLOAD_FALLBACK_FLAG (1)
+
+static inline struct ipsec_sa *
+ipsec_mask_saptr(void *ptr)
+{
+	uintptr_t i = (uintptr_t)ptr;
+	static const uintptr_t mask = IPSEC_SA_OFFLOAD_FALLBACK_FLAG;
+
+	i &= ~mask;
+
+	return (struct ipsec_sa *)i;
+}
+
 struct ipsec_sa {
-	struct rte_ipsec_session ips; /* one session per sa for now */
+	struct rte_ipsec_session sessions[IPSEC_SESSION_MAX];
 	uint32_t spi;
 	uint32_t cdev_id_qp;
 	uint64_t seq;
 	uint32_t salt;
+	uint32_t fallback_sessions;
 	enum rte_crypto_cipher_algorithm cipher_algo;
 	enum rte_crypto_auth_algorithm auth_algo;
 	enum rte_crypto_aead_algorithm aead_algo;
@@ -210,7 +230,7 @@ struct cnt_blk {
 struct traffic_type {
 	const uint8_t *data[MAX_PKT_BURST * 2];
 	struct rte_mbuf *pkts[MAX_PKT_BURST * 2];
-	struct ipsec_sa *saptr[MAX_PKT_BURST * 2];
+	void *saptr[MAX_PKT_BURST * 2];
 	uint32_t res[MAX_PKT_BURST * 2];
 	uint32_t num;
 };
@@ -278,16 +298,22 @@ get_sym_cop(struct rte_crypto_op *cop)
 }
 
 static inline struct rte_ipsec_session *
-ipsec_get_session(struct ipsec_sa *sa)
+ipsec_get_primary_session(struct ipsec_sa *sa)
+{
+	return &sa->sessions[IPSEC_SESSION_PRIMARY];
+}
+
+static inline struct rte_ipsec_session *
+ipsec_get_fallback_session(struct ipsec_sa *sa)
 {
-	return &sa->ips;
+	return &sa->sessions[IPSEC_SESSION_FALLBACK];
 }
 
 static inline enum rte_security_session_action_type
 ipsec_get_action_type(struct ipsec_sa *sa)
 {
 	struct rte_ipsec_session *ips;
-	ips = ipsec_get_session(sa);
+	ips = ipsec_get_primary_session(sa);
 	return ips->type;
 }
 
@@ -296,11 +322,11 @@ inbound_sa_check(struct sa_ctx *sa_ctx, struct rte_mbuf *m, uint32_t sa_idx);
 
 void
 inbound_sa_lookup(struct sa_ctx *sa_ctx, struct rte_mbuf *pkts[],
-		struct ipsec_sa *sa[], uint16_t nb_pkts);
+		void *sa[], uint16_t nb_pkts);
 
 void
 outbound_sa_lookup(struct sa_ctx *sa_ctx, uint32_t sa_idx[],
-		struct ipsec_sa *sa[], uint16_t nb_pkts);
+		void *sa[], uint16_t nb_pkts);
 
 void
 sp4_init(struct socket_ctx *ctx, int32_t socket_id);
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index 239d81ef6..2eb5c8b34 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -18,7 +18,6 @@
 	(((t) & RTE_IPSEC_SATP_IPV_MASK) == RTE_IPSEC_SATP_IPV4)) || \
 	((t) & RTE_IPSEC_SATP_MODE_MASK) == RTE_IPSEC_SATP_MODE_TUNLV4)
 
-
 /* helper routine to free bulk of packets */
 static inline void
 free_pkts(struct rte_mbuf *mb[], uint32_t n)
@@ -118,7 +117,7 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
  * group input packets byt the SA they belong to.
  */
 static uint32_t
-sa_group(struct ipsec_sa *sa_ptr[], struct rte_mbuf *pkts[],
+sa_group(void *sa_ptr[], struct rte_mbuf *pkts[],
 	struct rte_ipsec_group grp[], uint32_t num)
 {
 	uint32_t i, n, spi;
@@ -185,6 +184,37 @@ copy_to_trf(struct ipsec_traffic *trf, uint64_t satp, struct rte_mbuf *mb[],
 	out->num += num;
 }
 
+static uint32_t
+ipsec_prepare_crypto_group(struct ipsec_ctx *ctx, struct ipsec_sa *sa,
+		struct rte_ipsec_session *ips, struct rte_mbuf **m,
+		unsigned int cnt)
+{
+	struct cdev_qp *cqp;
+	struct rte_crypto_op *cop[cnt];
+	uint32_t j, k;
+	struct ipsec_mbuf_metadata *priv;
+
+	cqp = &ctx->tbl[sa->cdev_id_qp];
+
+	/* for that app each mbuf has it's own crypto op */
+	for (j = 0; j != cnt; j++) {
+		priv = get_priv(m[j]);
+		cop[j] = &priv->cop;
+		/*
+		 * this is just to satisfy inbound_sa_check()
+		 * should be removed in future.
+		 */
+		priv->sa = sa;
+	}
+
+	/* prepare and enqueue crypto ops */
+	k = rte_ipsec_pkt_crypto_prepare(ips, m, cop, cnt);
+	if (k != 0)
+		enqueue_cop_bulk(cqp, cop, k);
+
+	return k;
+}
+
 /*
  * Process ipsec packets.
  * If packet belong to SA that is subject of inline-crypto,
@@ -201,18 +231,15 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 	struct ipsec_mbuf_metadata *priv;
 	struct rte_ipsec_group *pg;
 	struct rte_ipsec_session *ips;
-	struct cdev_qp *cqp;
-	struct rte_crypto_op *cop[RTE_DIM(trf->ipsec.pkts)];
 	struct rte_ipsec_group grp[RTE_DIM(trf->ipsec.pkts)];
 
 	n = sa_group(trf->ipsec.saptr, trf->ipsec.pkts, grp, trf->ipsec.num);
 
 	for (i = 0; i != n; i++) {
-
 		pg = grp + i;
-		sa = pg->id.ptr;
+		sa = ipsec_mask_saptr(pg->id.ptr);
 
-		ips = ipsec_get_session(sa);
+		ips = ipsec_get_primary_session(sa);
 
 		/* no valid HW session for that SA, try to create one */
 		if (sa == NULL || (ips->crypto.ses == NULL &&
@@ -224,6 +251,7 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 				ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
 
+			/* get SA type */
 			satp = rte_ipsec_sa_type(ips->sa);
 
 			/*
@@ -236,30 +264,33 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 				priv->sa = sa;
 			}
 
-			k = rte_ipsec_pkt_process(ips, pg->m, pg->cnt);
-			copy_to_trf(trf, satp, pg->m, k);
+			/* fallback to cryptodev with RX packets which inline
+			 * processor was unable to process
+			 */
+			if (pg->id.val & IPSEC_SA_OFFLOAD_FALLBACK_FLAG) {
+				/* offload packets to cryptodev */
+				struct rte_ipsec_session *fallback;
+
+				fallback = ipsec_get_fallback_session(sa);
+				if (fallback->crypto.ses == NULL &&
+					fill_ipsec_session(fallback, ctx, sa)
+					!= 0)
+					k = 0;
+				else
+					k = ipsec_prepare_crypto_group(ctx, sa,
+						fallback, pg->m, pg->cnt);
+			} else {
+				/* finish processing of packets successfully
+				 * decrypted by an inline processor
+				 */
+				k = rte_ipsec_pkt_process(ips, pg->m, pg->cnt);
+				copy_to_trf(trf, satp, pg->m, k);
 
+			}
 		/* enqueue packets to crypto dev */
 		} else {
-
-			cqp = &ctx->tbl[sa->cdev_id_qp];
-
-			/* for that app each mbuf has it's own crypto op */
-			for (j = 0; j != pg->cnt; j++) {
-				priv = get_priv(pg->m[j]);
-				cop[j] = &priv->cop;
-				/*
-				 * this is just to satisfy inbound_sa_check()
-				 * should be removed in future.
-				 */
-				priv->sa = sa;
-			}
-
-			/* prepare and enqueue crypto ops */
-			k = rte_ipsec_pkt_crypto_prepare(ips, pg->m, cop,
+			k = ipsec_prepare_crypto_group(ctx, sa, ips, pg->m,
 				pg->cnt);
-			if (k != 0)
-				enqueue_cop_bulk(cqp, cop, k);
 		}
 
 		/* drop packets that cannot be enqueued/processed */
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index 52752e15f..4cb90857c 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -31,6 +31,8 @@
 
 #define IP6_FULL_MASK (sizeof(((struct ip_addr *)NULL)->ip.ip6.ip6) * CHAR_BIT)
 
+#define MBUF_NO_SEC_OFFLOAD(m) ((m->ol_flags & PKT_RX_SEC_OFFLOAD) == 0)
+
 struct supported_cipher_algo {
 	const char *keyword;
 	enum rte_crypto_cipher_algorithm algo;
@@ -235,6 +237,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	uint32_t mode_p = 0;
 	uint32_t type_p = 0;
 	uint32_t portid_p = 0;
+	uint32_t fallback_p = 0;
 
 	if (strcmp(tokens[0], "in") == 0) {
 		ri = &nb_sa_in;
@@ -245,6 +248,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 			return;
 
 		rule = &sa_in[*ri];
+		rule->direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS;
 	} else {
 		ri = &nb_sa_out;
 
@@ -254,6 +258,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 			return;
 
 		rule = &sa_out[*ri];
+		rule->direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS;
 	}
 
 	/* spi number */
@@ -263,7 +268,7 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 	if (atoi(tokens[1]) == INVALID_SPI)
 		return;
 	rule->spi = atoi(tokens[1]);
-	ips = ipsec_get_session(rule);
+	ips = ipsec_get_primary_session(rule);
 
 	for (ti = 2; ti < n_tokens; ti++) {
 		if (strcmp(tokens[ti], "mode") == 0) {
@@ -596,6 +601,45 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 			continue;
 		}
 
+		if (strcmp(tokens[ti], "fallback") == 0) {
+			struct rte_ipsec_session *fb;
+
+			APP_CHECK(app_sa_prm.enable, status, "Fallback session "
+				"not allowed for legacy mode.");
+			if (status->status < 0)
+				return;
+			APP_CHECK(ips->type ==
+				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO, status,
+				"Fallback session allowed if primary session "
+				"is of type inline-crypto-offload only.");
+			if (status->status < 0)
+				return;
+			APP_CHECK(rule->direction ==
+				RTE_SECURITY_IPSEC_SA_DIR_INGRESS, status,
+				"Fallback session not allowed for egress "
+				"rule");
+			if (status->status < 0)
+				return;
+			APP_CHECK_PRESENCE(fallback_p, tokens[ti], status);
+			if (status->status < 0)
+				return;
+			INCREMENT_TOKEN_INDEX(ti, n_tokens, status);
+			if (status->status < 0)
+				return;
+			fb = ipsec_get_fallback_session(rule);
+			if (strcmp(tokens[ti], "lookaside-none") == 0) {
+				fb->type = RTE_SECURITY_ACTION_TYPE_NONE;
+			} else {
+				APP_CHECK(0, status, "unrecognized fallback "
+					"type %s.", tokens[ti]);
+				return;
+			}
+
+			rule->fallback_sessions = 1;
+			fallback_p = 1;
+			continue;
+		}
+
 		/* unrecognizeable input */
 		APP_CHECK(0, status, "unrecognized input \"%s\"",
 			tokens[ti]);
@@ -643,6 +687,7 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 	uint32_t i;
 	uint8_t a, b, c, d;
 	const struct rte_ipsec_session *ips;
+	const struct rte_ipsec_session *fallback_ips;
 
 	printf("\tspi_%s(%3u):", inbound?"in":"out", sa->spi);
 
@@ -699,7 +744,7 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 		break;
 	}
 
-	ips = &sa->ips;
+	ips = &sa->sessions[IPSEC_SESSION_PRIMARY];
 	printf(" type:");
 	switch (ips->type) {
 	case RTE_SECURITY_ACTION_TYPE_NONE:
@@ -715,6 +760,15 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 		printf("lookaside-protocol-offload ");
 		break;
 	}
+
+	fallback_ips = &sa->sessions[IPSEC_SESSION_FALLBACK];
+	if (fallback_ips != NULL && sa->fallback_sessions > 0) {
+		printf("inline fallback: ");
+		if (fallback_ips->type == RTE_SECURITY_ACTION_TYPE_NONE)
+			printf("lookaside-none");
+		else
+			printf("invalid");
+	}
 	printf("\n");
 }
 
@@ -904,7 +958,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 		}
 		*sa = entries[i];
 		sa->seq = 0;
-		ips = ipsec_get_session(sa);
+		ips = ipsec_get_primary_session(sa);
 
 		if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
 			ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
@@ -912,9 +966,6 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 				return -EINVAL;
 		}
 
-		sa->direction = (inbound == 1) ?
-				RTE_SECURITY_IPSEC_SA_DIR_INGRESS :
-				RTE_SECURITY_IPSEC_SA_DIR_EGRESS;
 
 		switch (WITHOUT_TRANSPORT_VERSION(sa->flags)) {
 		case IP4_TUNNEL:
@@ -954,7 +1005,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			sa->xforms = &sa_ctx->xf[idx].a;
 
-			ips = ipsec_get_session(sa);
+			ips = ipsec_get_primary_session(sa);
 			if (ips->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
 				ips->type ==
@@ -1168,9 +1219,15 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	/* init processing session */
-	ips = ipsec_get_session(lsa);
+	/* init primary processing session */
+	ips = ipsec_get_primary_session(lsa);
 	rc = fill_ipsec_session(ips, sa);
+	if (rc != 0)
+		return rc;
+
+	/* init inline fallback processing session */
+	if (lsa->fallback_sessions == 1)
+		rc = fill_ipsec_session(ipsec_get_fallback_session(lsa), sa);
 
 	return rc;
 }
@@ -1326,13 +1383,14 @@ inbound_sa_check(struct sa_ctx *sa_ctx, struct rte_mbuf *m, uint32_t sa_idx)
 
 static inline void
 single_inbound_lookup(struct ipsec_sa *sadb, struct rte_mbuf *pkt,
-		struct ipsec_sa **sa_ret)
+		void **sa_ret)
 {
 	struct rte_esp_hdr *esp;
 	struct ip *ip;
 	uint32_t *src4_addr;
 	uint8_t *src6_addr;
 	struct ipsec_sa *sa;
+	void *result_sa;
 
 	*sa_ret = NULL;
 
@@ -1342,33 +1400,48 @@ single_inbound_lookup(struct ipsec_sa *sadb, struct rte_mbuf *pkt,
 	if (esp->spi == INVALID_SPI)
 		return;
 
-	sa = &sadb[SPI2IDX(rte_be_to_cpu_32(esp->spi))];
+	result_sa = sa = &sadb[SPI2IDX(rte_be_to_cpu_32(esp->spi))];
 	if (rte_be_to_cpu_32(esp->spi) != sa->spi)
 		return;
 
+	/*
+	 * Mark need for inline offload fallback on the LSB of SA pointer.
+	 * Thanks to packet grouping mechanism which ipsec_process is using
+	 * packets marked for fallback processing will form separate group.
+	 *
+	 * Because it is not safe to use SA pointer it is casted to generic
+	 * pointer to prevent from unintentional use. Use ipsec_mask_saptr
+	 * to get valid struct pointer.
+	 */
+	if (MBUF_NO_SEC_OFFLOAD(pkt) && sa->fallback_sessions > 0) {
+		uintptr_t intsa = (uintptr_t)sa;
+		intsa |= IPSEC_SA_OFFLOAD_FALLBACK_FLAG;
+		result_sa = (void *)intsa;
+	}
+
 	switch (WITHOUT_TRANSPORT_VERSION(sa->flags)) {
 	case IP4_TUNNEL:
 		src4_addr = RTE_PTR_ADD(ip, offsetof(struct ip, ip_src));
 		if ((ip->ip_v == IPVERSION) &&
 				(sa->src.ip.ip4 == *src4_addr) &&
 				(sa->dst.ip.ip4 == *(src4_addr + 1)))
-			*sa_ret = sa;
+			*sa_ret = result_sa;
 		break;
 	case IP6_TUNNEL:
 		src6_addr = RTE_PTR_ADD(ip, offsetof(struct ip6_hdr, ip6_src));
 		if ((ip->ip_v == IP6_VERSION) &&
 				!memcmp(&sa->src.ip.ip6.ip6, src6_addr, 16) &&
 				!memcmp(&sa->dst.ip.ip6.ip6, src6_addr + 16, 16))
-			*sa_ret = sa;
+			*sa_ret = result_sa;
 		break;
 	case TRANSPORT:
-		*sa_ret = sa;
+		*sa_ret = result_sa;
 	}
 }
 
 void
 inbound_sa_lookup(struct sa_ctx *sa_ctx, struct rte_mbuf *pkts[],
-		struct ipsec_sa *sa[], uint16_t nb_pkts)
+		void *sa[], uint16_t nb_pkts)
 {
 	uint32_t i;
 
@@ -1378,7 +1451,7 @@ inbound_sa_lookup(struct sa_ctx *sa_ctx, struct rte_mbuf *pkts[],
 
 void
 outbound_sa_lookup(struct sa_ctx *sa_ctx, uint32_t sa_idx[],
-		struct ipsec_sa *sa[], uint16_t nb_pkts)
+		void *sa[], uint16_t nb_pkts)
 {
 	uint32_t i;
 
-- 
2.17.1

[dpdk-dev] [PATCH v7 3/4] examples/ipsec-secgw: add frag TTL cmdline option

[Marcin Smoczynski]

Due to fragment loss on highly saturated links and long fragment
lifetime, ipsec-secgw application quickly runs out of free reassembly
buckets. As a result new fragments are being dropped.

Introduce --frag-ttl option which allow user to lower default fragment
lifitime which solves problem of saturated reassembly buckets with high
bandwidth fragmented traffic.

Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
Tested-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
---
 doc/guides/sample_app_ug/ipsec_secgw.rst |  7 +++++
 examples/ipsec-secgw/ipsec-secgw.c       | 40 ++++++++++++++++++------
 2 files changed, 37 insertions(+), 10 deletions(-)

diff --git a/doc/guides/sample_app_ug/ipsec_secgw.rst b/doc/guides/sample_app_ug/ipsec_secgw.rst
index cb6fadb35..c1e8a0bda 100644
--- a/doc/guides/sample_app_ug/ipsec_secgw.rst
+++ b/doc/guides/sample_app_ug/ipsec_secgw.rst
@@ -154,6 +154,13 @@ Where:
     Incoming packets with length bigger then MTU will be discarded.
     Default value: 1500.
 
+*   ``--frag-ttl FRAG_TTL_NS``: fragment lifetime (in nanoseconds).
+    If packet is not reassembled within this time, received fragments
+    will be discarded. Fragment lifetime should be decreased when
+    there is a high fragmented traffic loss in high bandwidth networks.
+    Should be lower for for low number of reassembly buckets.
+    Valid values: from 1 ns to 10 s. Default value: 10000000 (10 s).
+
 *   ``--reassemble NUM``: max number of entries in reassemble fragment table.
     Zero value disables reassembly functionality.
     Default value: 0.
diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index 904ee65d9..b12936470 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -112,7 +112,7 @@ static uint16_t nb_txd = IPSEC_SECGW_TX_DESC_DEFAULT;
 		0, 0)
 
 #define	FRAG_TBL_BUCKET_ENTRIES	4
-#define	FRAG_TTL_MS		(10 * MS_PER_S)
+#define	MAX_FRAG_TTL_NS		(10LL * NS_PER_S)
 
 #define MTU_TO_FRAMELEN(x)	((x) + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN)
 
@@ -135,6 +135,7 @@ struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS] = {
 #define CMD_LINE_OPT_TX_OFFLOAD		"txoffload"
 #define CMD_LINE_OPT_REASSEMBLE		"reassemble"
 #define CMD_LINE_OPT_MTU		"mtu"
+#define CMD_LINE_OPT_FRAG_TTL		"frag-ttl"
 
 enum {
 	/* long options mapped to a short option */
@@ -150,6 +151,7 @@ enum {
 	CMD_LINE_OPT_TX_OFFLOAD_NUM,
 	CMD_LINE_OPT_REASSEMBLE_NUM,
 	CMD_LINE_OPT_MTU_NUM,
+	CMD_LINE_OPT_FRAG_TTL_NUM,
 };
 
 static const struct option lgopts[] = {
@@ -160,6 +162,7 @@ static const struct option lgopts[] = {
 	{CMD_LINE_OPT_TX_OFFLOAD, 1, 0, CMD_LINE_OPT_TX_OFFLOAD_NUM},
 	{CMD_LINE_OPT_REASSEMBLE, 1, 0, CMD_LINE_OPT_REASSEMBLE_NUM},
 	{CMD_LINE_OPT_MTU, 1, 0, CMD_LINE_OPT_MTU_NUM},
+	{CMD_LINE_OPT_FRAG_TTL, 1, 0, CMD_LINE_OPT_FRAG_TTL_NUM},
 	{NULL, 0, 0, 0}
 };
 
@@ -186,6 +189,7 @@ static uint64_t dev_tx_offload = UINT64_MAX;
 static uint32_t frag_tbl_sz;
 static uint32_t frame_buf_size = RTE_MBUF_DEFAULT_BUF_SIZE;
 static uint32_t mtu_size = RTE_ETHER_MTU;
+static uint64_t frag_ttl_ns = MAX_FRAG_TTL_NS;
 
 /* application wide librte_ipsec/SA parameters */
 struct app_sa_prm app_sa_prm = {.enable = 0};
@@ -1305,6 +1309,9 @@ print_usage(const char *prgname)
 		": MTU value on all ports (default value: 1500)\n"
 		"    outgoing packets with bigger size will be fragmented\n"
 		"    incoming packets with bigger size will be discarded\n"
+		"  --" CMD_LINE_OPT_FRAG_TTL " FRAG_TTL_NS"
+		": fragments lifetime in nanoseconds, default\n"
+		"    and maximum value is 10.000.000.000 ns (10 s)\n"
 		"\n",
 		prgname);
 }
@@ -1341,14 +1348,15 @@ parse_portmask(const char *portmask)
 	return pm;
 }
 
-static int32_t
+static int64_t
 parse_decimal(const char *str)
 {
 	char *end = NULL;
-	unsigned long num;
+	uint64_t num;
 
-	num = strtoul(str, &end, 10);
-	if ((str[0] == '\0') || (end == NULL) || (*end != '\0'))
+	num = strtoull(str, &end, 10);
+	if ((str[0] == '\0') || (end == NULL) || (*end != '\0')
+		|| num > INT64_MAX)
 		return -1;
 
 	return num;
@@ -1422,12 +1430,14 @@ print_app_sa_prm(const struct app_sa_prm *prm)
 	printf("replay window size: %u\n", prm->window_size);
 	printf("ESN: %s\n", (prm->enable_esn == 0) ? "disabled" : "enabled");
 	printf("SA flags: %#" PRIx64 "\n", prm->flags);
+	printf("Frag TTL: %" PRIu64 " ns\n", frag_ttl_ns);
 }
 
 static int32_t
 parse_args(int32_t argc, char **argv)
 {
-	int32_t opt, ret;
+	int opt;
+	int64_t ret;
 	char **argvopt;
 	int32_t option_index;
 	char *prgname = argv[0];
@@ -1506,7 +1516,7 @@ parse_args(int32_t argc, char **argv)
 			break;
 		case CMD_LINE_OPT_SINGLE_SA_NUM:
 			ret = parse_decimal(optarg);
-			if (ret == -1) {
+			if (ret == -1 || ret > UINT32_MAX) {
 				printf("Invalid argument[sa_idx]\n");
 				print_usage(prgname);
 				return -1;
@@ -1549,7 +1559,7 @@ parse_args(int32_t argc, char **argv)
 			break;
 		case CMD_LINE_OPT_REASSEMBLE_NUM:
 			ret = parse_decimal(optarg);
-			if (ret < 0) {
+			if (ret < 0 || ret > UINT32_MAX) {
 				printf("Invalid argument for \'%s\': %s\n",
 					CMD_LINE_OPT_REASSEMBLE, optarg);
 				print_usage(prgname);
@@ -1567,6 +1577,16 @@ parse_args(int32_t argc, char **argv)
 			}
 			mtu_size = ret;
 			break;
+		case CMD_LINE_OPT_FRAG_TTL_NUM:
+			ret = parse_decimal(optarg);
+			if (ret < 0 || ret > MAX_FRAG_TTL_NS) {
+				printf("Invalid argument for \'%s\': %s\n",
+					CMD_LINE_OPT_MTU, optarg);
+				print_usage(prgname);
+				return -1;
+			}
+			frag_ttl_ns = ret;
+			break;
 		default:
 			print_usage(prgname);
 			return -1;
@@ -2344,8 +2364,8 @@ reassemble_lcore_init(struct lcore_conf *lc, uint32_t cid)
 
 	/* create fragment table */
 	sid = rte_lcore_to_socket_id(cid);
-	frag_cycles = (rte_get_tsc_hz() + MS_PER_S - 1) /
-		MS_PER_S * FRAG_TTL_MS;
+	frag_cycles = (rte_get_tsc_hz() + NS_PER_S - 1) /
+		NS_PER_S * frag_ttl_ns;
 
 	lc->frag.tbl = rte_ip_frag_table_create(frag_tbl_sz,
 		FRAG_TBL_BUCKET_ENTRIES, frag_tbl_sz, frag_cycles, sid);
-- 
2.17.1

[dpdk-dev] [PATCH v7 4/4] examples/ipsec-secgw: add offload fallback tests

[Marcin Smoczynski]

Add tests for offload fallback feature; add inbound config modificator
SGW_CFG_XPRM_IN (offload fallback setting can be set only for inbound
SAs). Tests are using cryptodev for outbound SA.

To test fragmentation with QAT set:
MULTI_SEG_TEST="--reassemble=4096 --cryptodev_mask=0x5555"

Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
Tested-by: Bernard Iremonger <bernard.iremonger@intel.com>
Signed-off-by: Marcin Smoczynski <marcinx.smoczynski@intel.com>
---
 doc/guides/sample_app_ug/ipsec_secgw.rst                    | 5 +++++
 examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh         | 4 ++--
 .../test/trs_aesgcm_inline_crypto_fallback_defs.sh          | 5 +++++
 examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh         | 6 ++++--
 .../test/tun_aesgcm_inline_crypto_fallback_defs.sh          | 5 +++++
 5 files changed, 21 insertions(+), 4 deletions(-)
 create mode 100644 examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
 create mode 100644 examples/ipsec-secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh

diff --git a/doc/guides/sample_app_ug/ipsec_secgw.rst b/doc/guides/sample_app_ug/ipsec_secgw.rst
index c1e8a0bda..ae8cce235 100644
--- a/doc/guides/sample_app_ug/ipsec_secgw.rst
+++ b/doc/guides/sample_app_ug/ipsec_secgw.rst
@@ -753,6 +753,11 @@ Also the user can optionally setup:
 *   ``CRYPTO_DEV``: crypto device to be used ('-w <pci-id>'). If none specified
     appropriate vdevs will be created by the script
 
+*   ``MULTI_SEG_TEST``: ipsec-secgw option to enable reassembly support and
+    specify size of reassembly table (e.g.
+    ``MULTI_SEG_TEST='--reassemble 128'``). This option must be set for
+    fallback session tests.
+
 Note that most of the tests require the appropriate crypto PMD/device to be
 available.
 
diff --git a/examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh b/examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh
index f6c5bf5a7..17f2f86d2 100644
--- a/examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh
+++ b/examples/ipsec-secgw/test/trs_aesgcm_common_defs.sh
@@ -29,11 +29,11 @@ sp ipv6 out esp bypass pri 1 sport 0:65535 dport 0:65535
 #SA in rules
 sa in 7 aead_algo aes-128-gcm \
 aead_key de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef \
-mode transport ${SGW_CFG_XPRM}
+mode transport ${SGW_CFG_XPRM} ${SGW_CFG_XPRM_IN}
 
 sa in 9 aead_algo aes-128-gcm \
 aead_key de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef \
-mode transport ${SGW_CFG_XPRM}
+mode transport ${SGW_CFG_XPRM} ${SGW_CFG_XPRM_IN}
 
 #SA out rules
 sa out 7 aead_algo aes-128-gcm \
diff --git a/examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh b/examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
new file mode 100644
index 000000000..875a7457d
--- /dev/null
+++ b/examples/ipsec-secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
@@ -0,0 +1,5 @@
+#! /bin/bash
+
+. ${DIR}/trs_aesgcm_defs.sh
+
+SGW_CFG_XPRM_IN='port_id 0 type inline-crypto-offload fallback lookaside-none'
diff --git a/examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh b/examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh
index 278377967..7490baded 100644
--- a/examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh
+++ b/examples/ipsec-secgw/test/tun_aesgcm_common_defs.sh
@@ -29,11 +29,13 @@ sp ipv6 out esp bypass pri 1 sport 0:65535 dport 0:65535
 #SA in rules
 sa in 7 aead_algo aes-128-gcm \
 aead_key de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef \
-mode ipv4-tunnel src ${REMOTE_IPV4} dst ${LOCAL_IPV4} ${SGW_CFG_XPRM}
+mode ipv4-tunnel src ${REMOTE_IPV4} dst ${LOCAL_IPV4} ${SGW_CFG_XPRM} \
+${SGW_CFG_XPRM_IN}
 
 sa in 9 aead_algo aes-128-gcm \
 aead_key de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef \
-mode ipv6-tunnel src ${REMOTE_IPV6} dst ${LOCAL_IPV6} ${SGW_CFG_XPRM}
+mode ipv6-tunnel src ${REMOTE_IPV6} dst ${LOCAL_IPV6} ${SGW_CFG_XPRM} \
+${SGW_CFG_XPRM_IN}
 
 #SA out rules
 sa out 7 aead_algo aes-128-gcm \
diff --git a/examples/ipsec-secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh b/examples/ipsec-secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh
new file mode 100644
index 000000000..696848432
--- /dev/null
+++ b/examples/ipsec-secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh
@@ -0,0 +1,5 @@
+#! /bin/bash
+
+. ${DIR}/tun_aesgcm_defs.sh
+
+SGW_CFG_XPRM_IN='port_id 0 type inline-crypto-offload fallback lookaside-none'
-- 
2.17.1

Re: [dpdk-dev] [PATCH v7 0/4] add fallback session

[Mcnamara, John]

> -----Original Message-----
> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Marcin Smoczynski
> Sent: Monday, October 14, 2019 2:49 PM
> To: anoobj@marvell.com; akhil.goyal@nxp.com; Ananyev, Konstantin
> <konstantin.ananyev@intel.com>
> Cc: dev@dpdk.org; Smoczynski, MarcinX <marcinx.smoczynski@intel.com>
> Subject: [dpdk-dev] [PATCH v7 0/4] add fallback session
> 
> Add fallback session feature allowing to process packets that inline
> processor is unable to handle (e.g. fragmented traffic). Processing
> takes place in a secondary session defined for SA in a configuration
> file.
> 
> This feature is limited to ingress IPsec traffic only. IPsec anti-
> replay window and ESN are supported in conjunction with fallback
> session when following conditions are met:
>  * primary session is 'inline-crypto-offload,
>  * fallback sessions is 'lookaside-none'.
> Due to different processing times of inline and lookaside modes,
> fallback session introduces some packet reordering, therefore when
> using with IPsec window, its value should be increased.
> 
> v6 to v7 changes:
>  - remove partial support for "*-protocol" fall-back session types
>  - rebase on master (31b798a)


Hi Akhil,

What is the current status of this patchset. Will it be merged for RC1?

John

Re: [dpdk-dev] [PATCH v7 0/4] add fallback session

[Akhil Goyal]

> >
> > Add fallback session feature allowing to process packets that inline
> > processor is unable to handle (e.g. fragmented traffic). Processing
> > takes place in a secondary session defined for SA in a configuration
> > file.
> >
> > This feature is limited to ingress IPsec traffic only. IPsec anti-
> > replay window and ESN are supported in conjunction with fallback
> > session when following conditions are met:
> >  * primary session is 'inline-crypto-offload,
> >  * fallback sessions is 'lookaside-none'.
> > Due to different processing times of inline and lookaside modes,
> > fallback session introduces some packet reordering, therefore when
> > using with IPsec window, its value should be increased.
> >
> > v6 to v7 changes:
> >  - remove partial support for "*-protocol" fall-back session types
> >  - rebase on master (31b798a)
> 
Hi John,
> 
> Hi Akhil,
> 
> What is the current status of this patchset. Will it be merged for RC1?

Anoob had some issues in the patchset and I have not done thorough review of this patchset.
So will be pushing it for RC2.

-Akhil

Re: [dpdk-dev] [PATCH v7 0/4] add fallback session

[Akhil Goyal]

> 
> Add fallback session feature allowing to process packets that inline
> processor is unable to handle (e.g. fragmented traffic). Processing
> takes place in a secondary session defined for SA in a configuration
> file.
> 
> This feature is limited to ingress IPsec traffic only. IPsec
> anti-replay window and ESN are supported in conjunction with fallback
> session when following conditions are met:
>  * primary session is 'inline-crypto-offload,
>  * fallback sessions is 'lookaside-none'.
> Due to different processing times of inline and lookaside modes,
> fallback session introduces some packet reordering, therefore when
> using with IPsec window, its value should be increased.
> 
> v6 to v7 changes:
>  - remove partial support for "*-protocol" fall-back session types
>  - rebase on master (31b798a)
> 
> v5 to v6 changes:
>  - add sanity check: fail to parse SA configuration if fallback session
>    is configured but primary session is not inline crypto
>  - update documentation for ipsec-secgw tests (MULTI_SEG_TESTS var
>    described)
>  - add release notes
>  - minor commit log changes
> 
> v4 to v5 changes:
>  - fix build errors related to frag TTL command line option and
>    parse_decimal method
> 
> v3 to v4 changes:
>  - add info about packet reordering to the documentation regarding
>    fallback session
>  - add patch with --frag-ttl command line option which allows to change
>    fragment lifetime
> 
> v2 to v3 changes:
>  - doc and commit log update - explicitly state feature limitations
> 
> v1 to v2 changes:
>  - disable fallback offload for outbound SAs
>  - add test scripts
> 
> Marcin Smoczynski (4):
>   examples/ipsec-secgw: sa structure cleanup
>   examples/ipsec-secgw: add fallback session feature
>   examples/ipsec-secgw: add frag TTL cmdline option
>   examples/ipsec-secgw: add offload fallback tests
> 
>  doc/guides/rel_notes/release_19_11.rst        |   8 +
>  doc/guides/sample_app_ug/ipsec_secgw.rst      |  34 +++-
>  examples/ipsec-secgw/esp.c                    |  35 ++--
>  examples/ipsec-secgw/ipsec-secgw.c            |  56 ++++--
>  examples/ipsec-secgw/ipsec.c                  | 101 +++++------
>  examples/ipsec-secgw/ipsec.h                  |  61 +++++--
>  examples/ipsec-secgw/ipsec_process.c          | 113 ++++++++-----
>  examples/ipsec-secgw/sa.c                     | 159 +++++++++++++-----
>  .../test/trs_aesgcm_common_defs.sh            |   4 +-
>  .../trs_aesgcm_inline_crypto_fallback_defs.sh |   5 +
>  .../test/tun_aesgcm_common_defs.sh            |   6 +-
>  .../tun_aesgcm_inline_crypto_fallback_defs.sh |   5 +
>  12 files changed, 409 insertions(+), 178 deletions(-)
>  create mode 100644 examples/ipsec-
> secgw/test/trs_aesgcm_inline_crypto_fallback_defs.sh
>  create mode 100644 examples/ipsec-
> secgw/test/tun_aesgcm_inline_crypto_fallback_defs.sh
> 
> --
> 2.17.1
Series Acked-by: Akhil Goyal <akhil.goyal@nxp.com>

Applied to dpdk-next-crypto
Release notes update merged with previous entry of IPsec changes.

Thanks.