[dpdk-dev] [PATCH 0/6] examples/ipsec-secgw: fix 1st pkt dropped

[dpdk-dev] [PATCH 0/6] examples/ipsec-secgw: fix 1st pkt dropped

[Bernard Iremonger]

This patchset fixes the issue of the first inbound packet
being dropped for inline crypto. It also improves the debug 
output in esp.c, sa.c and ipsec-secgw.c 

Bernard Iremonger (6):
  examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
  examples/ipsec-secgw: fix 1st packet dropped patch two
  examples/ipsec-secgw: fix 1st packet dropped patch three
  examples/ipsec-secgw: fix debug in esp.c
  examples/ipsec-secgw: fix debug in sa.c
  examples/ipsec-secgw: fix debug in ipsec-secgw.c

 examples/ipsec-secgw/esp.c           |   5 +-
 examples/ipsec-secgw/ipsec-secgw.c   | 252 ++++++++++++++++++-----------------
 examples/ipsec-secgw/ipsec.c         | 123 +++++++++++------
 examples/ipsec-secgw/ipsec.h         |   5 +-
 examples/ipsec-secgw/ipsec_process.c |   9 +-
 examples/ipsec-secgw/sa.c            |  67 ++++++++--
 6 files changed, 274 insertions(+), 187 deletions(-)

-- 
2.7.4

[dpdk-dev] [PATCH 1/6] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Bernard Iremonger]

Refactor create_session() into create_inline_session() and
create_lookaside_session() in ipsec.c
Use socket_ctx in create_inline_session()

Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
Cc: stable@dpdk.org
Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/ipsec.c         | 123 +++++++++++++++++++++++------------
 examples/ipsec-secgw/ipsec.h         |   5 +-
 examples/ipsec-secgw/ipsec_process.c |   9 +--
 3 files changed, 90 insertions(+), 47 deletions(-)

diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index 4352cb8..7090dbe 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -40,7 +40,7 @@ set_ipsec_conf(struct ipsec_sa *sa, struct rte_security_ipsec_xform *ipsec)
 }
 
 int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 {
 	struct rte_cryptodev_info cdev_info;
 	unsigned long cdev_id_qp = 0;
@@ -108,23 +108,82 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 				"SEC Session init failed: err: %d\n", ret);
 				return -1;
 			}
-		} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		} else if ((sa->type ==
+				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) ||
+				(sa->type ==
+				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)) {
+					RTE_LOG(ERR, IPSEC,
+						"Inline not supported\n");
+					return -1;
+		}
+	} else {
+		sa->crypto_session = rte_cryptodev_sym_session_create(
+				ipsec_ctx->session_pool);
+		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
+				sa->crypto_session, sa->xforms,
+				ipsec_ctx->session_priv_pool);
+
+		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
+				&cdev_info);
+	}
+
+	sa->cdev_id_qp = cdev_id_qp;
+
+	return 0;
+}
+
+int
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
+{
+	unsigned long cdev_id_qp = 0;
+	int32_t ret = 0;
+	struct rte_security_ctx *sec_ctx;
+
+	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on port %u\n",
+		sa->spi, sa->portid);
+
+	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
+		struct rte_security_session_conf sess_conf = {
+			.action_type = sa->type,
+			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
+			{.ipsec = {
+				.spi = sa->spi,
+				.salt = sa->salt,
+				.options = { 0 },
+				.direction = sa->direction,
+				.proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
+				.mode = (sa->flags == IP4_TUNNEL ||
+						sa->flags == IP6_TUNNEL) ?
+					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL :
+					RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
+			} },
+			.crypto_xform = sa->xforms,
+			.userdata = NULL,
+		};
+
+		if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
 			struct rte_flow_error err;
-			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
-							rte_eth_dev_get_sec_ctx(
-							sa->portid);
 			const struct rte_security_capability *sec_cap;
 			int ret = 0;
 
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
+			sec_ctx = (struct rte_security_ctx *)
+							rte_eth_dev_get_sec_ctx(
+							sa->portid);
+			if (sec_ctx == NULL) {
+				RTE_LOG(ERR, IPSEC,
+				" rte_eth_dev_get_sec_ctx failed\n");
+				return -1;
+			}
+
+			sa->sec_session = rte_security_session_create(sec_ctx,
+					&sess_conf, skt_ctx->session_pool);
 			if (sa->sec_session == NULL) {
 				RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 				return -1;
 			}
 
-			sec_cap = rte_security_capabilities_get(ctx);
+			sec_cap = rte_security_capabilities_get(sec_ctx);
 
 			/* iterate until ESP tunnel*/
 			while (sec_cap->action !=
@@ -147,7 +206,7 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			}
 
 			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
+			sa->security_ctx = sec_ctx;
 			sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
 
 			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
@@ -196,7 +255,7 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 				/* Try RSS. */
 				sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
 				sa->action[1].conf = &action_rss;
-				eth_dev = ctx->device;
+				eth_dev = sec_ctx->device;
 				rte_eth_dev_rss_hash_conf_get(sa->portid,
 							      &rss_conf);
 				for (i = 0, j = 0;
@@ -252,12 +311,12 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			}
 		} else if (sa->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
-			struct rte_security_ctx *ctx =
-					(struct rte_security_ctx *)
-					rte_eth_dev_get_sec_ctx(sa->portid);
 			const struct rte_security_capability *sec_cap;
 
-			if (ctx == NULL) {
+			sec_ctx = (struct rte_security_ctx *)
+					rte_eth_dev_get_sec_ctx(sa->portid);
+
+			if (sec_ctx == NULL) {
 				RTE_LOG(ERR, IPSEC,
 				"Ethernet device doesn't have security features registered\n");
 				return -1;
@@ -279,15 +338,15 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 
 			sess_conf.userdata = (void *) sa;
 
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
+			sa->sec_session = rte_security_session_create(sec_ctx,
+					&sess_conf, skt_ctx->session_pool);
 			if (sa->sec_session == NULL) {
 				RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 				return -1;
 			}
 
-			sec_cap = rte_security_capabilities_get(ctx);
+			sec_cap = rte_security_capabilities_get(sec_ctx);
 
 			if (sec_cap == NULL) {
 				RTE_LOG(ERR, IPSEC,
@@ -316,17 +375,8 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			}
 
 			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
+			sa->security_ctx = sec_ctx;
 		}
-	} else {
-		sa->crypto_session = rte_cryptodev_sym_session_create(
-				ipsec_ctx->session_pool);
-		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
-				sa->crypto_session, sa->xforms,
-				ipsec_ctx->session_priv_pool);
-
-		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
-				&cdev_info);
 	}
 	sa->cdev_id_qp = cdev_id_qp;
 
@@ -395,7 +445,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			rte_prefetch0(&priv->sym_cop);
 
 			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
+				create_lookaside_session(ipsec_ctx, sa)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -414,7 +464,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			rte_prefetch0(&priv->sym_cop);
 
 			if ((unlikely(sa->crypto_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
+				create_lookaside_session(ipsec_ctx, sa)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -429,12 +479,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 			break;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
+			RTE_ASSERT(sa->sec_session != NULL);
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
 			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
@@ -442,17 +487,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 						sa->sec_session, pkts[i], NULL);
 			continue;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
+			RTE_ASSERT(sa->sec_session != NULL);
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			rte_security_attach_session(&priv->cop,
 					sa->sec_session);
 
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index 99f49d6..db5a5c1 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -306,6 +306,9 @@ void
 enqueue_cop_burst(struct cdev_qp *cqp);
 
 int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+
+int
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa);
 
 #endif /* __IPSEC_H__ */
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index e403c46..152a684 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -95,22 +95,23 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
 	/* setup crypto section */
 	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
 		if (sa->crypto_session == NULL) {
-			rc = create_session(ctx, sa);
+			rc = create_lookaside_session(ctx, sa);
 			if (rc != 0)
 				return rc;
 		}
 		ss->crypto.ses = sa->crypto_session;
 	/* setup session action type */
-	} else {
+	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
 		if (sa->sec_session == NULL) {
-			rc = create_session(ctx, sa);
+			rc = create_lookaside_session(ctx, sa);
 			if (rc != 0)
 				return rc;
 		}
 		ss->security.ses = sa->sec_session;
 		ss->security.ctx = sa->security_ctx;
 		ss->security.ol_flags = sa->ol_flags;
-	}
+	} else
+		RTE_ASSERT(0);
 
 	rc = rte_ipsec_session_prepare(ss);
 	if (rc != 0)
-- 
2.7.4

[dpdk-dev] [PATCH 2/6] examples/ipsec-secgw: fix 1st packet dropped patch two

[Bernard Iremonger]

Call create_inline_session() at initialisition in sa.c
Call rte_ipsec_session_prepare() in fill_ipsec_session() for inline.

Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
Cc: stable@dpdk.org
Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/sa.c | 46 ++++++++++++++++++++++++++++++++++++----------
 1 file changed, 36 insertions(+), 10 deletions(-)

diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index 414fcd2..7fb1929 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -762,11 +762,13 @@ check_eth_dev_caps(uint16_t portid, uint32_t inbound)
 
 static int
 sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries, uint32_t inbound)
+		uint32_t nb_entries, uint32_t inbound,
+		struct socket_ctx *skt_ctx)
 {
 	struct ipsec_sa *sa;
 	uint32_t i, idx;
 	uint16_t iv_length, aad_length;
+	int32_t rc;
 
 	/* for ESN upper 32 bits of SQN also need to be part of AAD */
 	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
@@ -819,6 +821,17 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			sa->xforms = &sa_ctx->xf[idx].a;
 
+			if (sa->type ==
+				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
+				sa->type ==
+				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+				rc = create_inline_session(skt_ctx, sa);
+				if (rc != 0) {
+					RTE_LOG(ERR, IPSEC_ESP,
+						"create_inline_session() failed\n");
+					return -EINVAL;
+				}
+			}
 			print_one_sa_rule(sa, inbound);
 		} else {
 			switch (sa->cipher_algo) {
@@ -894,16 +907,16 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 static inline int
 sa_out_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 0);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx);
 }
 
 static inline int
 sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 1);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx);
 }
 
 /*
@@ -997,10 +1010,12 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm, const struct ipsec_sa *ss,
 	return 0;
 }
 
-static void
+static int
 fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 	const struct ipsec_sa *lsa)
 {
+	int32_t rc = 0;
+
 	ss->sa = sa;
 	ss->type = lsa->type;
 
@@ -1013,6 +1028,17 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 		ss->security.ctx = lsa->security_ctx;
 		ss->security.ol_flags = lsa->ol_flags;
 	}
+
+	if (ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+		ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
+		if (ss->security.ses != NULL) {
+			rc = rte_ipsec_session_prepare(ss);
+			if (rc != 0)
+				memset(ss, 0, sizeof(*ss));
+		}
+	}
+
+	return rc;
 }
 
 /*
@@ -1047,8 +1073,8 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	fill_ipsec_session(&lsa->ips, sa, lsa);
-	return 0;
+	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
+	return rc;
 }
 
 /*
@@ -1126,7 +1152,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in);
+		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx);
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_in, sa_in, nb_sa_in,
@@ -1146,7 +1172,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out);
+		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx);
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_out, sa_out, nb_sa_out,
-- 
2.7.4

[dpdk-dev] [PATCH 3/6] examples/ipsec-secgw: fix 1st packet dropped patch three

[Bernard Iremonger]

Refactor cryprodev_init() and main() in ipsec-secgw.c
Add max_session_size().
Start ports before adding flows in main().

Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
Cc: stable@dpdk.org
Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/ipsec-secgw.c | 246 ++++++++++++++++++-------------------
 1 file changed, 123 insertions(+), 123 deletions(-)

diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index 8e7cd1b..805e6b4 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -1029,6 +1029,9 @@ main_loop(__attribute__((unused)) void *dummy)
 				drain_outbound_crypto_queues(qconf,
 					&qconf->outbound);
 		}
+
+		drain_inbound_crypto_queues(qconf, &qconf->inbound);
+		drain_outbound_crypto_queues(qconf, &qconf->outbound);
 	}
 }
 
@@ -1627,7 +1630,7 @@ cryptodevs_init(void)
 	struct rte_cryptodev_config dev_conf;
 	struct rte_cryptodev_qp_conf qp_conf;
 	uint16_t idx, max_nb_qps, qp, i;
-	int16_t cdev_id, port_id;
+	int16_t cdev_id;
 	struct rte_hash_parameters params = { 0 };
 
 	params.entries = CDEV_MAP_ENTRIES;
@@ -1650,45 +1653,6 @@ cryptodevs_init(void)
 
 	printf("lcore/cryptodev/qp mappings:\n");
 
-	uint32_t max_sess_sz = 0, sess_sz;
-	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
-		void *sec_ctx;
-
-		/* Get crypto priv session size */
-		sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-
-		/*
-		 * If crypto device is security capable, need to check the
-		 * size of security session as well.
-		 */
-
-		/* Get security context of the crypto device */
-		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		/* Get size of security session */
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-	RTE_ETH_FOREACH_DEV(port_id) {
-		void *sec_ctx;
-
-		if ((enabled_port_mask & (1 << port_id)) == 0)
-			continue;
-
-		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-
 	idx = 0;
 	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
 		struct rte_cryptodev_info cdev_info;
@@ -1726,45 +1690,6 @@ cryptodevs_init(void)
 				"Device does not support at least %u "
 				"sessions", CDEV_MP_NB_OBJS);
 
-		if (!socket_ctx[dev_conf.socket_id].session_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_%u", dev_conf.socket_id);
-			sess_mp = rte_cryptodev_sym_session_pool_create(
-					mp_name, CDEV_MP_NB_OBJS,
-					0, CDEV_MP_CACHE_SZ, 0,
-					dev_conf.socket_id);
-			socket_ctx[dev_conf.socket_id].session_pool = sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_priv_%u", dev_conf.socket_id);
-			sess_mp = rte_mempool_create(mp_name,
-					CDEV_MP_NB_OBJS,
-					max_sess_sz,
-					CDEV_MP_CACHE_SZ,
-					0, NULL, NULL, NULL,
-					NULL, dev_conf.socket_id,
-					0);
-			socket_ctx[dev_conf.socket_id].session_priv_pool =
-					sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool ||
-				!socket_ctx[dev_conf.socket_id].session_pool)
-			rte_exit(EXIT_FAILURE,
-				"Cannot create session pool on socket %d\n",
-				dev_conf.socket_id);
-		else
-			printf("Allocated session pool on socket %d\n",
-					dev_conf.socket_id);
-
 		if (rte_cryptodev_configure(cdev_id, &dev_conf))
 			rte_panic("Failed to initialize cryptodev %u\n",
 					cdev_id);
@@ -1785,38 +1710,6 @@ cryptodevs_init(void)
 					cdev_id);
 	}
 
-	/* create session pools for eth devices that implement security */
-	RTE_ETH_FOREACH_DEV(port_id) {
-		if ((enabled_port_mask & (1 << port_id)) &&
-				rte_eth_dev_get_sec_ctx(port_id)) {
-			int socket_id = rte_eth_dev_socket_id(port_id);
-
-			if (!socket_ctx[socket_id].session_pool) {
-				char mp_name[RTE_MEMPOOL_NAMESIZE];
-				struct rte_mempool *sess_mp;
-
-				snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-						"sess_mp_%u", socket_id);
-				sess_mp = rte_mempool_create(mp_name,
-						(CDEV_MP_NB_OBJS * 2),
-						max_sess_sz,
-						CDEV_MP_CACHE_SZ,
-						0, NULL, NULL, NULL,
-						NULL, socket_id,
-						0);
-				if (sess_mp == NULL)
-					rte_exit(EXIT_FAILURE,
-						"Cannot create session pool "
-						"on socket %d\n", socket_id);
-				else
-					printf("Allocated session pool "
-						"on socket %d\n", socket_id);
-				socket_ctx[socket_id].session_pool = sess_mp;
-			}
-		}
-	}
-
-
 	printf("\n");
 
 	return 0;
@@ -1982,6 +1875,99 @@ port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads)
 	printf("\n");
 }
 
+static size_t
+max_session_size(void)
+{
+	size_t max_sz, sz;
+	void *sec_ctx;
+	int16_t cdev_id, port_id, n;
+
+	max_sz = 0;
+	n =  rte_cryptodev_count();
+	for (cdev_id = 0; cdev_id != n; cdev_id++) {
+		sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
+		if (sz > max_sz)
+			max_sz = sz;
+		/*
+		 * If crypto device is security capable, need to check the
+		 * size of security session as well.
+		 */
+
+		/* Get security context of the crypto device */
+		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		/* Get size of security session */
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	RTE_ETH_FOREACH_DEV(port_id) {
+		if ((enabled_port_mask & (1 << port_id)) == 0)
+			continue;
+
+		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	return max_sz;
+}
+
+static void
+session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_%u", socket_id);
+	sess_mp = rte_cryptodev_sym_session_pool_create(
+			mp_name, CDEV_MP_NB_OBJS,
+			sess_sz, CDEV_MP_CACHE_SZ, 0,
+			socket_id);
+	ctx->session_pool = sess_mp;
+
+	if (ctx->session_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session pool on socket %d\n", socket_id);
+	else
+		printf("Allocated session pool on socket %d\n",	socket_id);
+}
+
+static void
+session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
+	size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_priv_%u", socket_id);
+	sess_mp = rte_mempool_create(mp_name,
+			CDEV_MP_NB_OBJS,
+			sess_sz,
+			CDEV_MP_CACHE_SZ,
+			0, NULL, NULL, NULL,
+			NULL, socket_id,
+			0);
+	ctx->session_priv_pool = sess_mp;
+
+	if (ctx->session_priv_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session priv pool on socket %d\n",
+			socket_id);
+	else
+		printf("Allocated session priv pool on socket %d\n",
+			socket_id);
+}
+
 static void
 pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf)
 {
@@ -2062,9 +2048,11 @@ main(int32_t argc, char **argv)
 {
 	int32_t ret;
 	uint32_t lcore_id;
+	uint32_t i;
 	uint8_t socket_id;
 	uint16_t portid;
 	uint64_t req_rx_offloads, req_tx_offloads;
+	size_t sess_sz;
 
 	/* init EAL */
 	ret = rte_eal_init(argc, argv);
@@ -2092,7 +2080,8 @@ main(int32_t argc, char **argv)
 
 	nb_lcores = rte_lcore_count();
 
-	/* Replicate each context per socket */
+	sess_sz = max_session_size();
+
 	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
 		if (rte_lcore_is_enabled(lcore_id) == 0)
 			continue;
@@ -2102,20 +2091,14 @@ main(int32_t argc, char **argv)
 		else
 			socket_id = 0;
 
+		/* mbuf_pool is initialised by the pool_init() function*/
 		if (socket_ctx[socket_id].mbuf_pool)
 			continue;
 
-		/* initilaze SPD */
-		sp4_init(&socket_ctx[socket_id], socket_id);
-
-		sp6_init(&socket_ctx[socket_id], socket_id);
-
-		/* initilaze SAD */
-		sa_init(&socket_ctx[socket_id], socket_id);
-
-		rt_init(&socket_ctx[socket_id], socket_id);
-
 		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
+		session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
+		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
+			sess_sz);
 	}
 
 	RTE_ETH_FOREACH_DEV(portid) {
@@ -2133,7 +2116,11 @@ main(int32_t argc, char **argv)
 		if ((enabled_port_mask & (1 << portid)) == 0)
 			continue;
 
-		/* Start device */
+		/*
+		 * Start device
+		 * note: device must be started before a flow rule
+		 * can be installed.
+		 */
 		ret = rte_eth_dev_start(portid);
 		if (ret < 0)
 			rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
@@ -2151,6 +2138,19 @@ main(int32_t argc, char **argv)
 			RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback, NULL);
 	}
 
+	/* Replicate each context per socket */
+	for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
+		socket_id = rte_socket_id_by_idx(i);
+		if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
+			(socket_ctx[socket_id].sa_in == NULL) &&
+			(socket_ctx[socket_id].sa_out == NULL)) {
+			sa_init(&socket_ctx[socket_id], socket_id);
+			sp4_init(&socket_ctx[socket_id], socket_id);
+			sp6_init(&socket_ctx[socket_id], socket_id);
+			rt_init(&socket_ctx[socket_id], socket_id);
+		}
+	}
+
 	check_all_ports_link_status(enabled_port_mask);
 
 	/* launch per-lcore init on every lcore */
-- 
2.7.4

[dpdk-dev] [PATCH 4/6] examples/ipsec-secgw: fix debug in esp.c

[Bernard Iremonger]

Improve debug code in esp.c

Fixes: f159e70b0922 ("examples/ipsec-secgw: support transport mode")
Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
Cc: stable@dpdk.org
Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/esp.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/examples/ipsec-secgw/esp.c b/examples/ipsec-secgw/esp.c
index e33232c..faa84dd 100644
--- a/examples/ipsec-secgw/esp.c
+++ b/examples/ipsec-secgw/esp.c
@@ -162,7 +162,7 @@ esp_inbound_post(struct rte_mbuf *m, struct ipsec_sa *sa,
 	}
 
 	if (cop->status != RTE_CRYPTO_OP_STATUS_SUCCESS) {
-		RTE_LOG(ERR, IPSEC_ESP, "failed crypto op\n");
+		RTE_LOG(ERR, IPSEC_ESP, "%s() failed crypto op\n", __func__);
 		return -1;
 	}
 
@@ -455,7 +455,8 @@ esp_outbound_post(struct rte_mbuf *m,
 	} else {
 		RTE_ASSERT(cop != NULL);
 		if (cop->status != RTE_CRYPTO_OP_STATUS_SUCCESS) {
-			RTE_LOG(ERR, IPSEC_ESP, "Failed crypto op\n");
+			RTE_LOG(ERR, IPSEC_ESP, "%s() failed crypto op\n",
+				__func__);
 			return -1;
 		}
 	}
-- 
2.7.4

[dpdk-dev] [PATCH 5/6] examples/ipsec-secgw: fix debug in sa.c

[Bernard Iremonger]

Improve debug code in sa.c

Fixes: 0d547ed03717 ("examples/ipsec-secgw: support configuration file")
Cc: stable@dpdk.org
Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/sa.c | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index 7fb1929..4e583cd 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -688,7 +688,22 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 		}
 		break;
 	case TRANSPORT:
-		printf("Transport");
+		printf("Transport ");
+		break;
+	}
+	printf(" type:");
+	switch (sa->type) {
+	case RTE_SECURITY_ACTION_TYPE_NONE:
+		printf("no-offload ");
+		break;
+	case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
+		printf("inline-crypto-offload ");
+		break;
+	case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
+		printf("inline-protocol-offload ");
+		break;
+	case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL:
+		printf("lookaside-protocol-offload ");
 		break;
 	}
 	printf("\n");
@@ -716,8 +731,8 @@ sa_create(const char *name, int32_t socket_id)
 	snprintf(s, sizeof(s), "%s_%u", name, socket_id);
 
 	/* Create SA array table */
-	printf("Creating SA context with %u maximum entries\n",
-			IPSEC_SA_MAX_ENTRIES);
+	printf("Creating SA context with %u maximum entries on socket %d\n",
+			IPSEC_SA_MAX_ENTRIES, socket_id);
 
 	mz_size = sizeof(struct sa_ctx);
 	mz = rte_memzone_reserve(s, mz_size, socket_id,
-- 
2.7.4

[dpdk-dev] [PATCH 6/6] examples/ipsec-secgw: fix debug in ipsec-secgw.c

[Bernard Iremonger]

Improve debug in ipsec-secgw.c

Fixes: 906257e965b7 ("examples/ipsec-secgw: support IPv6")
Cc: stable@dpdk.org
Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/ipsec-secgw.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index 805e6b4..0c3c8f8 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -260,7 +260,8 @@ prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)
 		pkt->l3_len = sizeof(struct ip6_hdr);
 	} else {
 		/* Unknown/Unsupported type, drop the packet */
-		RTE_LOG(ERR, IPSEC, "Unsupported packet type\n");
+		RTE_LOG(ERR, IPSEC, "Unsupported packet type 0x%x\n",
+			rte_be_to_cpu_16(eth->ether_type));
 		rte_pktmbuf_free(pkt);
 	}
 
@@ -984,7 +985,8 @@ main_loop(__attribute__((unused)) void *dummy)
 			socket_ctx[socket_id].session_priv_pool;
 
 	if (qconf->nb_rx_queue == 0) {
-		RTE_LOG(INFO, IPSEC, "lcore %u has nothing to do\n", lcore_id);
+		RTE_LOG(DEBUG, IPSEC, "lcore %u has nothing to do\n",
+			lcore_id);
 		return 0;
 	}
 
-- 
2.7.4

Re: [dpdk-dev] [PATCH 0/6] examples/ipsec-secgw: fix 1st pkt dropped

[Akhil Goyal]

Hi Bernard,

On 3/6/2019 9:30 PM, Bernard Iremonger wrote:
> This patchset fixes the issue of the first inbound packet
> being dropped for inline crypto. It also improves the debug
> output in esp.c, sa.c and ipsec-secgw.c
>
> Bernard Iremonger (6):
>    examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
>    examples/ipsec-secgw: fix 1st packet dropped patch two
>    examples/ipsec-secgw: fix 1st packet dropped patch three
patch titles are not good. You can probably add some logical comments in 
the description.
you may also consider squashing the patches if no logical bifurcation 
can be made.

>    examples/ipsec-secgw: fix debug in esp.c
>    examples/ipsec-secgw: fix debug in sa.c
>    examples/ipsec-secgw: fix debug in ipsec-secgw.c
Above 3 patches can be squashed and can be separated from this series.
>   examples/ipsec-secgw/esp.c           |   5 +-
>   examples/ipsec-secgw/ipsec-secgw.c   | 252 ++++++++++++++++++-----------------
>   examples/ipsec-secgw/ipsec.c         | 123 +++++++++++------
>   examples/ipsec-secgw/ipsec.h         |   5 +-
>   examples/ipsec-secgw/ipsec_process.c |   9 +-
>   examples/ipsec-secgw/sa.c            |  67 ++++++++--
>   6 files changed, 274 insertions(+), 187 deletions(-)
>

Re: [dpdk-dev] [PATCH 2/6] examples/ipsec-secgw: fix 1st packet dropped patch two

[Ananyev, Konstantin]

Hi Bernard,

> 
> Call create_inline_session() at initialisition in sa.c
> Call rte_ipsec_session_prepare() in fill_ipsec_session() for inline.

Here and in other places - it probably worth to explain what is the purpose
for  these changes. 
As a side notice, as these series fixes that problem, it probably worse to add a patch
into series that removes the following:

       # to overcome problem with ipsec-secgw for inline mode,
        # when first packet(s) will be always dropped.
        # note that ping will fail here
        ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}

from examples/ipsec-secgw/test/(tun|trs)_aesgcm_defs.sh
Konstantin

> 
> Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
> Cc: stable@dpdk.org
> Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> ---
>  examples/ipsec-secgw/sa.c | 46 ++++++++++++++++++++++++++++++++++++----------
>  1 file changed, 36 insertions(+), 10 deletions(-)
> 
> diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
> index 414fcd2..7fb1929 100644
> --- a/examples/ipsec-secgw/sa.c
> +++ b/examples/ipsec-secgw/sa.c
> @@ -762,11 +762,13 @@ check_eth_dev_caps(uint16_t portid, uint32_t inbound)
> 
>  static int
>  sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> -		uint32_t nb_entries, uint32_t inbound)
> +		uint32_t nb_entries, uint32_t inbound,
> +		struct socket_ctx *skt_ctx)
>  {
>  	struct ipsec_sa *sa;
>  	uint32_t i, idx;
>  	uint16_t iv_length, aad_length;
> +	int32_t rc;
> 
>  	/* for ESN upper 32 bits of SQN also need to be part of AAD */
>  	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
> @@ -819,6 +821,17 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> 
>  			sa->xforms = &sa_ctx->xf[idx].a;
> 
> +			if (sa->type ==
> +				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
> +				sa->type ==
> +				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
> +				rc = create_inline_session(skt_ctx, sa);
> +				if (rc != 0) {
> +					RTE_LOG(ERR, IPSEC_ESP,
> +						"create_inline_session() failed\n");
> +					return -EINVAL;
> +				}
> +			}
>  			print_one_sa_rule(sa, inbound);
>  		} else {
>  			switch (sa->cipher_algo) {
> @@ -894,16 +907,16 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> 
>  static inline int
>  sa_out_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> -		uint32_t nb_entries)
> +		uint32_t nb_entries, struct socket_ctx *skt_ctx)
>  {
> -	return sa_add_rules(sa_ctx, entries, nb_entries, 0);
> +	return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx);
>  }
> 
>  static inline int
>  sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> -		uint32_t nb_entries)
> +		uint32_t nb_entries, struct socket_ctx *skt_ctx)
>  {
> -	return sa_add_rules(sa_ctx, entries, nb_entries, 1);
> +	return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx);
>  }
> 
>  /*
> @@ -997,10 +1010,12 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm, const struct ipsec_sa *ss,
>  	return 0;
>  }
> 
> -static void
> +static int
>  fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
>  	const struct ipsec_sa *lsa)
>  {
> +	int32_t rc = 0;
> +
>  	ss->sa = sa;
>  	ss->type = lsa->type;
> 
> @@ -1013,6 +1028,17 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
>  		ss->security.ctx = lsa->security_ctx;
>  		ss->security.ol_flags = lsa->ol_flags;
>  	}
> +
> +	if (ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
> +		ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
> +		if (ss->security.ses != NULL) {
> +			rc = rte_ipsec_session_prepare(ss);
> +			if (rc != 0)
> +				memset(ss, 0, sizeof(*ss));
> +		}
> +	}
> +
> +	return rc;
>  }
> 
>  /*
> @@ -1047,8 +1073,8 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
>  	if (rc < 0)
>  		return rc;
> 
> -	fill_ipsec_session(&lsa->ips, sa, lsa);
> -	return 0;
> +	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
> +	return rc;
>  }
> 
>  /*
> @@ -1126,7 +1152,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
>  				"context %s in socket %d\n", rte_errno,
>  				name, socket_id);
> 
> -		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in);
> +		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx);
> 
>  		if (app_sa_prm.enable != 0) {
>  			rc = ipsec_satbl_init(ctx->sa_in, sa_in, nb_sa_in,
> @@ -1146,7 +1172,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
>  				"context %s in socket %d\n", rte_errno,
>  				name, socket_id);
> 
> -		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out);
> +		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx);
> 
>  		if (app_sa_prm.enable != 0) {
>  			rc = ipsec_satbl_init(ctx->sa_out, sa_out, nb_sa_out,
> --
> 2.7.4

Re: [dpdk-dev] [PATCH 2/6] examples/ipsec-secgw: fix 1st packet dropped patch two

[Iremonger, Bernard]

Hi Konstantin,

<snip>

> Subject: RE: [PATCH 2/6] examples/ipsec-secgw: fix 1st packet dropped patch
> two
> 
> 
> Hi Bernard,
> 
> >
> > Call create_inline_session() at initialisition in sa.c Call
> > rte_ipsec_session_prepare() in fill_ipsec_session() for inline.
> 
> Here and in other places - it probably worth to explain what is the purpose for
> these changes.

I will improve explanation.

> As a side notice, as these series fixes that problem, it probably worse to add a
> patch into series that removes the following:
> 
>        # to overcome problem with ipsec-secgw for inline mode,
>         # when first packet(s) will be always dropped.
>         # note that ping will fail here
>         ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
> 
> from examples/ipsec-secgw/test/(tun|trs)_aesgcm_defs.sh
> Konstantin

I will add a patch to fix the test scripts.

<snip>

Regards,

Bernard.

Re: [dpdk-dev] [PATCH 0/6] examples/ipsec-secgw: fix 1st pkt dropped

[Iremonger, Bernard]

Hi Akhil,

<snip>


> Subject: Re: [dpdk-dev] [PATCH 0/6] examples/ipsec-secgw: fix 1st pkt dropped
> 
> Hi Bernard,
> 
> On 3/6/2019 9:30 PM, Bernard Iremonger wrote:
> > This patchset fixes the issue of the first inbound packet being
> > dropped for inline crypto. It also improves the debug output in esp.c,
> > sa.c and ipsec-secgw.c
> >
> > Bernard Iremonger (6):
> >    examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
> >    examples/ipsec-secgw: fix 1st packet dropped patch two
> >    examples/ipsec-secgw: fix 1st packet dropped patch three
> patch titles are not good. You can probably add some logical comments in the
> description.

I will update the patch titles and descriptions.

> you may also consider squashing the patches if no logical bifurcation can be
> made.

I thought it might be easier to review the changes as separate patches.
If you prefer one patch I can squash them.

> >    examples/ipsec-secgw: fix debug in esp.c
> >    examples/ipsec-secgw: fix debug in sa.c
> >    examples/ipsec-secgw: fix debug in ipsec-secgw.c
> Above 3 patches can be squashed and can be separated from this series.

I will remove from this patch set, squash and send as an independent patch.  

> >   examples/ipsec-secgw/esp.c           |   5 +-
> >   examples/ipsec-secgw/ipsec-secgw.c   | 252 ++++++++++++++++++------------
> -----
> >   examples/ipsec-secgw/ipsec.c         | 123 +++++++++++------
> >   examples/ipsec-secgw/ipsec.h         |   5 +-
> >   examples/ipsec-secgw/ipsec_process.c |   9 +-
> >   examples/ipsec-secgw/sa.c            |  67 ++++++++--
> >   6 files changed, 274 insertions(+), 187 deletions(-)
> >

Regards,

Bernard.

[dpdk-dev] [PATCH v2 0/2] examples/ipsec-secgw: fix 1st pkt dropped

[Bernard Iremonger]

This patchset fixes the issue of the first inbound packet
being dropped for inline crypto. 
 
Changes in v2:
-------------
The first three patches of the v1 have been squashed.
The commit message for the squashed patch has been updated.
Patches 4,5 and 6 of the v1 have been dropped from this patchset.
A patch to fix the test scripts has been added.

Bernard Iremonger (2):
  examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
  examples/ipsec-secgw/test: fix inline test scripts

 examples/ipsec-secgw/ipsec-secgw.c           | 243 +++++++++++++--------------
 examples/ipsec-secgw/ipsec.c                 | 123 +++++++++-----
 examples/ipsec-secgw/ipsec.h                 |   5 +-
 examples/ipsec-secgw/ipsec_process.c         |   9 +-
 examples/ipsec-secgw/sa.c                    |  46 +++--
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 --
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 --
 7 files changed, 246 insertions(+), 200 deletions(-)

-- 
2.7.4

[dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Bernard Iremonger]

Inline crypto installs a flow rule in the NIC. This flow
rule must be installed before the first inbound packet is
received.

The create_session() function installs the flow rule,
create_session() has been refactored into create_inline_session()
and create_lookaside_session(). The create_inline_session() function
uses the socket_ctx data and is now called at initialisation in
sa_add_rules().

The max_session_size() function has been added to calculate memory
requirements.

The cryprodev_init() function has been refactored to drop calls to
rte_mempool_create() and to drop calculation of memory requirements.

The main() function has been refactored to call max_session_size() and
to call session_pool_init() and session_priv_pool_init() earlier.
The ports are started now before adding a flow rule in main().
The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
now called after the ports have been started.

The rte_ipsec_session_prepare() function is called in fill_ipsec_session()
for inline which is called from the ipsec_sa_init() function.

Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/ipsec-secgw.c   | 243 +++++++++++++++++------------------
 examples/ipsec-secgw/ipsec.c         | 123 ++++++++++++------
 examples/ipsec-secgw/ipsec.h         |   5 +-
 examples/ipsec-secgw/ipsec_process.c |   9 +-
 examples/ipsec-secgw/sa.c            |  46 +++++--
 5 files changed, 246 insertions(+), 180 deletions(-)

diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index 8e7cd1b..835cc98 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -1627,7 +1627,7 @@ cryptodevs_init(void)
 	struct rte_cryptodev_config dev_conf;
 	struct rte_cryptodev_qp_conf qp_conf;
 	uint16_t idx, max_nb_qps, qp, i;
-	int16_t cdev_id, port_id;
+	int16_t cdev_id;
 	struct rte_hash_parameters params = { 0 };
 
 	params.entries = CDEV_MAP_ENTRIES;
@@ -1650,45 +1650,6 @@ cryptodevs_init(void)
 
 	printf("lcore/cryptodev/qp mappings:\n");
 
-	uint32_t max_sess_sz = 0, sess_sz;
-	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
-		void *sec_ctx;
-
-		/* Get crypto priv session size */
-		sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-
-		/*
-		 * If crypto device is security capable, need to check the
-		 * size of security session as well.
-		 */
-
-		/* Get security context of the crypto device */
-		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		/* Get size of security session */
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-	RTE_ETH_FOREACH_DEV(port_id) {
-		void *sec_ctx;
-
-		if ((enabled_port_mask & (1 << port_id)) == 0)
-			continue;
-
-		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-
 	idx = 0;
 	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
 		struct rte_cryptodev_info cdev_info;
@@ -1726,45 +1687,6 @@ cryptodevs_init(void)
 				"Device does not support at least %u "
 				"sessions", CDEV_MP_NB_OBJS);
 
-		if (!socket_ctx[dev_conf.socket_id].session_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_%u", dev_conf.socket_id);
-			sess_mp = rte_cryptodev_sym_session_pool_create(
-					mp_name, CDEV_MP_NB_OBJS,
-					0, CDEV_MP_CACHE_SZ, 0,
-					dev_conf.socket_id);
-			socket_ctx[dev_conf.socket_id].session_pool = sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_priv_%u", dev_conf.socket_id);
-			sess_mp = rte_mempool_create(mp_name,
-					CDEV_MP_NB_OBJS,
-					max_sess_sz,
-					CDEV_MP_CACHE_SZ,
-					0, NULL, NULL, NULL,
-					NULL, dev_conf.socket_id,
-					0);
-			socket_ctx[dev_conf.socket_id].session_priv_pool =
-					sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool ||
-				!socket_ctx[dev_conf.socket_id].session_pool)
-			rte_exit(EXIT_FAILURE,
-				"Cannot create session pool on socket %d\n",
-				dev_conf.socket_id);
-		else
-			printf("Allocated session pool on socket %d\n",
-					dev_conf.socket_id);
-
 		if (rte_cryptodev_configure(cdev_id, &dev_conf))
 			rte_panic("Failed to initialize cryptodev %u\n",
 					cdev_id);
@@ -1785,38 +1707,6 @@ cryptodevs_init(void)
 					cdev_id);
 	}
 
-	/* create session pools for eth devices that implement security */
-	RTE_ETH_FOREACH_DEV(port_id) {
-		if ((enabled_port_mask & (1 << port_id)) &&
-				rte_eth_dev_get_sec_ctx(port_id)) {
-			int socket_id = rte_eth_dev_socket_id(port_id);
-
-			if (!socket_ctx[socket_id].session_pool) {
-				char mp_name[RTE_MEMPOOL_NAMESIZE];
-				struct rte_mempool *sess_mp;
-
-				snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-						"sess_mp_%u", socket_id);
-				sess_mp = rte_mempool_create(mp_name,
-						(CDEV_MP_NB_OBJS * 2),
-						max_sess_sz,
-						CDEV_MP_CACHE_SZ,
-						0, NULL, NULL, NULL,
-						NULL, socket_id,
-						0);
-				if (sess_mp == NULL)
-					rte_exit(EXIT_FAILURE,
-						"Cannot create session pool "
-						"on socket %d\n", socket_id);
-				else
-					printf("Allocated session pool "
-						"on socket %d\n", socket_id);
-				socket_ctx[socket_id].session_pool = sess_mp;
-			}
-		}
-	}
-
-
 	printf("\n");
 
 	return 0;
@@ -1982,6 +1872,99 @@ port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads)
 	printf("\n");
 }
 
+static size_t
+max_session_size(void)
+{
+	size_t max_sz, sz;
+	void *sec_ctx;
+	int16_t cdev_id, port_id, n;
+
+	max_sz = 0;
+	n =  rte_cryptodev_count();
+	for (cdev_id = 0; cdev_id != n; cdev_id++) {
+		sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
+		if (sz > max_sz)
+			max_sz = sz;
+		/*
+		 * If crypto device is security capable, need to check the
+		 * size of security session as well.
+		 */
+
+		/* Get security context of the crypto device */
+		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		/* Get size of security session */
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	RTE_ETH_FOREACH_DEV(port_id) {
+		if ((enabled_port_mask & (1 << port_id)) == 0)
+			continue;
+
+		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	return max_sz;
+}
+
+static void
+session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_%u", socket_id);
+	sess_mp = rte_cryptodev_sym_session_pool_create(
+			mp_name, CDEV_MP_NB_OBJS,
+			sess_sz, CDEV_MP_CACHE_SZ, 0,
+			socket_id);
+	ctx->session_pool = sess_mp;
+
+	if (ctx->session_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session pool on socket %d\n", socket_id);
+	else
+		printf("Allocated session pool on socket %d\n",	socket_id);
+}
+
+static void
+session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
+	size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_priv_%u", socket_id);
+	sess_mp = rte_mempool_create(mp_name,
+			CDEV_MP_NB_OBJS,
+			sess_sz,
+			CDEV_MP_CACHE_SZ,
+			0, NULL, NULL, NULL,
+			NULL, socket_id,
+			0);
+	ctx->session_priv_pool = sess_mp;
+
+	if (ctx->session_priv_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session priv pool on socket %d\n",
+			socket_id);
+	else
+		printf("Allocated session priv pool on socket %d\n",
+			socket_id);
+}
+
 static void
 pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf)
 {
@@ -2062,9 +2045,11 @@ main(int32_t argc, char **argv)
 {
 	int32_t ret;
 	uint32_t lcore_id;
+	uint32_t i;
 	uint8_t socket_id;
 	uint16_t portid;
 	uint64_t req_rx_offloads, req_tx_offloads;
+	size_t sess_sz;
 
 	/* init EAL */
 	ret = rte_eal_init(argc, argv);
@@ -2092,7 +2077,8 @@ main(int32_t argc, char **argv)
 
 	nb_lcores = rte_lcore_count();
 
-	/* Replicate each context per socket */
+	sess_sz = max_session_size();
+
 	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
 		if (rte_lcore_is_enabled(lcore_id) == 0)
 			continue;
@@ -2102,20 +2088,14 @@ main(int32_t argc, char **argv)
 		else
 			socket_id = 0;
 
+		/* mbuf_pool is initialised by the pool_init() function*/
 		if (socket_ctx[socket_id].mbuf_pool)
 			continue;
 
-		/* initilaze SPD */
-		sp4_init(&socket_ctx[socket_id], socket_id);
-
-		sp6_init(&socket_ctx[socket_id], socket_id);
-
-		/* initilaze SAD */
-		sa_init(&socket_ctx[socket_id], socket_id);
-
-		rt_init(&socket_ctx[socket_id], socket_id);
-
 		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
+		session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
+		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
+			sess_sz);
 	}
 
 	RTE_ETH_FOREACH_DEV(portid) {
@@ -2133,7 +2113,11 @@ main(int32_t argc, char **argv)
 		if ((enabled_port_mask & (1 << portid)) == 0)
 			continue;
 
-		/* Start device */
+		/*
+		 * Start device
+		 * note: device must be started before a flow rule
+		 * can be installed.
+		 */
 		ret = rte_eth_dev_start(portid);
 		if (ret < 0)
 			rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
@@ -2151,6 +2135,19 @@ main(int32_t argc, char **argv)
 			RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback, NULL);
 	}
 
+	/* Replicate each context per socket */
+	for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
+		socket_id = rte_socket_id_by_idx(i);
+		if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
+			(socket_ctx[socket_id].sa_in == NULL) &&
+			(socket_ctx[socket_id].sa_out == NULL)) {
+			sa_init(&socket_ctx[socket_id], socket_id);
+			sp4_init(&socket_ctx[socket_id], socket_id);
+			sp6_init(&socket_ctx[socket_id], socket_id);
+			rt_init(&socket_ctx[socket_id], socket_id);
+		}
+	}
+
 	check_all_ports_link_status(enabled_port_mask);
 
 	/* launch per-lcore init on every lcore */
diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index 4352cb8..7090dbe 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -40,7 +40,7 @@ set_ipsec_conf(struct ipsec_sa *sa, struct rte_security_ipsec_xform *ipsec)
 }
 
 int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 {
 	struct rte_cryptodev_info cdev_info;
 	unsigned long cdev_id_qp = 0;
@@ -108,23 +108,82 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 				"SEC Session init failed: err: %d\n", ret);
 				return -1;
 			}
-		} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		} else if ((sa->type ==
+				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) ||
+				(sa->type ==
+				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)) {
+					RTE_LOG(ERR, IPSEC,
+						"Inline not supported\n");
+					return -1;
+		}
+	} else {
+		sa->crypto_session = rte_cryptodev_sym_session_create(
+				ipsec_ctx->session_pool);
+		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
+				sa->crypto_session, sa->xforms,
+				ipsec_ctx->session_priv_pool);
+
+		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
+				&cdev_info);
+	}
+
+	sa->cdev_id_qp = cdev_id_qp;
+
+	return 0;
+}
+
+int
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
+{
+	unsigned long cdev_id_qp = 0;
+	int32_t ret = 0;
+	struct rte_security_ctx *sec_ctx;
+
+	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on port %u\n",
+		sa->spi, sa->portid);
+
+	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
+		struct rte_security_session_conf sess_conf = {
+			.action_type = sa->type,
+			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
+			{.ipsec = {
+				.spi = sa->spi,
+				.salt = sa->salt,
+				.options = { 0 },
+				.direction = sa->direction,
+				.proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
+				.mode = (sa->flags == IP4_TUNNEL ||
+						sa->flags == IP6_TUNNEL) ?
+					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL :
+					RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
+			} },
+			.crypto_xform = sa->xforms,
+			.userdata = NULL,
+		};
+
+		if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
 			struct rte_flow_error err;
-			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
-							rte_eth_dev_get_sec_ctx(
-							sa->portid);
 			const struct rte_security_capability *sec_cap;
 			int ret = 0;
 
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
+			sec_ctx = (struct rte_security_ctx *)
+							rte_eth_dev_get_sec_ctx(
+							sa->portid);
+			if (sec_ctx == NULL) {
+				RTE_LOG(ERR, IPSEC,
+				" rte_eth_dev_get_sec_ctx failed\n");
+				return -1;
+			}
+
+			sa->sec_session = rte_security_session_create(sec_ctx,
+					&sess_conf, skt_ctx->session_pool);
 			if (sa->sec_session == NULL) {
 				RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 				return -1;
 			}
 
-			sec_cap = rte_security_capabilities_get(ctx);
+			sec_cap = rte_security_capabilities_get(sec_ctx);
 
 			/* iterate until ESP tunnel*/
 			while (sec_cap->action !=
@@ -147,7 +206,7 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			}
 
 			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
+			sa->security_ctx = sec_ctx;
 			sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
 
 			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
@@ -196,7 +255,7 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 				/* Try RSS. */
 				sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
 				sa->action[1].conf = &action_rss;
-				eth_dev = ctx->device;
+				eth_dev = sec_ctx->device;
 				rte_eth_dev_rss_hash_conf_get(sa->portid,
 							      &rss_conf);
 				for (i = 0, j = 0;
@@ -252,12 +311,12 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			}
 		} else if (sa->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
-			struct rte_security_ctx *ctx =
-					(struct rte_security_ctx *)
-					rte_eth_dev_get_sec_ctx(sa->portid);
 			const struct rte_security_capability *sec_cap;
 
-			if (ctx == NULL) {
+			sec_ctx = (struct rte_security_ctx *)
+					rte_eth_dev_get_sec_ctx(sa->portid);
+
+			if (sec_ctx == NULL) {
 				RTE_LOG(ERR, IPSEC,
 				"Ethernet device doesn't have security features registered\n");
 				return -1;
@@ -279,15 +338,15 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 
 			sess_conf.userdata = (void *) sa;
 
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
+			sa->sec_session = rte_security_session_create(sec_ctx,
+					&sess_conf, skt_ctx->session_pool);
 			if (sa->sec_session == NULL) {
 				RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 				return -1;
 			}
 
-			sec_cap = rte_security_capabilities_get(ctx);
+			sec_cap = rte_security_capabilities_get(sec_ctx);
 
 			if (sec_cap == NULL) {
 				RTE_LOG(ERR, IPSEC,
@@ -316,17 +375,8 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			}
 
 			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
+			sa->security_ctx = sec_ctx;
 		}
-	} else {
-		sa->crypto_session = rte_cryptodev_sym_session_create(
-				ipsec_ctx->session_pool);
-		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
-				sa->crypto_session, sa->xforms,
-				ipsec_ctx->session_priv_pool);
-
-		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
-				&cdev_info);
 	}
 	sa->cdev_id_qp = cdev_id_qp;
 
@@ -395,7 +445,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			rte_prefetch0(&priv->sym_cop);
 
 			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
+				create_lookaside_session(ipsec_ctx, sa)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -414,7 +464,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			rte_prefetch0(&priv->sym_cop);
 
 			if ((unlikely(sa->crypto_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
+				create_lookaside_session(ipsec_ctx, sa)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -429,12 +479,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 			break;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
+			RTE_ASSERT(sa->sec_session != NULL);
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
 			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
@@ -442,17 +487,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 						sa->sec_session, pkts[i], NULL);
 			continue;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
+			RTE_ASSERT(sa->sec_session != NULL);
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			rte_security_attach_session(&priv->cop,
 					sa->sec_session);
 
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index 99f49d6..db5a5c1 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -306,6 +306,9 @@ void
 enqueue_cop_burst(struct cdev_qp *cqp);
 
 int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+
+int
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa);
 
 #endif /* __IPSEC_H__ */
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index e403c46..152a684 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -95,22 +95,23 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
 	/* setup crypto section */
 	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
 		if (sa->crypto_session == NULL) {
-			rc = create_session(ctx, sa);
+			rc = create_lookaside_session(ctx, sa);
 			if (rc != 0)
 				return rc;
 		}
 		ss->crypto.ses = sa->crypto_session;
 	/* setup session action type */
-	} else {
+	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
 		if (sa->sec_session == NULL) {
-			rc = create_session(ctx, sa);
+			rc = create_lookaside_session(ctx, sa);
 			if (rc != 0)
 				return rc;
 		}
 		ss->security.ses = sa->sec_session;
 		ss->security.ctx = sa->security_ctx;
 		ss->security.ol_flags = sa->ol_flags;
-	}
+	} else
+		RTE_ASSERT(0);
 
 	rc = rte_ipsec_session_prepare(ss);
 	if (rc != 0)
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index 414fcd2..7fb1929 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -762,11 +762,13 @@ check_eth_dev_caps(uint16_t portid, uint32_t inbound)
 
 static int
 sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries, uint32_t inbound)
+		uint32_t nb_entries, uint32_t inbound,
+		struct socket_ctx *skt_ctx)
 {
 	struct ipsec_sa *sa;
 	uint32_t i, idx;
 	uint16_t iv_length, aad_length;
+	int32_t rc;
 
 	/* for ESN upper 32 bits of SQN also need to be part of AAD */
 	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
@@ -819,6 +821,17 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			sa->xforms = &sa_ctx->xf[idx].a;
 
+			if (sa->type ==
+				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
+				sa->type ==
+				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+				rc = create_inline_session(skt_ctx, sa);
+				if (rc != 0) {
+					RTE_LOG(ERR, IPSEC_ESP,
+						"create_inline_session() failed\n");
+					return -EINVAL;
+				}
+			}
 			print_one_sa_rule(sa, inbound);
 		} else {
 			switch (sa->cipher_algo) {
@@ -894,16 +907,16 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 static inline int
 sa_out_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 0);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx);
 }
 
 static inline int
 sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 1);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx);
 }
 
 /*
@@ -997,10 +1010,12 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm, const struct ipsec_sa *ss,
 	return 0;
 }
 
-static void
+static int
 fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 	const struct ipsec_sa *lsa)
 {
+	int32_t rc = 0;
+
 	ss->sa = sa;
 	ss->type = lsa->type;
 
@@ -1013,6 +1028,17 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 		ss->security.ctx = lsa->security_ctx;
 		ss->security.ol_flags = lsa->ol_flags;
 	}
+
+	if (ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+		ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
+		if (ss->security.ses != NULL) {
+			rc = rte_ipsec_session_prepare(ss);
+			if (rc != 0)
+				memset(ss, 0, sizeof(*ss));
+		}
+	}
+
+	return rc;
 }
 
 /*
@@ -1047,8 +1073,8 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	fill_ipsec_session(&lsa->ips, sa, lsa);
-	return 0;
+	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
+	return rc;
 }
 
 /*
@@ -1126,7 +1152,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in);
+		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx);
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_in, sa_in, nb_sa_in,
@@ -1146,7 +1172,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out);
+		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx);
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_out, sa_out, nb_sa_out,
-- 
2.7.4

[dpdk-dev] [PATCH v2 2/2] examples/ipsec-secgw/test: fix inline test scripts

[Bernard Iremonger]

Remove workaround in tun_aesgcm_defs.sh and trs_aesgcm_defs.sh
to get around the bug where the first inbound packet is dropped
for inline crypto.

Fixes: 929784452094 ("examples/ipsec-secgw: add scripts for functional test")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh | 10 ----------
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh | 10 ----------
 2 files changed, 20 deletions(-)

diff --git a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
index a4d902b..8382d3d 100644
--- a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
@@ -33,11 +33,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -68,9 +63,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV6}
 }
diff --git a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
index 1764ef6..8ae6532 100644
--- a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
@@ -35,11 +35,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -72,9 +67,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping6 -c 1 ${LOCAL_IPV6}
 }
-- 
2.7.4

Re: [dpdk-dev] [PATCH v2 0/2] examples/ipsec-secgw: fix 1st pkt dropped

[Ananyev, Konstantin]

> -----Original Message-----
> From: Iremonger, Bernard
> Sent: Thursday, March 7, 2019 2:58 PM
> To: dev@dpdk.org; Ananyev, Konstantin <konstantin.ananyev@intel.com>; akhil.goyal@nxp.com
> Cc: Iremonger, Bernard <bernard.iremonger@intel.com>
> Subject: [PATCH v2 0/2] examples/ipsec-secgw: fix 1st pkt dropped
> 
> This patchset fixes the issue of the first inbound packet
> being dropped for inline crypto.
> 
> Changes in v2:
> -------------
> The first three patches of the v1 have been squashed.
> The commit message for the squashed patch has been updated.
> Patches 4,5 and 6 of the v1 have been dropped from this patchset.
> A patch to fix the test scripts has been added.
> 
> Bernard Iremonger (2):
>   examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
>   examples/ipsec-secgw/test: fix inline test scripts
> 
>  examples/ipsec-secgw/ipsec-secgw.c           | 243 +++++++++++++--------------
>  examples/ipsec-secgw/ipsec.c                 | 123 +++++++++-----
>  examples/ipsec-secgw/ipsec.h                 |   5 +-
>  examples/ipsec-secgw/ipsec_process.c         |   9 +-
>  examples/ipsec-secgw/sa.c                    |  46 +++--
>  examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 --
>  examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 --
>  7 files changed, 246 insertions(+), 200 deletions(-)
> 
> --

Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>

> 2.7.4

Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Akhil Goyal]

Hi Bernard,

On 3/7/2019 8:27 PM, Bernard Iremonger wrote:
> Inline crypto installs a flow rule in the NIC. This flow
> rule must be installed before the first inbound packet is
> received.
>
> The create_session() function installs the flow rule,
> create_session() has been refactored into create_inline_session()
> and create_lookaside_session(). The create_inline_session() function
> uses the socket_ctx data and is now called at initialisation in
> sa_add_rules().
why do we need a separate function for session creation for inline and 
lookaside cases?
Why can't we initialize the sessions on sa_init in both the cases?
>
> The max_session_size() function has been added to calculate memory
> requirements.
>
> The cryprodev_init() function has been refactored to drop calls to
> rte_mempool_create() and to drop calculation of memory requirements.
>
> The main() function has been refactored to call max_session_size() and
> to call session_pool_init() and session_priv_pool_init() earlier.
> The ports are started now before adding a flow rule in main().
> The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
> now called after the ports have been started.
>
> The rte_ipsec_session_prepare() function is called in fill_ipsec_session()
> for inline which is called from the ipsec_sa_init() function.
>
> Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
> Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
> Cc: stable@dpdk.org
>
> Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> ---
>   examples/ipsec-secgw/ipsec-secgw.c   | 243 +++++++++++++++++------------------
>   examples/ipsec-secgw/ipsec.c         | 123 ++++++++++++------
>   examples/ipsec-secgw/ipsec.h         |   5 +-
>   examples/ipsec-secgw/ipsec_process.c |   9 +-
>   examples/ipsec-secgw/sa.c            |  46 +++++--
>   5 files changed, 246 insertions(+), 180 deletions(-)
>
>

Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Akhil Goyal]

Hi Bernard,

On 3/7/2019 8:27 PM, Bernard Iremonger wrote:
> Inline crypto installs a flow rule in the NIC. This flow
> rule must be installed before the first inbound packet is
> received.
>
> The create_session() function installs the flow rule,
> create_session() has been refactored into create_inline_session()
> and create_lookaside_session(). The create_inline_session() function
> uses the socket_ctx data and is now called at initialisation in
> sa_add_rules().
why do we need a separate function for session creation for inline and 
lookaside cases?
Why can't we initialize the sessions on sa_init in both the cases?
>
> The max_session_size() function has been added to calculate memory
> requirements.
>
> The cryprodev_init() function has been refactored to drop calls to
> rte_mempool_create() and to drop calculation of memory requirements.
>
> The main() function has been refactored to call max_session_size() and
> to call session_pool_init() and session_priv_pool_init() earlier.
> The ports are started now before adding a flow rule in main().
> The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
> now called after the ports have been started.
>
> The rte_ipsec_session_prepare() function is called in fill_ipsec_session()
> for inline which is called from the ipsec_sa_init() function.
>
> Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
> Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
> Cc: stable@dpdk.org
>
> Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> ---
>   examples/ipsec-secgw/ipsec-secgw.c   | 243 +++++++++++++++++------------------
>   examples/ipsec-secgw/ipsec.c         | 123 ++++++++++++------
>   examples/ipsec-secgw/ipsec.h         |   5 +-
>   examples/ipsec-secgw/ipsec_process.c |   9 +-
>   examples/ipsec-secgw/sa.c            |  46 +++++--
>   5 files changed, 246 insertions(+), 180 deletions(-)
>
>

Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Iremonger, Bernard]

Hi Akhil,

<snip>
> Subject: Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt
> dropped for inline crypto
> 
> Hi Bernard,
> 
> On 3/7/2019 8:27 PM, Bernard Iremonger wrote:
> > Inline crypto installs a flow rule in the NIC. This flow rule must be
> > installed before the first inbound packet is received.
> >
> > The create_session() function installs the flow rule,
> > create_session() has been refactored into create_inline_session() and
> > create_lookaside_session(). The create_inline_session() function uses
> > the socket_ctx data and is now called at initialisation in
> > sa_add_rules().
> why do we need a separate function for session creation for inline and
> lookaside cases?
> Why can't we initialize the sessions on sa_init in both the cases?

For the create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa) function,
all of the required data is available in the in the skt_ctx variable.
The skt_ctx variable is already setup when sa_init() is called.

For the create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa) function,
the required data is available in the ipsec_ctx variable.

The ipsec_ctx variable is not setup when sa_init() is called.
It is setup in the main_loop() function  when the variable qconf is setup.
The main_loop() function is called after the sa_init() function is called.

I hope this  answers your question

<snip>

Regards,

Bernard.

Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Iremonger, Bernard]

Hi Akhil,

<snip>
> Subject: Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt
> dropped for inline crypto
> 
> Hi Bernard,
> 
> On 3/7/2019 8:27 PM, Bernard Iremonger wrote:
> > Inline crypto installs a flow rule in the NIC. This flow rule must be
> > installed before the first inbound packet is received.
> >
> > The create_session() function installs the flow rule,
> > create_session() has been refactored into create_inline_session() and
> > create_lookaside_session(). The create_inline_session() function uses
> > the socket_ctx data and is now called at initialisation in
> > sa_add_rules().
> why do we need a separate function for session creation for inline and
> lookaside cases?
> Why can't we initialize the sessions on sa_init in both the cases?

For the create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa) function,
all of the required data is available in the in the skt_ctx variable.
The skt_ctx variable is already setup when sa_init() is called.

For the create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa) function,
the required data is available in the ipsec_ctx variable.

The ipsec_ctx variable is not setup when sa_init() is called.
It is setup in the main_loop() function  when the variable qconf is setup.
The main_loop() function is called after the sa_init() function is called.

I hope this  answers your question

<snip>

Regards,

Bernard.

Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Akhil Goyal]

Hi Bernard,

On 3/26/2019 3:52 PM, Iremonger, Bernard wrote:
> Hi Akhil,
>
> <snip>
>> Subject: Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt
>> dropped for inline crypto
>>
>> Hi Bernard,
>>
>> On 3/7/2019 8:27 PM, Bernard Iremonger wrote:
>>> Inline crypto installs a flow rule in the NIC. This flow rule must be
>>> installed before the first inbound packet is received.
>>>
>>> The create_session() function installs the flow rule,
>>> create_session() has been refactored into create_inline_session() and
>>> create_lookaside_session(). The create_inline_session() function uses
>>> the socket_ctx data and is now called at initialisation in
>>> sa_add_rules().
>> why do we need a separate function for session creation for inline and
>> lookaside cases?
>> Why can't we initialize the sessions on sa_init in both the cases?
> For the create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa) function,
> all of the required data is available in the in the skt_ctx variable.
> The skt_ctx variable is already setup when sa_init() is called.
>
> For the create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa) function,
> the required data is available in the ipsec_ctx variable.
>
> The ipsec_ctx variable is not setup when sa_init() is called.
> It is setup in the main_loop() function  when the variable qconf is setup.
> The main_loop() function is called after the sa_init() function is called.
>
> I hope this  answers your question
Whatever information that is required for session creation is available 
before we call the main loop() in both the cases.
My point is both the sessions(inline/lookaside) can be init at the same 
position, we do not need to have a separate path for them.
If it is not possible in sa_init(), it may be somewhere else before the 
actual data path is started.

The problem with inline processing is that, h/w need to know the SA 
before the first packet is received. So we cannot init the session on 
receive of first packet. However there is no such limitation in case of 
lookaside, it can be initialized anywhere.

-Akhil

>
> <snip>
>
> Regards,
>
> Bernard.

Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Akhil Goyal]

Hi Bernard,

On 3/26/2019 3:52 PM, Iremonger, Bernard wrote:
> Hi Akhil,
>
> <snip>
>> Subject: Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt
>> dropped for inline crypto
>>
>> Hi Bernard,
>>
>> On 3/7/2019 8:27 PM, Bernard Iremonger wrote:
>>> Inline crypto installs a flow rule in the NIC. This flow rule must be
>>> installed before the first inbound packet is received.
>>>
>>> The create_session() function installs the flow rule,
>>> create_session() has been refactored into create_inline_session() and
>>> create_lookaside_session(). The create_inline_session() function uses
>>> the socket_ctx data and is now called at initialisation in
>>> sa_add_rules().
>> why do we need a separate function for session creation for inline and
>> lookaside cases?
>> Why can't we initialize the sessions on sa_init in both the cases?
> For the create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa) function,
> all of the required data is available in the in the skt_ctx variable.
> The skt_ctx variable is already setup when sa_init() is called.
>
> For the create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa) function,
> the required data is available in the ipsec_ctx variable.
>
> The ipsec_ctx variable is not setup when sa_init() is called.
> It is setup in the main_loop() function  when the variable qconf is setup.
> The main_loop() function is called after the sa_init() function is called.
>
> I hope this  answers your question
Whatever information that is required for session creation is available 
before we call the main loop() in both the cases.
My point is both the sessions(inline/lookaside) can be init at the same 
position, we do not need to have a separate path for them.
If it is not possible in sa_init(), it may be somewhere else before the 
actual data path is started.

The problem with inline processing is that, h/w need to know the SA 
before the first packet is received. So we cannot init the session on 
receive of first packet. However there is no such limitation in case of 
lookaside, it can be initialized anywhere.

-Akhil

>
> <snip>
>
> Regards,
>
> Bernard.

Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Iremonger, Bernard]

Hi Akhil,

<snip>

> >> Subject: Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st
> >> pkt dropped for inline crypto
> >>
> >> Hi Bernard,
> >>
> >> On 3/7/2019 8:27 PM, Bernard Iremonger wrote:
> >>> Inline crypto installs a flow rule in the NIC. This flow rule must
> >>> be installed before the first inbound packet is received.
> >>>
> >>> The create_session() function installs the flow rule,
> >>> create_session() has been refactored into create_inline_session()
> >>> and create_lookaside_session(). The create_inline_session() function
> >>> uses the socket_ctx data and is now called at initialisation in
> >>> sa_add_rules().
> >> why do we need a separate function for session creation for inline
> >> and lookaside cases?
> >> Why can't we initialize the sessions on sa_init in both the cases?
> > For the create_inline_session(struct socket_ctx *skt_ctx, struct
> > ipsec_sa *sa) function, all of the required data is available in the in the
> skt_ctx variable.
> > The skt_ctx variable is already setup when sa_init() is called.
> >
> > For the create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct
> > ipsec_sa *sa) function, the required data is available in the ipsec_ctx
> variable.
> >
> > The ipsec_ctx variable is not setup when sa_init() is called.
> > It is setup in the main_loop() function  when the variable qconf is setup.
> > The main_loop() function is called after the sa_init() function is called.
> >
> > I hope this  answers your question
> Whatever information that is required for session creation is available before
> we call the main loop() in both the cases.
> My point is both the sessions(inline/lookaside) can be init at the same
> position, we do not need to have a separate path for them.
> If it is not possible in sa_init(), it may be somewhere else before the actual
> data path is started.
> 
> The problem with inline processing is that, h/w need to know the SA before
> the first packet is received. So we cannot init the session on receive of first
> packet. However there is no such limitation in case of lookaside, it can be
> initialized anywhere.
> 
> -Akhil

This patch is intended to fix the bug in the inline processing, which is that  the flow rule must be installed, before the first packet is received while leaving the lookaside processing as it was originally.

It is not intended to refactor the lookaside processing.

Regards,

Bernard

Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Iremonger, Bernard]

Hi Akhil,

<snip>

> >> Subject: Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st
> >> pkt dropped for inline crypto
> >>
> >> Hi Bernard,
> >>
> >> On 3/7/2019 8:27 PM, Bernard Iremonger wrote:
> >>> Inline crypto installs a flow rule in the NIC. This flow rule must
> >>> be installed before the first inbound packet is received.
> >>>
> >>> The create_session() function installs the flow rule,
> >>> create_session() has been refactored into create_inline_session()
> >>> and create_lookaside_session(). The create_inline_session() function
> >>> uses the socket_ctx data and is now called at initialisation in
> >>> sa_add_rules().
> >> why do we need a separate function for session creation for inline
> >> and lookaside cases?
> >> Why can't we initialize the sessions on sa_init in both the cases?
> > For the create_inline_session(struct socket_ctx *skt_ctx, struct
> > ipsec_sa *sa) function, all of the required data is available in the in the
> skt_ctx variable.
> > The skt_ctx variable is already setup when sa_init() is called.
> >
> > For the create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct
> > ipsec_sa *sa) function, the required data is available in the ipsec_ctx
> variable.
> >
> > The ipsec_ctx variable is not setup when sa_init() is called.
> > It is setup in the main_loop() function  when the variable qconf is setup.
> > The main_loop() function is called after the sa_init() function is called.
> >
> > I hope this  answers your question
> Whatever information that is required for session creation is available before
> we call the main loop() in both the cases.
> My point is both the sessions(inline/lookaside) can be init at the same
> position, we do not need to have a separate path for them.
> If it is not possible in sa_init(), it may be somewhere else before the actual
> data path is started.
> 
> The problem with inline processing is that, h/w need to know the SA before
> the first packet is received. So we cannot init the session on receive of first
> packet. However there is no such limitation in case of lookaside, it can be
> initialized anywhere.
> 
> -Akhil

This patch is intended to fix the bug in the inline processing, which is that  the flow rule must be installed, before the first packet is received while leaving the lookaside processing as it was originally.

It is not intended to refactor the lookaside processing.

Regards,

Bernard

Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Akhil Goyal]

Hi Bernard,

On 3/26/2019 5:11 PM, Iremonger, Bernard wrote:
> Hi Akhil,
>
> <snip>
>
>>>> Subject: Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st
>>>> pkt dropped for inline crypto
>>>>
>>>> Hi Bernard,
>>>>
>>>> On 3/7/2019 8:27 PM, Bernard Iremonger wrote:
>>>>> Inline crypto installs a flow rule in the NIC. This flow rule must
>>>>> be installed before the first inbound packet is received.
>>>>>
>>>>> The create_session() function installs the flow rule,
>>>>> create_session() has been refactored into create_inline_session()
>>>>> and create_lookaside_session(). The create_inline_session() function
>>>>> uses the socket_ctx data and is now called at initialisation in
>>>>> sa_add_rules().
>>>> why do we need a separate function for session creation for inline
>>>> and lookaside cases?
>>>> Why can't we initialize the sessions on sa_init in both the cases?
>>> For the create_inline_session(struct socket_ctx *skt_ctx, struct
>>> ipsec_sa *sa) function, all of the required data is available in the in the
>> skt_ctx variable.
>>> The skt_ctx variable is already setup when sa_init() is called.
>>>
>>> For the create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct
>>> ipsec_sa *sa) function, the required data is available in the ipsec_ctx
>> variable.
>>> The ipsec_ctx variable is not setup when sa_init() is called.
>>> It is setup in the main_loop() function  when the variable qconf is setup.
>>> The main_loop() function is called after the sa_init() function is called.
>>>
>>> I hope this  answers your question
>> Whatever information that is required for session creation is available before
>> we call the main loop() in both the cases.
>> My point is both the sessions(inline/lookaside) can be init at the same
>> position, we do not need to have a separate path for them.
>> If it is not possible in sa_init(), it may be somewhere else before the actual
>> data path is started.
>>
>> The problem with inline processing is that, h/w need to know the SA before
>> the first packet is received. So we cannot init the session on receive of first
>> packet. However there is no such limitation in case of lookaside, it can be
>> initialized anywhere.
>>
>> -Akhil
> This patch is intended to fix the bug in the inline processing, which is that  the flow rule must be installed, before the first packet is received while leaving the lookaside processing as it was originally.
>
> It is not intended to refactor the lookaside processing.
The fix provided should not make the code complex and difficult to 
understand when we have the option available to fix that in a simpler 
way. Application is already complex enough(4 action types) and by adding 
separate code for session init at two different places will be difficult 
to maintain in future and will reduce the code readability.

-Akhil

> Regards,
>
> Bernard

Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Akhil Goyal]

Hi Bernard,

On 3/26/2019 5:11 PM, Iremonger, Bernard wrote:
> Hi Akhil,
>
> <snip>
>
>>>> Subject: Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st
>>>> pkt dropped for inline crypto
>>>>
>>>> Hi Bernard,
>>>>
>>>> On 3/7/2019 8:27 PM, Bernard Iremonger wrote:
>>>>> Inline crypto installs a flow rule in the NIC. This flow rule must
>>>>> be installed before the first inbound packet is received.
>>>>>
>>>>> The create_session() function installs the flow rule,
>>>>> create_session() has been refactored into create_inline_session()
>>>>> and create_lookaside_session(). The create_inline_session() function
>>>>> uses the socket_ctx data and is now called at initialisation in
>>>>> sa_add_rules().
>>>> why do we need a separate function for session creation for inline
>>>> and lookaside cases?
>>>> Why can't we initialize the sessions on sa_init in both the cases?
>>> For the create_inline_session(struct socket_ctx *skt_ctx, struct
>>> ipsec_sa *sa) function, all of the required data is available in the in the
>> skt_ctx variable.
>>> The skt_ctx variable is already setup when sa_init() is called.
>>>
>>> For the create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct
>>> ipsec_sa *sa) function, the required data is available in the ipsec_ctx
>> variable.
>>> The ipsec_ctx variable is not setup when sa_init() is called.
>>> It is setup in the main_loop() function  when the variable qconf is setup.
>>> The main_loop() function is called after the sa_init() function is called.
>>>
>>> I hope this  answers your question
>> Whatever information that is required for session creation is available before
>> we call the main loop() in both the cases.
>> My point is both the sessions(inline/lookaside) can be init at the same
>> position, we do not need to have a separate path for them.
>> If it is not possible in sa_init(), it may be somewhere else before the actual
>> data path is started.
>>
>> The problem with inline processing is that, h/w need to know the SA before
>> the first packet is received. So we cannot init the session on receive of first
>> packet. However there is no such limitation in case of lookaside, it can be
>> initialized anywhere.
>>
>> -Akhil
> This patch is intended to fix the bug in the inline processing, which is that  the flow rule must be installed, before the first packet is received while leaving the lookaside processing as it was originally.
>
> It is not intended to refactor the lookaside processing.
The fix provided should not make the code complex and difficult to 
understand when we have the option available to fix that in a simpler 
way. Application is already complex enough(4 action types) and by adding 
separate code for session init at two different places will be difficult 
to maintain in future and will reduce the code readability.

-Akhil

> Regards,
>
> Bernard

Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Ananyev, Konstantin]

Hi Akhil,

> >
> >>>> Subject: Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st
> >>>> pkt dropped for inline crypto
> >>>>
> >>>> Hi Bernard,
> >>>>
> >>>> On 3/7/2019 8:27 PM, Bernard Iremonger wrote:
> >>>>> Inline crypto installs a flow rule in the NIC. This flow rule must
> >>>>> be installed before the first inbound packet is received.
> >>>>>
> >>>>> The create_session() function installs the flow rule,
> >>>>> create_session() has been refactored into create_inline_session()
> >>>>> and create_lookaside_session(). The create_inline_session() function
> >>>>> uses the socket_ctx data and is now called at initialisation in
> >>>>> sa_add_rules().
> >>>> why do we need a separate function for session creation for inline
> >>>> and lookaside cases?
> >>>> Why can't we initialize the sessions on sa_init in both the cases?
> >>> For the create_inline_session(struct socket_ctx *skt_ctx, struct
> >>> ipsec_sa *sa) function, all of the required data is available in the in the
> >> skt_ctx variable.
> >>> The skt_ctx variable is already setup when sa_init() is called.
> >>>
> >>> For the create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct
> >>> ipsec_sa *sa) function, the required data is available in the ipsec_ctx
> >> variable.
> >>> The ipsec_ctx variable is not setup when sa_init() is called.
> >>> It is setup in the main_loop() function  when the variable qconf is setup.
> >>> The main_loop() function is called after the sa_init() function is called.
> >>>
> >>> I hope this  answers your question
> >> Whatever information that is required for session creation is available before
> >> we call the main loop() in both the cases.
> >> My point is both the sessions(inline/lookaside) can be init at the same
> >> position, we do not need to have a separate path for them.
> >> If it is not possible in sa_init(), it may be somewhere else before the actual
> >> data path is started.
> >>
> >> The problem with inline processing is that, h/w need to know the SA before
> >> the first packet is received. So we cannot init the session on receive of first
> >> packet. However there is no such limitation in case of lookaside, it can be
> >> initialized anywhere.
> >>
> >> -Akhil
> > This patch is intended to fix the bug in the inline processing, which is that  the flow rule must be installed, before the first packet is
> received while leaving the lookaside processing as it was originally.
> >
> > It is not intended to refactor the lookaside processing.
> The fix provided should not make the code complex and difficult to
> understand when we have the option available to fix that in a simpler
> way. Application is already complex enough(4 action types) and by adding
> separate code for session init at two different places will be difficult
> to maintain in future and will reduce the code readability.

It can be changed to do initialization for both security and crypto sessions at sa_init()
time, but it would cause much more changes in the code:
the idea is for each SA go through all available crypto-devices, check their capabilities,
and if they match, we'll invoke init_session() for that SA and device.
That way at the end of sa_init() we'll have crypto-sessions that can be used on any attached
crypto-dev.
As a drawback -  each session might occupy more memory.
Though as I said, it would require more changes (and testing) in init code.
So my thought to have this patch as it is and then add these changes as a separate patch series (phase 2).
What do you think?
Konstantin

Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Ananyev, Konstantin]

Hi Akhil,

> >
> >>>> Subject: Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st
> >>>> pkt dropped for inline crypto
> >>>>
> >>>> Hi Bernard,
> >>>>
> >>>> On 3/7/2019 8:27 PM, Bernard Iremonger wrote:
> >>>>> Inline crypto installs a flow rule in the NIC. This flow rule must
> >>>>> be installed before the first inbound packet is received.
> >>>>>
> >>>>> The create_session() function installs the flow rule,
> >>>>> create_session() has been refactored into create_inline_session()
> >>>>> and create_lookaside_session(). The create_inline_session() function
> >>>>> uses the socket_ctx data and is now called at initialisation in
> >>>>> sa_add_rules().
> >>>> why do we need a separate function for session creation for inline
> >>>> and lookaside cases?
> >>>> Why can't we initialize the sessions on sa_init in both the cases?
> >>> For the create_inline_session(struct socket_ctx *skt_ctx, struct
> >>> ipsec_sa *sa) function, all of the required data is available in the in the
> >> skt_ctx variable.
> >>> The skt_ctx variable is already setup when sa_init() is called.
> >>>
> >>> For the create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct
> >>> ipsec_sa *sa) function, the required data is available in the ipsec_ctx
> >> variable.
> >>> The ipsec_ctx variable is not setup when sa_init() is called.
> >>> It is setup in the main_loop() function  when the variable qconf is setup.
> >>> The main_loop() function is called after the sa_init() function is called.
> >>>
> >>> I hope this  answers your question
> >> Whatever information that is required for session creation is available before
> >> we call the main loop() in both the cases.
> >> My point is both the sessions(inline/lookaside) can be init at the same
> >> position, we do not need to have a separate path for them.
> >> If it is not possible in sa_init(), it may be somewhere else before the actual
> >> data path is started.
> >>
> >> The problem with inline processing is that, h/w need to know the SA before
> >> the first packet is received. So we cannot init the session on receive of first
> >> packet. However there is no such limitation in case of lookaside, it can be
> >> initialized anywhere.
> >>
> >> -Akhil
> > This patch is intended to fix the bug in the inline processing, which is that  the flow rule must be installed, before the first packet is
> received while leaving the lookaside processing as it was originally.
> >
> > It is not intended to refactor the lookaside processing.
> The fix provided should not make the code complex and difficult to
> understand when we have the option available to fix that in a simpler
> way. Application is already complex enough(4 action types) and by adding
> separate code for session init at two different places will be difficult
> to maintain in future and will reduce the code readability.

It can be changed to do initialization for both security and crypto sessions at sa_init()
time, but it would cause much more changes in the code:
the idea is for each SA go through all available crypto-devices, check their capabilities,
and if they match, we'll invoke init_session() for that SA and device.
That way at the end of sa_init() we'll have crypto-sessions that can be used on any attached
crypto-dev.
As a drawback -  each session might occupy more memory.
Though as I said, it would require more changes (and testing) in init code.
So my thought to have this patch as it is and then add these changes as a separate patch series (phase 2).
What do you think?
Konstantin

Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Akhil Goyal]

Hi Konstantin,

On 3/26/2019 5:59 PM, Ananyev, Konstantin wrote:
> Hi Akhil,
>
>>>>>> Subject: Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st
>>>>>> pkt dropped for inline crypto
>>>>>>
>>>>>> Hi Bernard,
>>>>>>
>>>>>> On 3/7/2019 8:27 PM, Bernard Iremonger wrote:
>>>>>>> Inline crypto installs a flow rule in the NIC. This flow rule must
>>>>>>> be installed before the first inbound packet is received.
>>>>>>>
>>>>>>> The create_session() function installs the flow rule,
>>>>>>> create_session() has been refactored into create_inline_session()
>>>>>>> and create_lookaside_session(). The create_inline_session() function
>>>>>>> uses the socket_ctx data and is now called at initialisation in
>>>>>>> sa_add_rules().
>>>>>> why do we need a separate function for session creation for inline
>>>>>> and lookaside cases?
>>>>>> Why can't we initialize the sessions on sa_init in both the cases?
>>>>> For the create_inline_session(struct socket_ctx *skt_ctx, struct
>>>>> ipsec_sa *sa) function, all of the required data is available in the in the
>>>> skt_ctx variable.
>>>>> The skt_ctx variable is already setup when sa_init() is called.
>>>>>
>>>>> For the create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct
>>>>> ipsec_sa *sa) function, the required data is available in the ipsec_ctx
>>>> variable.
>>>>> The ipsec_ctx variable is not setup when sa_init() is called.
>>>>> It is setup in the main_loop() function  when the variable qconf is setup.
>>>>> The main_loop() function is called after the sa_init() function is called.
>>>>>
>>>>> I hope this  answers your question
>>>> Whatever information that is required for session creation is available before
>>>> we call the main loop() in both the cases.
>>>> My point is both the sessions(inline/lookaside) can be init at the same
>>>> position, we do not need to have a separate path for them.
>>>> If it is not possible in sa_init(), it may be somewhere else before the actual
>>>> data path is started.
>>>>
>>>> The problem with inline processing is that, h/w need to know the SA before
>>>> the first packet is received. So we cannot init the session on receive of first
>>>> packet. However there is no such limitation in case of lookaside, it can be
>>>> initialized anywhere.
>>>>
>>>> -Akhil
>>> This patch is intended to fix the bug in the inline processing, which is that  the flow rule must be installed, before the first packet is
>> received while leaving the lookaside processing as it was originally.
>>> It is not intended to refactor the lookaside processing.
>> The fix provided should not make the code complex and difficult to
>> understand when we have the option available to fix that in a simpler
>> way. Application is already complex enough(4 action types) and by adding
>> separate code for session init at two different places will be difficult
>> to maintain in future and will reduce the code readability.
> It can be changed to do initialization for both security and crypto sessions at sa_init()
> time, but it would cause much more changes in the code:
> the idea is for each SA go through all available crypto-devices, check their capabilities,
> and if they match, we'll invoke init_session() for that SA and device.
> That way at the end of sa_init() we'll have crypto-sessions that can be used on any attached
> crypto-dev.
Agreed, my point is we should have same code path for all action types 
as much as possible to make the user comfortable to port their 
applications over DPDK.
> As a drawback -  each session might occupy more memory.
That is still there for the inline cases.
> Though as I said, it would require more changes (and testing) in init code.
> So my thought to have this patch as it is and then add these changes as a separate patch series (phase 2).
> What do you think?
The code changes are still there in this patch as well. And the testing 
cycle has not started yet for this release.

5 files changed, 246 insertions(+), 180 deletions(-)

If we intend to defer it, we will have similar code movement again. So IMO, it is good that we do the changes in one go.
This is an application code and we can have it merged in RC2 as well.

> Konstantin
>

Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Akhil Goyal]

Hi Konstantin,

On 3/26/2019 5:59 PM, Ananyev, Konstantin wrote:
> Hi Akhil,
>
>>>>>> Subject: Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st
>>>>>> pkt dropped for inline crypto
>>>>>>
>>>>>> Hi Bernard,
>>>>>>
>>>>>> On 3/7/2019 8:27 PM, Bernard Iremonger wrote:
>>>>>>> Inline crypto installs a flow rule in the NIC. This flow rule must
>>>>>>> be installed before the first inbound packet is received.
>>>>>>>
>>>>>>> The create_session() function installs the flow rule,
>>>>>>> create_session() has been refactored into create_inline_session()
>>>>>>> and create_lookaside_session(). The create_inline_session() function
>>>>>>> uses the socket_ctx data and is now called at initialisation in
>>>>>>> sa_add_rules().
>>>>>> why do we need a separate function for session creation for inline
>>>>>> and lookaside cases?
>>>>>> Why can't we initialize the sessions on sa_init in both the cases?
>>>>> For the create_inline_session(struct socket_ctx *skt_ctx, struct
>>>>> ipsec_sa *sa) function, all of the required data is available in the in the
>>>> skt_ctx variable.
>>>>> The skt_ctx variable is already setup when sa_init() is called.
>>>>>
>>>>> For the create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct
>>>>> ipsec_sa *sa) function, the required data is available in the ipsec_ctx
>>>> variable.
>>>>> The ipsec_ctx variable is not setup when sa_init() is called.
>>>>> It is setup in the main_loop() function  when the variable qconf is setup.
>>>>> The main_loop() function is called after the sa_init() function is called.
>>>>>
>>>>> I hope this  answers your question
>>>> Whatever information that is required for session creation is available before
>>>> we call the main loop() in both the cases.
>>>> My point is both the sessions(inline/lookaside) can be init at the same
>>>> position, we do not need to have a separate path for them.
>>>> If it is not possible in sa_init(), it may be somewhere else before the actual
>>>> data path is started.
>>>>
>>>> The problem with inline processing is that, h/w need to know the SA before
>>>> the first packet is received. So we cannot init the session on receive of first
>>>> packet. However there is no such limitation in case of lookaside, it can be
>>>> initialized anywhere.
>>>>
>>>> -Akhil
>>> This patch is intended to fix the bug in the inline processing, which is that  the flow rule must be installed, before the first packet is
>> received while leaving the lookaside processing as it was originally.
>>> It is not intended to refactor the lookaside processing.
>> The fix provided should not make the code complex and difficult to
>> understand when we have the option available to fix that in a simpler
>> way. Application is already complex enough(4 action types) and by adding
>> separate code for session init at two different places will be difficult
>> to maintain in future and will reduce the code readability.
> It can be changed to do initialization for both security and crypto sessions at sa_init()
> time, but it would cause much more changes in the code:
> the idea is for each SA go through all available crypto-devices, check their capabilities,
> and if they match, we'll invoke init_session() for that SA and device.
> That way at the end of sa_init() we'll have crypto-sessions that can be used on any attached
> crypto-dev.
Agreed, my point is we should have same code path for all action types 
as much as possible to make the user comfortable to port their 
applications over DPDK.
> As a drawback -  each session might occupy more memory.
That is still there for the inline cases.
> Though as I said, it would require more changes (and testing) in init code.
> So my thought to have this patch as it is and then add these changes as a separate patch series (phase 2).
> What do you think?
The code changes are still there in this patch as well. And the testing 
cycle has not started yet for this release.

5 files changed, 246 insertions(+), 180 deletions(-)

If we intend to defer it, we will have similar code movement again. So IMO, it is good that we do the changes in one go.
This is an application code and we can have it merged in RC2 as well.

> Konstantin
>

[dpdk-dev] [PATCH v3 0/2] examples/ipsec-secgw: fix 1st pkt dropped

[Bernard Iremonger]

This patchset fixes the issue of the first inbound packet
being dropped for inline crypto. 
 
Changes in v3:
--------------
The previous refactoring of the create_session() function has been dropped.
The create_session() function is now called from sa_init() at startup.

The following functions have been added:
crypto_devid_fill() in ipsec-secgw.c
check_cryptodev_capability() in ipsec.c
check_cryptodev_aead_capability() in ipsec.c
create_sec_session() and create_crypto_session() in ipsec.c

The create_session() function has been refactored to call
the create_sec_session() and create_crypto_session() functions.


Changes in v2: 
--------------
The first three patches of the v1 have been squashed.
The commit message for the squashed patch has been updated.
Patches 4,5 and 6 of the v1 have been dropped from this patchset.
A patch to fix the test scripts has been added.

Bernard Iremonger (2):
  examples/ipsec-secgw: fix 1st packet dropped for inline crypto
  examples/ipsec-secgw/test: fix inline test scripts

 examples/ipsec-secgw/ipsec-secgw.c           | 271 +++++++------
 examples/ipsec-secgw/ipsec.c                 | 569 ++++++++++++++-------------
 examples/ipsec-secgw/ipsec.h                 |  10 +-
 examples/ipsec-secgw/ipsec_process.c         |  38 +-
 examples/ipsec-secgw/sa.c                    |  42 +-
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 -
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 -
 7 files changed, 495 insertions(+), 455 deletions(-)

-- 
2.7.4

[dpdk-dev] [PATCH v3 0/2] examples/ipsec-secgw: fix 1st pkt dropped

[Bernard Iremonger]

This patchset fixes the issue of the first inbound packet
being dropped for inline crypto. 
 
Changes in v3:
--------------
The previous refactoring of the create_session() function has been dropped.
The create_session() function is now called from sa_init() at startup.

The following functions have been added:
crypto_devid_fill() in ipsec-secgw.c
check_cryptodev_capability() in ipsec.c
check_cryptodev_aead_capability() in ipsec.c
create_sec_session() and create_crypto_session() in ipsec.c

The create_session() function has been refactored to call
the create_sec_session() and create_crypto_session() functions.


Changes in v2: 
--------------
The first three patches of the v1 have been squashed.
The commit message for the squashed patch has been updated.
Patches 4,5 and 6 of the v1 have been dropped from this patchset.
A patch to fix the test scripts has been added.

Bernard Iremonger (2):
  examples/ipsec-secgw: fix 1st packet dropped for inline crypto
  examples/ipsec-secgw/test: fix inline test scripts

 examples/ipsec-secgw/ipsec-secgw.c           | 271 +++++++------
 examples/ipsec-secgw/ipsec.c                 | 569 ++++++++++++++-------------
 examples/ipsec-secgw/ipsec.h                 |  10 +-
 examples/ipsec-secgw/ipsec_process.c         |  38 +-
 examples/ipsec-secgw/sa.c                    |  42 +-
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 -
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 -
 7 files changed, 495 insertions(+), 455 deletions(-)

-- 
2.7.4

[dpdk-dev] [PATCH v3 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Bernard Iremonger]

Inline crypto installs a flow rule in the NIC. This flow
rule must be installed before the first inbound packet is
received.

The create_session() function installs the flow rule.

Refactor ipsec-secgw.c, sa.c, ipsec.h and ipsec.c to create
sessions at startup to fix the issue of the first packet
being dropped for inline crypto.

The create_session() function is now called at initialisation in
sa_add_rules() which is called from sa_init().
The return code for add_rules is checked.
Calls to create_session() in other functions are dropped.

Add crypto_devid_fill() in ipsec-secgw.c
Add max_session_size() in ipsec-secgw.c
Add check_cryptodev_capability() in ipsec.c
Add check_cryptodev_aead_capability() in ipsec.c
Add create_sec_session() and create_crypto_session() in ipsec.c

The crypto_dev_fill() function has been added to find the
enabled crypto devices.

The max_session_size() function has been added to calculate memory
requirements.

The check_cryptodev_capability() and check_cryptodev_aead_capability()
functions have been added to check that the SA is supported by the
crypto device.

The create_session() function is refactored to use the
create_sec_session() and create_crypto_session() functions.

The cryprodev_init() function has been refactored to drop calls to
rte_mempool_create() and to drop calculation of memory requirements.

The main() function has been refactored to call crypto_devid_fill()
and max_session_size() and to call session_pool_init() and
session_priv_pool_init().
The ports are started now before adding a flow rule in main().
The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
now called after the ports have been started.

Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/ipsec-secgw.c   | 271 +++++++++--------
 examples/ipsec-secgw/ipsec.c         | 569 +++++++++++++++++++----------------
 examples/ipsec-secgw/ipsec.h         |  10 +-
 examples/ipsec-secgw/ipsec_process.c |  38 +--
 examples/ipsec-secgw/sa.c            |  42 ++-
 5 files changed, 495 insertions(+), 435 deletions(-)

diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index ffbd00b..cc8bb57 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -182,6 +182,14 @@ struct lcore_params {
 	uint8_t lcore_id;
 } __rte_cache_aligned;
 
+/*
+ * Number of enabled crypto devices
+ * This number is needed when checking crypto device capabilities
+ */
+uint8_t crypto_dev_num;
+/* array of crypto device ID's */
+uint8_t crypto_devid[RTE_CRYPTO_MAX_DEVS];
+
 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS];
 
 static struct lcore_params *lcore_params;
@@ -1623,13 +1631,27 @@ check_cryptodev_mask(uint8_t cdev_id)
 	return -1;
 }
 
+static void
+crypto_devid_fill(void)
+{
+	uint32_t i, n;
+
+	n = rte_cryptodev_count();
+
+	for (i = 0; i != n; i++) {
+		if (check_cryptodev_mask(i) == 0)
+			crypto_devid[crypto_dev_num++] = i;
+	}
+}
+
 static int32_t
 cryptodevs_init(void)
 {
 	struct rte_cryptodev_config dev_conf;
 	struct rte_cryptodev_qp_conf qp_conf;
 	uint16_t idx, max_nb_qps, qp, i;
-	int16_t cdev_id, port_id;
+	int16_t cdev_id;
+	uint32_t dev_max_sess;
 	struct rte_hash_parameters params = { 0 };
 
 	params.entries = CDEV_MAP_ENTRIES;
@@ -1652,45 +1674,6 @@ cryptodevs_init(void)
 
 	printf("lcore/cryptodev/qp mappings:\n");
 
-	uint32_t max_sess_sz = 0, sess_sz;
-	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
-		void *sec_ctx;
-
-		/* Get crypto priv session size */
-		sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-
-		/*
-		 * If crypto device is security capable, need to check the
-		 * size of security session as well.
-		 */
-
-		/* Get security context of the crypto device */
-		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		/* Get size of security session */
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-	RTE_ETH_FOREACH_DEV(port_id) {
-		void *sec_ctx;
-
-		if ((enabled_port_mask & (1 << port_id)) == 0)
-			continue;
-
-		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-
 	idx = 0;
 	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
 		struct rte_cryptodev_info cdev_info;
@@ -1722,51 +1705,12 @@ cryptodevs_init(void)
 		dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id);
 		dev_conf.nb_queue_pairs = qp;
 
-		uint32_t dev_max_sess = cdev_info.sym.max_nb_sessions;
+		dev_max_sess = cdev_info.sym.max_nb_sessions;
 		if (dev_max_sess != 0 && dev_max_sess < CDEV_MP_NB_OBJS)
 			rte_exit(EXIT_FAILURE,
 				"Device does not support at least %u "
 				"sessions", CDEV_MP_NB_OBJS);
 
-		if (!socket_ctx[dev_conf.socket_id].session_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_%u", dev_conf.socket_id);
-			sess_mp = rte_cryptodev_sym_session_pool_create(
-					mp_name, CDEV_MP_NB_OBJS,
-					0, CDEV_MP_CACHE_SZ, 0,
-					dev_conf.socket_id);
-			socket_ctx[dev_conf.socket_id].session_pool = sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_priv_%u", dev_conf.socket_id);
-			sess_mp = rte_mempool_create(mp_name,
-					CDEV_MP_NB_OBJS,
-					max_sess_sz,
-					CDEV_MP_CACHE_SZ,
-					0, NULL, NULL, NULL,
-					NULL, dev_conf.socket_id,
-					0);
-			socket_ctx[dev_conf.socket_id].session_priv_pool =
-					sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool ||
-				!socket_ctx[dev_conf.socket_id].session_pool)
-			rte_exit(EXIT_FAILURE,
-				"Cannot create session pool on socket %d\n",
-				dev_conf.socket_id);
-		else
-			printf("Allocated session pool on socket %d\n",
-					dev_conf.socket_id);
-
 		if (rte_cryptodev_configure(cdev_id, &dev_conf))
 			rte_panic("Failed to initialize cryptodev %u\n",
 					cdev_id);
@@ -1787,38 +1731,6 @@ cryptodevs_init(void)
 					cdev_id);
 	}
 
-	/* create session pools for eth devices that implement security */
-	RTE_ETH_FOREACH_DEV(port_id) {
-		if ((enabled_port_mask & (1 << port_id)) &&
-				rte_eth_dev_get_sec_ctx(port_id)) {
-			int socket_id = rte_eth_dev_socket_id(port_id);
-
-			if (!socket_ctx[socket_id].session_pool) {
-				char mp_name[RTE_MEMPOOL_NAMESIZE];
-				struct rte_mempool *sess_mp;
-
-				snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-						"sess_mp_%u", socket_id);
-				sess_mp = rte_mempool_create(mp_name,
-						(CDEV_MP_NB_OBJS * 2),
-						max_sess_sz,
-						CDEV_MP_CACHE_SZ,
-						0, NULL, NULL, NULL,
-						NULL, socket_id,
-						0);
-				if (sess_mp == NULL)
-					rte_exit(EXIT_FAILURE,
-						"Cannot create session pool "
-						"on socket %d\n", socket_id);
-				else
-					printf("Allocated session pool "
-						"on socket %d\n", socket_id);
-				socket_ctx[socket_id].session_pool = sess_mp;
-			}
-		}
-	}
-
-
 	printf("\n");
 
 	return 0;
@@ -1984,6 +1896,98 @@ port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads)
 	printf("\n");
 }
 
+static size_t
+max_session_size(void)
+{
+	size_t max_sz, sz;
+	void *sec_ctx;
+	int16_t cdev_id, port_id, n;
+
+	max_sz = 0;
+	n =  rte_cryptodev_count();
+	for (cdev_id = 0; cdev_id != n; cdev_id++) {
+		sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
+		if (sz > max_sz)
+			max_sz = sz;
+		/*
+		 * If crypto device is security capable, need to check the
+		 * size of security session as well.
+		 */
+
+		/* Get security context of the crypto device */
+		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		/* Get size of security session */
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	RTE_ETH_FOREACH_DEV(port_id) {
+		if ((enabled_port_mask & (1 << port_id)) == 0)
+			continue;
+
+		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	return max_sz;
+}
+
+static void
+session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_%u", socket_id);
+	sess_mp = rte_cryptodev_sym_session_pool_create(
+			mp_name, CDEV_MP_NB_OBJS,
+			sess_sz, CDEV_MP_CACHE_SZ, 0,
+			socket_id);
+	ctx->session_pool = sess_mp;
+
+	if (ctx->session_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session pool on socket %d\n", socket_id);
+	else
+		printf("Allocated session pool on socket %d\n",	socket_id);
+}
+
+static void
+session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
+	size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_priv_%u", socket_id);
+	sess_mp = rte_mempool_create(mp_name,
+			CDEV_MP_NB_OBJS,
+			sess_sz,
+			CDEV_MP_CACHE_SZ,
+			0, NULL, NULL, NULL,
+			NULL, socket_id,
+			0);
+	ctx->session_priv_pool = sess_mp;
+	if (ctx->session_priv_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session priv pool on socket %d\n",
+			socket_id);
+	else
+		printf("Allocated session priv pool on socket %d\n",
+			socket_id);
+}
+
 static void
 pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf)
 {
@@ -2064,9 +2068,11 @@ main(int32_t argc, char **argv)
 {
 	int32_t ret;
 	uint32_t lcore_id;
+	uint32_t i;
 	uint8_t socket_id;
 	uint16_t portid;
 	uint64_t req_rx_offloads, req_tx_offloads;
+	size_t sess_sz;
 
 	/* init EAL */
 	ret = rte_eal_init(argc, argv);
@@ -2094,7 +2100,10 @@ main(int32_t argc, char **argv)
 
 	nb_lcores = rte_lcore_count();
 
-	/* Replicate each context per socket */
+	crypto_devid_fill();
+
+	sess_sz = max_session_size();
+
 	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
 		if (rte_lcore_is_enabled(lcore_id) == 0)
 			continue;
@@ -2104,20 +2113,17 @@ main(int32_t argc, char **argv)
 		else
 			socket_id = 0;
 
+		/* mbuf_pool is initialised by the pool_init() function*/
 		if (socket_ctx[socket_id].mbuf_pool)
 			continue;
 
-		/* initilaze SPD */
-		sp4_init(&socket_ctx[socket_id], socket_id);
-
-		sp6_init(&socket_ctx[socket_id], socket_id);
-
-		/* initilaze SAD */
-		sa_init(&socket_ctx[socket_id], socket_id);
-
-		rt_init(&socket_ctx[socket_id], socket_id);
-
 		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
+		session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
+		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
+			sess_sz);
+
+		if (!numa_on)
+			break;
 	}
 
 	RTE_ETH_FOREACH_DEV(portid) {
@@ -2135,7 +2141,11 @@ main(int32_t argc, char **argv)
 		if ((enabled_port_mask & (1 << portid)) == 0)
 			continue;
 
-		/* Start device */
+		/*
+		 * Start device
+		 * note: device must be started before a flow rule
+		 * can be installed.
+		 */
 		ret = rte_eth_dev_start(portid);
 		if (ret < 0)
 			rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
@@ -2153,6 +2163,19 @@ main(int32_t argc, char **argv)
 			RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback, NULL);
 	}
 
+	/* Replicate each context per socket */
+	for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
+		socket_id = rte_socket_id_by_idx(i);
+		if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
+			(socket_ctx[socket_id].sa_in == NULL) &&
+			(socket_ctx[socket_id].sa_out == NULL)) {
+			sa_init(&socket_ctx[socket_id], socket_id);
+			sp4_init(&socket_ctx[socket_id], socket_id);
+			sp6_init(&socket_ctx[socket_id], socket_id);
+			rt_init(&socket_ctx[socket_id], socket_id);
+		}
+	}
+
 	check_all_ports_link_status(enabled_port_mask);
 
 	/* launch per-lcore init on every lcore */
diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index 4352cb8..e31d472 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -39,42 +39,17 @@ set_ipsec_conf(struct ipsec_sa *sa, struct rte_security_ipsec_xform *ipsec)
 	ipsec->esn_soft_limit = IPSEC_OFFLOAD_ESN_SOFTLIMIT;
 }
 
-int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
+static int
+create_sec_session(struct ipsec_sa *sa, struct rte_mempool *pool)
 {
-	struct rte_cryptodev_info cdev_info;
-	unsigned long cdev_id_qp = 0;
+	const struct rte_security_capability *sec_cap;
+	struct rte_security_ctx *ctx;
 	int32_t ret = 0;
-	struct cdev_key key = { 0 };
-
-	key.lcore_id = (uint8_t)rte_lcore_id();
-
-	key.cipher_algo = (uint8_t)sa->cipher_algo;
-	key.auth_algo = (uint8_t)sa->auth_algo;
-	key.aead_algo = (uint8_t)sa->aead_algo;
-
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
-				(void **)&cdev_id_qp);
-		if (ret < 0) {
-			RTE_LOG(ERR, IPSEC,
-				"No cryptodev: core %u, cipher_algo %u, "
-				"auth_algo %u, aead_algo %u\n",
-				key.lcore_id,
-				key.cipher_algo,
-				key.auth_algo,
-				key.aead_algo);
-			return -1;
-		}
-	}
 
-	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on cryptodev "
-			"%u qp %u\n", sa->spi,
-			ipsec_ctx->tbl[cdev_id_qp].id,
-			ipsec_ctx->tbl[cdev_id_qp].qp);
+	if ((sa == NULL) || (pool == NULL))
+		return -EINVAL;
 
-	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
-		struct rte_security_session_conf sess_conf = {
+	struct rte_security_session_conf sess_conf = {
 			.action_type = sa->type,
 			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 			{.ipsec = {
@@ -90,247 +65,340 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			} },
 			.crypto_xform = sa->xforms,
 			.userdata = NULL,
-
 		};
 
-		if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
-			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
-							rte_cryptodev_get_sec_ctx(
-							ipsec_ctx->tbl[cdev_id_qp].id);
-
-			/* Set IPsec parameters in conf */
-			set_ipsec_conf(sa, &(sess_conf.ipsec));
-
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"SEC Session init failed: err: %d\n", ret);
-				return -1;
-			}
-		} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
-			struct rte_flow_error err;
-			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
-							rte_eth_dev_get_sec_ctx(
-							sa->portid);
-			const struct rte_security_capability *sec_cap;
-			int ret = 0;
-
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"SEC Session init failed: err: %d\n", ret);
-				return -1;
-			}
+	if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
+		ctx = (struct rte_security_ctx *)
+				rte_eth_dev_get_sec_ctx(sa->portid);
 
-			sec_cap = rte_security_capabilities_get(ctx);
+		/* Set IPsec parameters in conf */
+		set_ipsec_conf(sa, &(sess_conf.ipsec));
 
-			/* iterate until ESP tunnel*/
-			while (sec_cap->action !=
-					RTE_SECURITY_ACTION_TYPE_NONE) {
+		sa->sec_session = rte_security_session_create(ctx,
+				&sess_conf, pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC,
+				"SEC Session init failed: err: %d\n",
+				ret);
+			return -1;
+		}
+	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		struct rte_flow_error err;
+		ctx = (struct rte_security_ctx *)
+				rte_eth_dev_get_sec_ctx(sa->portid);
+		sa->sec_session = rte_security_session_create(ctx,
+				&sess_conf, pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC, "SEC Session init failed\n");
+			return -1;
+		}
 
-				if (sec_cap->action == sa->type &&
-				    sec_cap->protocol ==
-					RTE_SECURITY_PROTOCOL_IPSEC &&
-				    sec_cap->ipsec.mode ==
-					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
-				    sec_cap->ipsec.direction == sa->direction)
-					break;
-				sec_cap++;
-			}
+		sec_cap = rte_security_capabilities_get(ctx);
+
+		/* iterate until ESP tunnel*/
+		while (sec_cap->action != RTE_SECURITY_ACTION_TYPE_NONE) {
+			if (sec_cap->action == sa->type &&
+				sec_cap->protocol ==
+				RTE_SECURITY_PROTOCOL_IPSEC &&
+				sec_cap->ipsec.mode ==
+				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
+				sec_cap->ipsec.direction == sa->direction)
+				break;
+			sec_cap++;
+		}
 
-			if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
-				RTE_LOG(ERR, IPSEC,
+		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
+			RTE_LOG(ERR, IPSEC,
 				"No suitable security capability found\n");
 				return -1;
-			}
+		}
 
-			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
-			sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
-
-			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
-			sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
-			if (sa->flags & IP6_TUNNEL) {
-				sa->pattern[1].spec = &sa->ipv6_spec;
-				memcpy(sa->ipv6_spec.hdr.dst_addr,
-					sa->dst.ip.ip6.ip6_b, 16);
-				memcpy(sa->ipv6_spec.hdr.src_addr,
-				       sa->src.ip.ip6.ip6_b, 16);
-			} else {
-				sa->pattern[1].spec = &sa->ipv4_spec;
-				sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
-				sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
-			}
+		sa->ol_flags = sec_cap->ol_flags;
+		sa->security_ctx = ctx;
+		sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
+
+		sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
+		sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
+		if (sa->flags & IP6_TUNNEL) {
+			sa->pattern[1].spec = &sa->ipv6_spec;
+			memcpy(sa->ipv6_spec.hdr.dst_addr,
+				sa->dst.ip.ip6.ip6_b, 16);
+			memcpy(sa->ipv6_spec.hdr.src_addr,
+			       sa->src.ip.ip6.ip6_b, 16);
+		} else {
+			sa->pattern[1].spec = &sa->ipv4_spec;
+			sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
+			sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
+		}
 
-			sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
-			sa->pattern[2].spec = &sa->esp_spec;
-			sa->pattern[2].mask = &rte_flow_item_esp_mask;
-			sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
+		sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
+		sa->pattern[2].spec = &sa->esp_spec;
+		sa->pattern[2].mask = &rte_flow_item_esp_mask;
+		sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
 
-			sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
+		sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
 
-			sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
-			sa->action[0].conf = sa->sec_session;
+		sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
+		sa->action[0].conf = sa->sec_session;
 
-			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
+		sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
 
-			sa->attr.egress = (sa->direction ==
+		sa->attr.egress = (sa->direction ==
 					RTE_SECURITY_IPSEC_SA_DIR_EGRESS);
-			sa->attr.ingress = (sa->direction ==
+		sa->attr.ingress = (sa->direction ==
 					RTE_SECURITY_IPSEC_SA_DIR_INGRESS);
-			if (sa->attr.ingress) {
-				uint8_t rss_key[40];
-				struct rte_eth_rss_conf rss_conf = {
-					.rss_key = rss_key,
-					.rss_key_len = 40,
-				};
-				struct rte_eth_dev *eth_dev;
-				uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
-				struct rte_flow_action_rss action_rss;
-				unsigned int i;
-				unsigned int j;
-
-				sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
-				/* Try RSS. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
-				sa->action[1].conf = &action_rss;
-				eth_dev = ctx->device;
-				rte_eth_dev_rss_hash_conf_get(sa->portid,
-							      &rss_conf);
-				for (i = 0, j = 0;
-				     i < eth_dev->data->nb_rx_queues; ++i)
-					if (eth_dev->data->rx_queues[i])
-						queue[j++] = i;
+		if (sa->attr.ingress) {
+			uint8_t rss_key[40];
+			struct rte_eth_rss_conf rss_conf = {
+				.rss_key = rss_key,
+				.rss_key_len = 40,
+			};
+			struct rte_eth_dev *eth_dev;
+			uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
+			struct rte_flow_action_rss action_rss;
+			unsigned int i;
+			unsigned int j;
+
+			sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
+			/* Try RSS. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
+			sa->action[1].conf = &action_rss;
+			eth_dev = ctx->device;
+			rte_eth_dev_rss_hash_conf_get(sa->portid,
+						&rss_conf);
+			for (i = 0, j = 0; i < eth_dev->data->nb_rx_queues;
+				++i)
+				if (eth_dev->data->rx_queues[i])
+					queue[j++] = i;
 				action_rss = (struct rte_flow_action_rss){
 					.types = rss_conf.rss_hf,
 					.key_len = rss_conf.rss_key_len,
 					.queue_num = j,
 					.key = rss_key,
 					.queue = queue,
-				};
-				ret = rte_flow_validate(sa->portid, &sa->attr,
-							sa->pattern, sa->action,
-							&err);
-				if (!ret)
-					goto flow_create;
-				/* Try Queue. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_QUEUE;
-				sa->action[1].conf =
-					&(struct rte_flow_action_queue){
-					.index = 0,
-				};
-				ret = rte_flow_validate(sa->portid, &sa->attr,
-							sa->pattern, sa->action,
-							&err);
-				/* Try End. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
-				sa->action[1].conf = NULL;
-				ret = rte_flow_validate(sa->portid, &sa->attr,
+			};
+			ret = rte_flow_validate(sa->portid, &sa->attr,
+						sa->pattern, sa->action,
+						&err);
+			if (!ret)
+				goto flow_create;
+			/* Try Queue. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_QUEUE;
+			sa->action[1].conf =
+				&(struct rte_flow_action_queue){
+				.index = 0,
+			};
+			ret = rte_flow_validate(sa->portid, &sa->attr,
+						sa->pattern, sa->action,
+						&err);
+			/* Try End. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
+			sa->action[1].conf = NULL;
+			ret = rte_flow_validate(sa->portid, &sa->attr,
 							sa->pattern, sa->action,
 							&err);
-				if (ret)
-					goto flow_create_failure;
-			} else if (sa->attr.egress &&
-				   (sa->ol_flags &
-				    RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
-				sa->action[1].type =
-					RTE_FLOW_ACTION_TYPE_PASSTHRU;
-				sa->action[2].type =
-					RTE_FLOW_ACTION_TYPE_END;
-			}
+			if (ret)
+				goto flow_create_failure;
+		} else if (sa->attr.egress &&
+				(sa->ol_flags &
+				RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
+			sa->action[1].type =
+				RTE_FLOW_ACTION_TYPE_PASSTHRU;
+			sa->action[2].type =
+				RTE_FLOW_ACTION_TYPE_END;
+		}
 flow_create:
-			sa->flow = rte_flow_create(sa->portid,
-				&sa->attr, sa->pattern, sa->action, &err);
-			if (sa->flow == NULL) {
+		sa->flow = rte_flow_create(sa->portid,
+			&sa->attr, sa->pattern, sa->action, &err);
+		if (sa->flow == NULL) {
 flow_create_failure:
-				RTE_LOG(ERR, IPSEC,
-					"Failed to create ipsec flow msg: %s\n",
-					err.message);
-				return -1;
-			}
-		} else if (sa->type ==
-				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
-			struct rte_security_ctx *ctx =
-					(struct rte_security_ctx *)
-					rte_eth_dev_get_sec_ctx(sa->portid);
-			const struct rte_security_capability *sec_cap;
-
-			if (ctx == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"Ethernet device doesn't have security features registered\n");
-				return -1;
-			}
+			RTE_LOG(ERR, IPSEC,
+				"Failed to create ipsec flow msg: %s\n",
+				err.message);
+			return -1;
+		}
+	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
+		struct rte_security_ctx *ctx =
+				(struct rte_security_ctx *)
+				rte_eth_dev_get_sec_ctx(sa->portid);
+		const struct rte_security_capability *sec_cap;
 
-			/* Set IPsec parameters in conf */
-			set_ipsec_conf(sa, &(sess_conf.ipsec));
-
-			/* Save SA as userdata for the security session. When
-			 * the packet is received, this userdata will be
-			 * retrieved using the metadata from the packet.
-			 *
-			 * The PMD is expected to set similar metadata for other
-			 * operations, like rte_eth_event, which are tied to
-			 * security session. In such cases, the userdata could
-			 * be obtained to uniquely identify the security
-			 * parameters denoted.
-			 */
-
-			sess_conf.userdata = (void *) sa;
-
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"SEC Session init failed: err: %d\n", ret);
-				return -1;
-			}
+		if (ctx == NULL) {
+			RTE_LOG(ERR, IPSEC,
+			"Ethernet device doesn't have security features registered\n");
+			return -1;
+		}
+
+		/* Set IPsec parameters in conf */
+		set_ipsec_conf(sa, &(sess_conf.ipsec));
+
+		/* Save SA as userdata for the security session. When
+		 * the packet is received, this userdata will be
+		 * retrieved using the metadata from the packet.
+		 *
+		 * The PMD is expected to set similar metadata for other
+		 * operations, like rte_eth_event, which are tied to
+		 * security session. In such cases, the userdata could
+		 * be obtained to uniquely identify the security
+		 * parameters denoted.
+		 */
+
+		sess_conf.userdata = (void *) sa;
+
+		sa->sec_session = rte_security_session_create(ctx,
+			&sess_conf, pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC,
+			"SEC Session init failed: err: %d\n", ret);
+			return -1;
+		}
 
-			sec_cap = rte_security_capabilities_get(ctx);
+		sec_cap = rte_security_capabilities_get(ctx);
 
-			if (sec_cap == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"No capabilities registered\n");
-				return -1;
-			}
+		if (sec_cap == NULL) {
+			RTE_LOG(ERR, IPSEC,
+			"No capabilities registered\n");
+			return -1;
+		}
 
-			/* iterate until ESP tunnel*/
-			while (sec_cap->action !=
+		/* iterate until ESP tunnel*/
+		while (sec_cap->action !=
 					RTE_SECURITY_ACTION_TYPE_NONE) {
 
-				if (sec_cap->action == sa->type &&
-				    sec_cap->protocol ==
-					RTE_SECURITY_PROTOCOL_IPSEC &&
-				    sec_cap->ipsec.mode ==
-					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
-				    sec_cap->ipsec.direction == sa->direction)
-					break;
-				sec_cap++;
-			}
+			if (sec_cap->action == sa->type &&
+				sec_cap->protocol ==
+				RTE_SECURITY_PROTOCOL_IPSEC &&
+				sec_cap->ipsec.mode ==
+				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
+				sec_cap->ipsec.direction == sa->direction)
+				break;
+			sec_cap++;
+		}
 
-			if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
-				RTE_LOG(ERR, IPSEC,
-				"No suitable security capability found\n");
-				return -1;
-			}
+		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
+			RTE_LOG(ERR, IPSEC,
+			"No suitable security capability found\n");
+			return -1;
+		}
+
+		sa->ol_flags = sec_cap->ol_flags;
+		sa->security_ctx = ctx;
+	} else
+		return -EINVAL;
+	return 0;
+}
+
+#define CDEV_IV_SIZE 12
+
+static int
+check_cryptodev_aead_capablity(const struct ipsec_sa *ss, uint8_t dev_id)
+{
+	struct rte_cryptodev_sym_capability_idx cap_idx;
+	const struct rte_cryptodev_symmetric_capability *cap;
 
-			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
+	cap_idx.type = RTE_CRYPTO_SYM_XFORM_AEAD;
+	cap_idx.algo.aead = ss->aead_algo;
+
+	cap = rte_cryptodev_sym_capability_get(dev_id, &cap_idx);
+	if (cap == NULL)
+		return -ENOENT;
+
+	return rte_cryptodev_sym_capability_check_aead(cap,
+					ss->cipher_key_len,
+					ss->digest_len,
+					ss->aad_len,
+					CDEV_IV_SIZE);
+}
+
+static int
+check_cryptodev_capablity(const struct ipsec_sa *ss, uint8_t dev_id)
+{
+	struct rte_cryptodev_sym_capability_idx cap_idx;
+	const struct rte_cryptodev_symmetric_capability *cap;
+	uint16_t auth_iv_len;
+	int rc = -1;
+
+	if (ss == NULL)
+		return rc;
+
+	if (ss->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
+		return check_cryptodev_aead_capablity(ss, dev_id);
+
+	auth_iv_len = 0;
+
+	cap_idx.type = RTE_CRYPTO_SYM_XFORM_AUTH;
+	cap_idx.algo.auth = ss->auth_algo;
+	cap = rte_cryptodev_sym_capability_get(dev_id, &cap_idx);
+	if (cap != NULL) {
+		rc = rte_cryptodev_sym_capability_check_auth(
+				cap, ss->auth_key_len, ss->digest_len,
+				auth_iv_len);
+		if (rc == 0) {
+			cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
+			cap_idx.algo.cipher = ss->cipher_algo;
+			cap = rte_cryptodev_sym_capability_get(dev_id,
+					&cap_idx);
+			if (cap != NULL)
+				rc = rte_cryptodev_sym_capability_check_cipher(
+						cap, ss->cipher_key_len,
+						ss->iv_len);
 		}
-	} else {
-		sa->crypto_session = rte_cryptodev_sym_session_create(
-				ipsec_ctx->session_pool);
-		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
-				sa->crypto_session, sa->xforms,
-				ipsec_ctx->session_priv_pool);
-
-		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
-				&cdev_info);
 	}
-	sa->cdev_id_qp = cdev_id_qp;
 
-	return 0;
+	return rc;
+}
+
+static int
+create_crypto_session(struct ipsec_sa *sa, struct rte_mempool *pool)
+{
+	int32_t rc;
+	uint32_t devnum, i;
+	struct rte_cryptodev_sym_session *s;
+	uint8_t devid[RTE_CRYPTO_MAX_DEVS];
+
+	/* check which cryptodevs support SA */
+	devnum = 0;
+	for (i = 0; i < crypto_dev_num; i++) {
+		rc = check_cryptodev_capablity(sa, crypto_devid[i]);
+		if (rc == 0)
+			devid[devnum++] = crypto_devid[i];
+	}
+
+	if (devnum == 0)
+		return -ENODEV;
+
+	s = rte_cryptodev_sym_session_create(pool);
+	if (s == NULL)
+		return -ENOMEM;
+
+	/* initialize SA crypto session for all supported devices */
+	for (i = 0; i != devnum; i++) {
+		rc = rte_cryptodev_sym_session_init(devid[i], s, sa->xforms,
+			pool);
+		if (rc != 0)
+			break;
+	}
+
+	if (i == devnum) {
+		sa->crypto_session = s;
+		return 0;
+	}
+
+	/* failure, do cleanup */
+	while (i-- != 0)
+		rte_cryptodev_sym_session_clear(devid[i], s);
+
+	rte_cryptodev_sym_session_free(s);
+	return rc;
+}
+
+int
+create_session(struct ipsec_sa *sa, struct rte_mempool *pool)
+{
+	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE)
+		return create_sec_session(sa, pool);
+	else
+		return create_crypto_session(sa, pool);
 }
 
 /*
@@ -393,13 +461,6 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			sym_cop = get_sym_cop(&priv->cop);
 			sym_cop->m_src = pkts[i];
 
@@ -412,13 +473,6 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->crypto_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			rte_crypto_op_attach_sym_session(&priv->cop,
 					sa->crypto_session);
 
@@ -429,12 +483,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 			break;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
+			RTE_ASSERT(sa->sec_session != NULL);
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
 			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
@@ -442,17 +491,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 						sa->sec_session, pkts[i], NULL);
 			continue;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
+			RTE_ASSERT(sa->sec_session != NULL);
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			rte_security_attach_session(&priv->cop,
 					sa->sec_session);
 
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index 99f49d6..804330c 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -83,6 +83,14 @@ struct app_sa_prm {
 
 extern struct app_sa_prm app_sa_prm;
 
+/*
+ * Number of enabled crypto devices
+ * This number is needed when checking crypto device capabilities
+ */
+extern uint8_t crypto_dev_num;
+/* array of crypto device ID's */
+extern uint8_t crypto_devid[RTE_CRYPTO_MAX_DEVS];
+
 struct ipsec_sa {
 	struct rte_ipsec_session ips; /* one session per sa for now */
 	uint32_t spi;
@@ -306,6 +314,6 @@ void
 enqueue_cop_burst(struct cdev_qp *cqp);
 
 int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+create_session(struct ipsec_sa *sa, struct rte_mempool *pool);
 
 #endif /* __IPSEC_H__ */
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index 3f9cacb..0df6969 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -86,39 +86,6 @@ enqueue_cop_bulk(struct cdev_qp *cqp, struct rte_crypto_op *cop[], uint32_t num)
 	cqp->len = len;
 }
 
-static inline int
-fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
-	struct ipsec_sa *sa)
-{
-	int32_t rc;
-
-	/* setup crypto section */
-	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		if (sa->crypto_session == NULL) {
-			rc = create_session(ctx, sa);
-			if (rc != 0)
-				return rc;
-		}
-		ss->crypto.ses = sa->crypto_session;
-	/* setup session action type */
-	} else {
-		if (sa->sec_session == NULL) {
-			rc = create_session(ctx, sa);
-			if (rc != 0)
-				return rc;
-		}
-		ss->security.ses = sa->sec_session;
-		ss->security.ctx = sa->security_ctx;
-		ss->security.ol_flags = sa->ol_flags;
-	}
-
-	rc = rte_ipsec_session_prepare(ss);
-	if (rc != 0)
-		memset(ss, 0, sizeof(*ss));
-
-	return rc;
-}
-
 /*
  * group input packets byt the SA they belong to.
  */
@@ -219,9 +186,8 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 
 		ips = &sa->ips;
 
-		/* no valid HW session for that SA, try to create one */
-		if (sa == NULL || (ips->crypto.ses == NULL &&
-				fill_ipsec_session(ips, ctx, sa) != 0))
+		/* no valid HW session for that SA */
+		if (sa == NULL || ips->crypto.ses == NULL)
 			k = 0;
 
 		/* process packets inline */
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index a7298a3..0f36f5b 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -774,14 +774,14 @@ check_eth_dev_caps(uint16_t portid, uint32_t inbound)
 	return 0;
 }
 
-
 static int
 sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries, uint32_t inbound)
+	uint32_t nb_entries, uint32_t inbound, struct socket_ctx *skt_ctx)
 {
 	struct ipsec_sa *sa;
 	uint32_t i, idx;
 	uint16_t iv_length, aad_length;
+	int32_t rc;
 
 	/* for ESN upper 32 bits of SQN also need to be part of AAD */
 	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
@@ -902,6 +902,12 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			print_one_sa_rule(sa, inbound);
 		}
+		rc = create_session(sa, skt_ctx->session_pool);
+		if (rc != 0) {
+			RTE_LOG(ERR, IPSEC_ESP,
+					"create_session() failed\n");
+			return -EINVAL;
+		}
 	}
 
 	return 0;
@@ -909,16 +915,16 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 static inline int
 sa_out_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 0);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx);
 }
 
 static inline int
 sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 1);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx);
 }
 
 /*
@@ -1012,10 +1018,12 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm, const struct ipsec_sa *ss,
 	return 0;
 }
 
-static void
+static int
 fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 	const struct ipsec_sa *lsa)
 {
+	int32_t rc = 0;
+
 	ss->sa = sa;
 	ss->type = lsa->type;
 
@@ -1028,6 +1036,12 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 		ss->security.ctx = lsa->security_ctx;
 		ss->security.ol_flags = lsa->ol_flags;
 	}
+
+	rc = rte_ipsec_session_prepare(ss);
+	if (rc != 0)
+		memset(ss, 0, sizeof(*ss));
+
+	return rc;
 }
 
 /*
@@ -1062,8 +1076,8 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	fill_ipsec_session(&lsa->ips, sa, lsa);
-	return 0;
+	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
+	return rc;
 }
 
 /*
@@ -1141,7 +1155,10 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in);
+		rc = sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx);
+		if (rc != 0)
+			rte_exit(EXIT_FAILURE,
+				"failed to add inbound rules\n");
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_in, sa_in, nb_sa_in,
@@ -1161,7 +1178,10 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out);
+		rc = sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx);
+		if (rc != 0)
+			rte_exit(EXIT_FAILURE,
+				"failed to add outbound rules\n");
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_out, sa_out, nb_sa_out,
-- 
2.7.4

[dpdk-dev] [PATCH v3 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Bernard Iremonger]

Inline crypto installs a flow rule in the NIC. This flow
rule must be installed before the first inbound packet is
received.

The create_session() function installs the flow rule.

Refactor ipsec-secgw.c, sa.c, ipsec.h and ipsec.c to create
sessions at startup to fix the issue of the first packet
being dropped for inline crypto.

The create_session() function is now called at initialisation in
sa_add_rules() which is called from sa_init().
The return code for add_rules is checked.
Calls to create_session() in other functions are dropped.

Add crypto_devid_fill() in ipsec-secgw.c
Add max_session_size() in ipsec-secgw.c
Add check_cryptodev_capability() in ipsec.c
Add check_cryptodev_aead_capability() in ipsec.c
Add create_sec_session() and create_crypto_session() in ipsec.c

The crypto_dev_fill() function has been added to find the
enabled crypto devices.

The max_session_size() function has been added to calculate memory
requirements.

The check_cryptodev_capability() and check_cryptodev_aead_capability()
functions have been added to check that the SA is supported by the
crypto device.

The create_session() function is refactored to use the
create_sec_session() and create_crypto_session() functions.

The cryprodev_init() function has been refactored to drop calls to
rte_mempool_create() and to drop calculation of memory requirements.

The main() function has been refactored to call crypto_devid_fill()
and max_session_size() and to call session_pool_init() and
session_priv_pool_init().
The ports are started now before adding a flow rule in main().
The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
now called after the ports have been started.

Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/ipsec-secgw.c   | 271 +++++++++--------
 examples/ipsec-secgw/ipsec.c         | 569 +++++++++++++++++++----------------
 examples/ipsec-secgw/ipsec.h         |  10 +-
 examples/ipsec-secgw/ipsec_process.c |  38 +--
 examples/ipsec-secgw/sa.c            |  42 ++-
 5 files changed, 495 insertions(+), 435 deletions(-)

diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index ffbd00b..cc8bb57 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -182,6 +182,14 @@ struct lcore_params {
 	uint8_t lcore_id;
 } __rte_cache_aligned;
 
+/*
+ * Number of enabled crypto devices
+ * This number is needed when checking crypto device capabilities
+ */
+uint8_t crypto_dev_num;
+/* array of crypto device ID's */
+uint8_t crypto_devid[RTE_CRYPTO_MAX_DEVS];
+
 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS];
 
 static struct lcore_params *lcore_params;
@@ -1623,13 +1631,27 @@ check_cryptodev_mask(uint8_t cdev_id)
 	return -1;
 }
 
+static void
+crypto_devid_fill(void)
+{
+	uint32_t i, n;
+
+	n = rte_cryptodev_count();
+
+	for (i = 0; i != n; i++) {
+		if (check_cryptodev_mask(i) == 0)
+			crypto_devid[crypto_dev_num++] = i;
+	}
+}
+
 static int32_t
 cryptodevs_init(void)
 {
 	struct rte_cryptodev_config dev_conf;
 	struct rte_cryptodev_qp_conf qp_conf;
 	uint16_t idx, max_nb_qps, qp, i;
-	int16_t cdev_id, port_id;
+	int16_t cdev_id;
+	uint32_t dev_max_sess;
 	struct rte_hash_parameters params = { 0 };
 
 	params.entries = CDEV_MAP_ENTRIES;
@@ -1652,45 +1674,6 @@ cryptodevs_init(void)
 
 	printf("lcore/cryptodev/qp mappings:\n");
 
-	uint32_t max_sess_sz = 0, sess_sz;
-	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
-		void *sec_ctx;
-
-		/* Get crypto priv session size */
-		sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-
-		/*
-		 * If crypto device is security capable, need to check the
-		 * size of security session as well.
-		 */
-
-		/* Get security context of the crypto device */
-		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		/* Get size of security session */
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-	RTE_ETH_FOREACH_DEV(port_id) {
-		void *sec_ctx;
-
-		if ((enabled_port_mask & (1 << port_id)) == 0)
-			continue;
-
-		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-
 	idx = 0;
 	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
 		struct rte_cryptodev_info cdev_info;
@@ -1722,51 +1705,12 @@ cryptodevs_init(void)
 		dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id);
 		dev_conf.nb_queue_pairs = qp;
 
-		uint32_t dev_max_sess = cdev_info.sym.max_nb_sessions;
+		dev_max_sess = cdev_info.sym.max_nb_sessions;
 		if (dev_max_sess != 0 && dev_max_sess < CDEV_MP_NB_OBJS)
 			rte_exit(EXIT_FAILURE,
 				"Device does not support at least %u "
 				"sessions", CDEV_MP_NB_OBJS);
 
-		if (!socket_ctx[dev_conf.socket_id].session_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_%u", dev_conf.socket_id);
-			sess_mp = rte_cryptodev_sym_session_pool_create(
-					mp_name, CDEV_MP_NB_OBJS,
-					0, CDEV_MP_CACHE_SZ, 0,
-					dev_conf.socket_id);
-			socket_ctx[dev_conf.socket_id].session_pool = sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_priv_%u", dev_conf.socket_id);
-			sess_mp = rte_mempool_create(mp_name,
-					CDEV_MP_NB_OBJS,
-					max_sess_sz,
-					CDEV_MP_CACHE_SZ,
-					0, NULL, NULL, NULL,
-					NULL, dev_conf.socket_id,
-					0);
-			socket_ctx[dev_conf.socket_id].session_priv_pool =
-					sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool ||
-				!socket_ctx[dev_conf.socket_id].session_pool)
-			rte_exit(EXIT_FAILURE,
-				"Cannot create session pool on socket %d\n",
-				dev_conf.socket_id);
-		else
-			printf("Allocated session pool on socket %d\n",
-					dev_conf.socket_id);
-
 		if (rte_cryptodev_configure(cdev_id, &dev_conf))
 			rte_panic("Failed to initialize cryptodev %u\n",
 					cdev_id);
@@ -1787,38 +1731,6 @@ cryptodevs_init(void)
 					cdev_id);
 	}
 
-	/* create session pools for eth devices that implement security */
-	RTE_ETH_FOREACH_DEV(port_id) {
-		if ((enabled_port_mask & (1 << port_id)) &&
-				rte_eth_dev_get_sec_ctx(port_id)) {
-			int socket_id = rte_eth_dev_socket_id(port_id);
-
-			if (!socket_ctx[socket_id].session_pool) {
-				char mp_name[RTE_MEMPOOL_NAMESIZE];
-				struct rte_mempool *sess_mp;
-
-				snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-						"sess_mp_%u", socket_id);
-				sess_mp = rte_mempool_create(mp_name,
-						(CDEV_MP_NB_OBJS * 2),
-						max_sess_sz,
-						CDEV_MP_CACHE_SZ,
-						0, NULL, NULL, NULL,
-						NULL, socket_id,
-						0);
-				if (sess_mp == NULL)
-					rte_exit(EXIT_FAILURE,
-						"Cannot create session pool "
-						"on socket %d\n", socket_id);
-				else
-					printf("Allocated session pool "
-						"on socket %d\n", socket_id);
-				socket_ctx[socket_id].session_pool = sess_mp;
-			}
-		}
-	}
-
-
 	printf("\n");
 
 	return 0;
@@ -1984,6 +1896,98 @@ port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads)
 	printf("\n");
 }
 
+static size_t
+max_session_size(void)
+{
+	size_t max_sz, sz;
+	void *sec_ctx;
+	int16_t cdev_id, port_id, n;
+
+	max_sz = 0;
+	n =  rte_cryptodev_count();
+	for (cdev_id = 0; cdev_id != n; cdev_id++) {
+		sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
+		if (sz > max_sz)
+			max_sz = sz;
+		/*
+		 * If crypto device is security capable, need to check the
+		 * size of security session as well.
+		 */
+
+		/* Get security context of the crypto device */
+		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		/* Get size of security session */
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	RTE_ETH_FOREACH_DEV(port_id) {
+		if ((enabled_port_mask & (1 << port_id)) == 0)
+			continue;
+
+		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	return max_sz;
+}
+
+static void
+session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_%u", socket_id);
+	sess_mp = rte_cryptodev_sym_session_pool_create(
+			mp_name, CDEV_MP_NB_OBJS,
+			sess_sz, CDEV_MP_CACHE_SZ, 0,
+			socket_id);
+	ctx->session_pool = sess_mp;
+
+	if (ctx->session_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session pool on socket %d\n", socket_id);
+	else
+		printf("Allocated session pool on socket %d\n",	socket_id);
+}
+
+static void
+session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
+	size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_priv_%u", socket_id);
+	sess_mp = rte_mempool_create(mp_name,
+			CDEV_MP_NB_OBJS,
+			sess_sz,
+			CDEV_MP_CACHE_SZ,
+			0, NULL, NULL, NULL,
+			NULL, socket_id,
+			0);
+	ctx->session_priv_pool = sess_mp;
+	if (ctx->session_priv_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session priv pool on socket %d\n",
+			socket_id);
+	else
+		printf("Allocated session priv pool on socket %d\n",
+			socket_id);
+}
+
 static void
 pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf)
 {
@@ -2064,9 +2068,11 @@ main(int32_t argc, char **argv)
 {
 	int32_t ret;
 	uint32_t lcore_id;
+	uint32_t i;
 	uint8_t socket_id;
 	uint16_t portid;
 	uint64_t req_rx_offloads, req_tx_offloads;
+	size_t sess_sz;
 
 	/* init EAL */
 	ret = rte_eal_init(argc, argv);
@@ -2094,7 +2100,10 @@ main(int32_t argc, char **argv)
 
 	nb_lcores = rte_lcore_count();
 
-	/* Replicate each context per socket */
+	crypto_devid_fill();
+
+	sess_sz = max_session_size();
+
 	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
 		if (rte_lcore_is_enabled(lcore_id) == 0)
 			continue;
@@ -2104,20 +2113,17 @@ main(int32_t argc, char **argv)
 		else
 			socket_id = 0;
 
+		/* mbuf_pool is initialised by the pool_init() function*/
 		if (socket_ctx[socket_id].mbuf_pool)
 			continue;
 
-		/* initilaze SPD */
-		sp4_init(&socket_ctx[socket_id], socket_id);
-
-		sp6_init(&socket_ctx[socket_id], socket_id);
-
-		/* initilaze SAD */
-		sa_init(&socket_ctx[socket_id], socket_id);
-
-		rt_init(&socket_ctx[socket_id], socket_id);
-
 		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
+		session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
+		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
+			sess_sz);
+
+		if (!numa_on)
+			break;
 	}
 
 	RTE_ETH_FOREACH_DEV(portid) {
@@ -2135,7 +2141,11 @@ main(int32_t argc, char **argv)
 		if ((enabled_port_mask & (1 << portid)) == 0)
 			continue;
 
-		/* Start device */
+		/*
+		 * Start device
+		 * note: device must be started before a flow rule
+		 * can be installed.
+		 */
 		ret = rte_eth_dev_start(portid);
 		if (ret < 0)
 			rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
@@ -2153,6 +2163,19 @@ main(int32_t argc, char **argv)
 			RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback, NULL);
 	}
 
+	/* Replicate each context per socket */
+	for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
+		socket_id = rte_socket_id_by_idx(i);
+		if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
+			(socket_ctx[socket_id].sa_in == NULL) &&
+			(socket_ctx[socket_id].sa_out == NULL)) {
+			sa_init(&socket_ctx[socket_id], socket_id);
+			sp4_init(&socket_ctx[socket_id], socket_id);
+			sp6_init(&socket_ctx[socket_id], socket_id);
+			rt_init(&socket_ctx[socket_id], socket_id);
+		}
+	}
+
 	check_all_ports_link_status(enabled_port_mask);
 
 	/* launch per-lcore init on every lcore */
diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index 4352cb8..e31d472 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -39,42 +39,17 @@ set_ipsec_conf(struct ipsec_sa *sa, struct rte_security_ipsec_xform *ipsec)
 	ipsec->esn_soft_limit = IPSEC_OFFLOAD_ESN_SOFTLIMIT;
 }
 
-int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
+static int
+create_sec_session(struct ipsec_sa *sa, struct rte_mempool *pool)
 {
-	struct rte_cryptodev_info cdev_info;
-	unsigned long cdev_id_qp = 0;
+	const struct rte_security_capability *sec_cap;
+	struct rte_security_ctx *ctx;
 	int32_t ret = 0;
-	struct cdev_key key = { 0 };
-
-	key.lcore_id = (uint8_t)rte_lcore_id();
-
-	key.cipher_algo = (uint8_t)sa->cipher_algo;
-	key.auth_algo = (uint8_t)sa->auth_algo;
-	key.aead_algo = (uint8_t)sa->aead_algo;
-
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
-				(void **)&cdev_id_qp);
-		if (ret < 0) {
-			RTE_LOG(ERR, IPSEC,
-				"No cryptodev: core %u, cipher_algo %u, "
-				"auth_algo %u, aead_algo %u\n",
-				key.lcore_id,
-				key.cipher_algo,
-				key.auth_algo,
-				key.aead_algo);
-			return -1;
-		}
-	}
 
-	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on cryptodev "
-			"%u qp %u\n", sa->spi,
-			ipsec_ctx->tbl[cdev_id_qp].id,
-			ipsec_ctx->tbl[cdev_id_qp].qp);
+	if ((sa == NULL) || (pool == NULL))
+		return -EINVAL;
 
-	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
-		struct rte_security_session_conf sess_conf = {
+	struct rte_security_session_conf sess_conf = {
 			.action_type = sa->type,
 			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 			{.ipsec = {
@@ -90,247 +65,340 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			} },
 			.crypto_xform = sa->xforms,
 			.userdata = NULL,
-
 		};
 
-		if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
-			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
-							rte_cryptodev_get_sec_ctx(
-							ipsec_ctx->tbl[cdev_id_qp].id);
-
-			/* Set IPsec parameters in conf */
-			set_ipsec_conf(sa, &(sess_conf.ipsec));
-
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"SEC Session init failed: err: %d\n", ret);
-				return -1;
-			}
-		} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
-			struct rte_flow_error err;
-			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
-							rte_eth_dev_get_sec_ctx(
-							sa->portid);
-			const struct rte_security_capability *sec_cap;
-			int ret = 0;
-
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"SEC Session init failed: err: %d\n", ret);
-				return -1;
-			}
+	if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
+		ctx = (struct rte_security_ctx *)
+				rte_eth_dev_get_sec_ctx(sa->portid);
 
-			sec_cap = rte_security_capabilities_get(ctx);
+		/* Set IPsec parameters in conf */
+		set_ipsec_conf(sa, &(sess_conf.ipsec));
 
-			/* iterate until ESP tunnel*/
-			while (sec_cap->action !=
-					RTE_SECURITY_ACTION_TYPE_NONE) {
+		sa->sec_session = rte_security_session_create(ctx,
+				&sess_conf, pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC,
+				"SEC Session init failed: err: %d\n",
+				ret);
+			return -1;
+		}
+	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		struct rte_flow_error err;
+		ctx = (struct rte_security_ctx *)
+				rte_eth_dev_get_sec_ctx(sa->portid);
+		sa->sec_session = rte_security_session_create(ctx,
+				&sess_conf, pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC, "SEC Session init failed\n");
+			return -1;
+		}
 
-				if (sec_cap->action == sa->type &&
-				    sec_cap->protocol ==
-					RTE_SECURITY_PROTOCOL_IPSEC &&
-				    sec_cap->ipsec.mode ==
-					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
-				    sec_cap->ipsec.direction == sa->direction)
-					break;
-				sec_cap++;
-			}
+		sec_cap = rte_security_capabilities_get(ctx);
+
+		/* iterate until ESP tunnel*/
+		while (sec_cap->action != RTE_SECURITY_ACTION_TYPE_NONE) {
+			if (sec_cap->action == sa->type &&
+				sec_cap->protocol ==
+				RTE_SECURITY_PROTOCOL_IPSEC &&
+				sec_cap->ipsec.mode ==
+				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
+				sec_cap->ipsec.direction == sa->direction)
+				break;
+			sec_cap++;
+		}
 
-			if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
-				RTE_LOG(ERR, IPSEC,
+		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
+			RTE_LOG(ERR, IPSEC,
 				"No suitable security capability found\n");
 				return -1;
-			}
+		}
 
-			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
-			sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
-
-			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
-			sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
-			if (sa->flags & IP6_TUNNEL) {
-				sa->pattern[1].spec = &sa->ipv6_spec;
-				memcpy(sa->ipv6_spec.hdr.dst_addr,
-					sa->dst.ip.ip6.ip6_b, 16);
-				memcpy(sa->ipv6_spec.hdr.src_addr,
-				       sa->src.ip.ip6.ip6_b, 16);
-			} else {
-				sa->pattern[1].spec = &sa->ipv4_spec;
-				sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
-				sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
-			}
+		sa->ol_flags = sec_cap->ol_flags;
+		sa->security_ctx = ctx;
+		sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
+
+		sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
+		sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
+		if (sa->flags & IP6_TUNNEL) {
+			sa->pattern[1].spec = &sa->ipv6_spec;
+			memcpy(sa->ipv6_spec.hdr.dst_addr,
+				sa->dst.ip.ip6.ip6_b, 16);
+			memcpy(sa->ipv6_spec.hdr.src_addr,
+			       sa->src.ip.ip6.ip6_b, 16);
+		} else {
+			sa->pattern[1].spec = &sa->ipv4_spec;
+			sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
+			sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
+		}
 
-			sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
-			sa->pattern[2].spec = &sa->esp_spec;
-			sa->pattern[2].mask = &rte_flow_item_esp_mask;
-			sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
+		sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
+		sa->pattern[2].spec = &sa->esp_spec;
+		sa->pattern[2].mask = &rte_flow_item_esp_mask;
+		sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
 
-			sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
+		sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
 
-			sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
-			sa->action[0].conf = sa->sec_session;
+		sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
+		sa->action[0].conf = sa->sec_session;
 
-			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
+		sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
 
-			sa->attr.egress = (sa->direction ==
+		sa->attr.egress = (sa->direction ==
 					RTE_SECURITY_IPSEC_SA_DIR_EGRESS);
-			sa->attr.ingress = (sa->direction ==
+		sa->attr.ingress = (sa->direction ==
 					RTE_SECURITY_IPSEC_SA_DIR_INGRESS);
-			if (sa->attr.ingress) {
-				uint8_t rss_key[40];
-				struct rte_eth_rss_conf rss_conf = {
-					.rss_key = rss_key,
-					.rss_key_len = 40,
-				};
-				struct rte_eth_dev *eth_dev;
-				uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
-				struct rte_flow_action_rss action_rss;
-				unsigned int i;
-				unsigned int j;
-
-				sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
-				/* Try RSS. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
-				sa->action[1].conf = &action_rss;
-				eth_dev = ctx->device;
-				rte_eth_dev_rss_hash_conf_get(sa->portid,
-							      &rss_conf);
-				for (i = 0, j = 0;
-				     i < eth_dev->data->nb_rx_queues; ++i)
-					if (eth_dev->data->rx_queues[i])
-						queue[j++] = i;
+		if (sa->attr.ingress) {
+			uint8_t rss_key[40];
+			struct rte_eth_rss_conf rss_conf = {
+				.rss_key = rss_key,
+				.rss_key_len = 40,
+			};
+			struct rte_eth_dev *eth_dev;
+			uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
+			struct rte_flow_action_rss action_rss;
+			unsigned int i;
+			unsigned int j;
+
+			sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
+			/* Try RSS. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
+			sa->action[1].conf = &action_rss;
+			eth_dev = ctx->device;
+			rte_eth_dev_rss_hash_conf_get(sa->portid,
+						&rss_conf);
+			for (i = 0, j = 0; i < eth_dev->data->nb_rx_queues;
+				++i)
+				if (eth_dev->data->rx_queues[i])
+					queue[j++] = i;
 				action_rss = (struct rte_flow_action_rss){
 					.types = rss_conf.rss_hf,
 					.key_len = rss_conf.rss_key_len,
 					.queue_num = j,
 					.key = rss_key,
 					.queue = queue,
-				};
-				ret = rte_flow_validate(sa->portid, &sa->attr,
-							sa->pattern, sa->action,
-							&err);
-				if (!ret)
-					goto flow_create;
-				/* Try Queue. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_QUEUE;
-				sa->action[1].conf =
-					&(struct rte_flow_action_queue){
-					.index = 0,
-				};
-				ret = rte_flow_validate(sa->portid, &sa->attr,
-							sa->pattern, sa->action,
-							&err);
-				/* Try End. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
-				sa->action[1].conf = NULL;
-				ret = rte_flow_validate(sa->portid, &sa->attr,
+			};
+			ret = rte_flow_validate(sa->portid, &sa->attr,
+						sa->pattern, sa->action,
+						&err);
+			if (!ret)
+				goto flow_create;
+			/* Try Queue. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_QUEUE;
+			sa->action[1].conf =
+				&(struct rte_flow_action_queue){
+				.index = 0,
+			};
+			ret = rte_flow_validate(sa->portid, &sa->attr,
+						sa->pattern, sa->action,
+						&err);
+			/* Try End. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
+			sa->action[1].conf = NULL;
+			ret = rte_flow_validate(sa->portid, &sa->attr,
 							sa->pattern, sa->action,
 							&err);
-				if (ret)
-					goto flow_create_failure;
-			} else if (sa->attr.egress &&
-				   (sa->ol_flags &
-				    RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
-				sa->action[1].type =
-					RTE_FLOW_ACTION_TYPE_PASSTHRU;
-				sa->action[2].type =
-					RTE_FLOW_ACTION_TYPE_END;
-			}
+			if (ret)
+				goto flow_create_failure;
+		} else if (sa->attr.egress &&
+				(sa->ol_flags &
+				RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
+			sa->action[1].type =
+				RTE_FLOW_ACTION_TYPE_PASSTHRU;
+			sa->action[2].type =
+				RTE_FLOW_ACTION_TYPE_END;
+		}
 flow_create:
-			sa->flow = rte_flow_create(sa->portid,
-				&sa->attr, sa->pattern, sa->action, &err);
-			if (sa->flow == NULL) {
+		sa->flow = rte_flow_create(sa->portid,
+			&sa->attr, sa->pattern, sa->action, &err);
+		if (sa->flow == NULL) {
 flow_create_failure:
-				RTE_LOG(ERR, IPSEC,
-					"Failed to create ipsec flow msg: %s\n",
-					err.message);
-				return -1;
-			}
-		} else if (sa->type ==
-				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
-			struct rte_security_ctx *ctx =
-					(struct rte_security_ctx *)
-					rte_eth_dev_get_sec_ctx(sa->portid);
-			const struct rte_security_capability *sec_cap;
-
-			if (ctx == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"Ethernet device doesn't have security features registered\n");
-				return -1;
-			}
+			RTE_LOG(ERR, IPSEC,
+				"Failed to create ipsec flow msg: %s\n",
+				err.message);
+			return -1;
+		}
+	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
+		struct rte_security_ctx *ctx =
+				(struct rte_security_ctx *)
+				rte_eth_dev_get_sec_ctx(sa->portid);
+		const struct rte_security_capability *sec_cap;
 
-			/* Set IPsec parameters in conf */
-			set_ipsec_conf(sa, &(sess_conf.ipsec));
-
-			/* Save SA as userdata for the security session. When
-			 * the packet is received, this userdata will be
-			 * retrieved using the metadata from the packet.
-			 *
-			 * The PMD is expected to set similar metadata for other
-			 * operations, like rte_eth_event, which are tied to
-			 * security session. In such cases, the userdata could
-			 * be obtained to uniquely identify the security
-			 * parameters denoted.
-			 */
-
-			sess_conf.userdata = (void *) sa;
-
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"SEC Session init failed: err: %d\n", ret);
-				return -1;
-			}
+		if (ctx == NULL) {
+			RTE_LOG(ERR, IPSEC,
+			"Ethernet device doesn't have security features registered\n");
+			return -1;
+		}
+
+		/* Set IPsec parameters in conf */
+		set_ipsec_conf(sa, &(sess_conf.ipsec));
+
+		/* Save SA as userdata for the security session. When
+		 * the packet is received, this userdata will be
+		 * retrieved using the metadata from the packet.
+		 *
+		 * The PMD is expected to set similar metadata for other
+		 * operations, like rte_eth_event, which are tied to
+		 * security session. In such cases, the userdata could
+		 * be obtained to uniquely identify the security
+		 * parameters denoted.
+		 */
+
+		sess_conf.userdata = (void *) sa;
+
+		sa->sec_session = rte_security_session_create(ctx,
+			&sess_conf, pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC,
+			"SEC Session init failed: err: %d\n", ret);
+			return -1;
+		}
 
-			sec_cap = rte_security_capabilities_get(ctx);
+		sec_cap = rte_security_capabilities_get(ctx);
 
-			if (sec_cap == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"No capabilities registered\n");
-				return -1;
-			}
+		if (sec_cap == NULL) {
+			RTE_LOG(ERR, IPSEC,
+			"No capabilities registered\n");
+			return -1;
+		}
 
-			/* iterate until ESP tunnel*/
-			while (sec_cap->action !=
+		/* iterate until ESP tunnel*/
+		while (sec_cap->action !=
 					RTE_SECURITY_ACTION_TYPE_NONE) {
 
-				if (sec_cap->action == sa->type &&
-				    sec_cap->protocol ==
-					RTE_SECURITY_PROTOCOL_IPSEC &&
-				    sec_cap->ipsec.mode ==
-					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
-				    sec_cap->ipsec.direction == sa->direction)
-					break;
-				sec_cap++;
-			}
+			if (sec_cap->action == sa->type &&
+				sec_cap->protocol ==
+				RTE_SECURITY_PROTOCOL_IPSEC &&
+				sec_cap->ipsec.mode ==
+				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
+				sec_cap->ipsec.direction == sa->direction)
+				break;
+			sec_cap++;
+		}
 
-			if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
-				RTE_LOG(ERR, IPSEC,
-				"No suitable security capability found\n");
-				return -1;
-			}
+		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
+			RTE_LOG(ERR, IPSEC,
+			"No suitable security capability found\n");
+			return -1;
+		}
+
+		sa->ol_flags = sec_cap->ol_flags;
+		sa->security_ctx = ctx;
+	} else
+		return -EINVAL;
+	return 0;
+}
+
+#define CDEV_IV_SIZE 12
+
+static int
+check_cryptodev_aead_capablity(const struct ipsec_sa *ss, uint8_t dev_id)
+{
+	struct rte_cryptodev_sym_capability_idx cap_idx;
+	const struct rte_cryptodev_symmetric_capability *cap;
 
-			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
+	cap_idx.type = RTE_CRYPTO_SYM_XFORM_AEAD;
+	cap_idx.algo.aead = ss->aead_algo;
+
+	cap = rte_cryptodev_sym_capability_get(dev_id, &cap_idx);
+	if (cap == NULL)
+		return -ENOENT;
+
+	return rte_cryptodev_sym_capability_check_aead(cap,
+					ss->cipher_key_len,
+					ss->digest_len,
+					ss->aad_len,
+					CDEV_IV_SIZE);
+}
+
+static int
+check_cryptodev_capablity(const struct ipsec_sa *ss, uint8_t dev_id)
+{
+	struct rte_cryptodev_sym_capability_idx cap_idx;
+	const struct rte_cryptodev_symmetric_capability *cap;
+	uint16_t auth_iv_len;
+	int rc = -1;
+
+	if (ss == NULL)
+		return rc;
+
+	if (ss->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
+		return check_cryptodev_aead_capablity(ss, dev_id);
+
+	auth_iv_len = 0;
+
+	cap_idx.type = RTE_CRYPTO_SYM_XFORM_AUTH;
+	cap_idx.algo.auth = ss->auth_algo;
+	cap = rte_cryptodev_sym_capability_get(dev_id, &cap_idx);
+	if (cap != NULL) {
+		rc = rte_cryptodev_sym_capability_check_auth(
+				cap, ss->auth_key_len, ss->digest_len,
+				auth_iv_len);
+		if (rc == 0) {
+			cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
+			cap_idx.algo.cipher = ss->cipher_algo;
+			cap = rte_cryptodev_sym_capability_get(dev_id,
+					&cap_idx);
+			if (cap != NULL)
+				rc = rte_cryptodev_sym_capability_check_cipher(
+						cap, ss->cipher_key_len,
+						ss->iv_len);
 		}
-	} else {
-		sa->crypto_session = rte_cryptodev_sym_session_create(
-				ipsec_ctx->session_pool);
-		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
-				sa->crypto_session, sa->xforms,
-				ipsec_ctx->session_priv_pool);
-
-		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
-				&cdev_info);
 	}
-	sa->cdev_id_qp = cdev_id_qp;
 
-	return 0;
+	return rc;
+}
+
+static int
+create_crypto_session(struct ipsec_sa *sa, struct rte_mempool *pool)
+{
+	int32_t rc;
+	uint32_t devnum, i;
+	struct rte_cryptodev_sym_session *s;
+	uint8_t devid[RTE_CRYPTO_MAX_DEVS];
+
+	/* check which cryptodevs support SA */
+	devnum = 0;
+	for (i = 0; i < crypto_dev_num; i++) {
+		rc = check_cryptodev_capablity(sa, crypto_devid[i]);
+		if (rc == 0)
+			devid[devnum++] = crypto_devid[i];
+	}
+
+	if (devnum == 0)
+		return -ENODEV;
+
+	s = rte_cryptodev_sym_session_create(pool);
+	if (s == NULL)
+		return -ENOMEM;
+
+	/* initialize SA crypto session for all supported devices */
+	for (i = 0; i != devnum; i++) {
+		rc = rte_cryptodev_sym_session_init(devid[i], s, sa->xforms,
+			pool);
+		if (rc != 0)
+			break;
+	}
+
+	if (i == devnum) {
+		sa->crypto_session = s;
+		return 0;
+	}
+
+	/* failure, do cleanup */
+	while (i-- != 0)
+		rte_cryptodev_sym_session_clear(devid[i], s);
+
+	rte_cryptodev_sym_session_free(s);
+	return rc;
+}
+
+int
+create_session(struct ipsec_sa *sa, struct rte_mempool *pool)
+{
+	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE)
+		return create_sec_session(sa, pool);
+	else
+		return create_crypto_session(sa, pool);
 }
 
 /*
@@ -393,13 +461,6 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			sym_cop = get_sym_cop(&priv->cop);
 			sym_cop->m_src = pkts[i];
 
@@ -412,13 +473,6 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->crypto_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			rte_crypto_op_attach_sym_session(&priv->cop,
 					sa->crypto_session);
 
@@ -429,12 +483,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 			break;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
+			RTE_ASSERT(sa->sec_session != NULL);
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
 			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
@@ -442,17 +491,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 						sa->sec_session, pkts[i], NULL);
 			continue;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
+			RTE_ASSERT(sa->sec_session != NULL);
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			rte_security_attach_session(&priv->cop,
 					sa->sec_session);
 
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index 99f49d6..804330c 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -83,6 +83,14 @@ struct app_sa_prm {
 
 extern struct app_sa_prm app_sa_prm;
 
+/*
+ * Number of enabled crypto devices
+ * This number is needed when checking crypto device capabilities
+ */
+extern uint8_t crypto_dev_num;
+/* array of crypto device ID's */
+extern uint8_t crypto_devid[RTE_CRYPTO_MAX_DEVS];
+
 struct ipsec_sa {
 	struct rte_ipsec_session ips; /* one session per sa for now */
 	uint32_t spi;
@@ -306,6 +314,6 @@ void
 enqueue_cop_burst(struct cdev_qp *cqp);
 
 int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+create_session(struct ipsec_sa *sa, struct rte_mempool *pool);
 
 #endif /* __IPSEC_H__ */
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index 3f9cacb..0df6969 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -86,39 +86,6 @@ enqueue_cop_bulk(struct cdev_qp *cqp, struct rte_crypto_op *cop[], uint32_t num)
 	cqp->len = len;
 }
 
-static inline int
-fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
-	struct ipsec_sa *sa)
-{
-	int32_t rc;
-
-	/* setup crypto section */
-	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		if (sa->crypto_session == NULL) {
-			rc = create_session(ctx, sa);
-			if (rc != 0)
-				return rc;
-		}
-		ss->crypto.ses = sa->crypto_session;
-	/* setup session action type */
-	} else {
-		if (sa->sec_session == NULL) {
-			rc = create_session(ctx, sa);
-			if (rc != 0)
-				return rc;
-		}
-		ss->security.ses = sa->sec_session;
-		ss->security.ctx = sa->security_ctx;
-		ss->security.ol_flags = sa->ol_flags;
-	}
-
-	rc = rte_ipsec_session_prepare(ss);
-	if (rc != 0)
-		memset(ss, 0, sizeof(*ss));
-
-	return rc;
-}
-
 /*
  * group input packets byt the SA they belong to.
  */
@@ -219,9 +186,8 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 
 		ips = &sa->ips;
 
-		/* no valid HW session for that SA, try to create one */
-		if (sa == NULL || (ips->crypto.ses == NULL &&
-				fill_ipsec_session(ips, ctx, sa) != 0))
+		/* no valid HW session for that SA */
+		if (sa == NULL || ips->crypto.ses == NULL)
 			k = 0;
 
 		/* process packets inline */
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index a7298a3..0f36f5b 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -774,14 +774,14 @@ check_eth_dev_caps(uint16_t portid, uint32_t inbound)
 	return 0;
 }
 
-
 static int
 sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries, uint32_t inbound)
+	uint32_t nb_entries, uint32_t inbound, struct socket_ctx *skt_ctx)
 {
 	struct ipsec_sa *sa;
 	uint32_t i, idx;
 	uint16_t iv_length, aad_length;
+	int32_t rc;
 
 	/* for ESN upper 32 bits of SQN also need to be part of AAD */
 	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
@@ -902,6 +902,12 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			print_one_sa_rule(sa, inbound);
 		}
+		rc = create_session(sa, skt_ctx->session_pool);
+		if (rc != 0) {
+			RTE_LOG(ERR, IPSEC_ESP,
+					"create_session() failed\n");
+			return -EINVAL;
+		}
 	}
 
 	return 0;
@@ -909,16 +915,16 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 static inline int
 sa_out_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 0);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx);
 }
 
 static inline int
 sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 1);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx);
 }
 
 /*
@@ -1012,10 +1018,12 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm, const struct ipsec_sa *ss,
 	return 0;
 }
 
-static void
+static int
 fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 	const struct ipsec_sa *lsa)
 {
+	int32_t rc = 0;
+
 	ss->sa = sa;
 	ss->type = lsa->type;
 
@@ -1028,6 +1036,12 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 		ss->security.ctx = lsa->security_ctx;
 		ss->security.ol_flags = lsa->ol_flags;
 	}
+
+	rc = rte_ipsec_session_prepare(ss);
+	if (rc != 0)
+		memset(ss, 0, sizeof(*ss));
+
+	return rc;
 }
 
 /*
@@ -1062,8 +1076,8 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	fill_ipsec_session(&lsa->ips, sa, lsa);
-	return 0;
+	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
+	return rc;
 }
 
 /*
@@ -1141,7 +1155,10 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in);
+		rc = sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx);
+		if (rc != 0)
+			rte_exit(EXIT_FAILURE,
+				"failed to add inbound rules\n");
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_in, sa_in, nb_sa_in,
@@ -1161,7 +1178,10 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out);
+		rc = sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx);
+		if (rc != 0)
+			rte_exit(EXIT_FAILURE,
+				"failed to add outbound rules\n");
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_out, sa_out, nb_sa_out,
-- 
2.7.4

[dpdk-dev] [PATCH v3 2/2] examples/ipsec-secgw/test: fix inline test scripts

[Bernard Iremonger]

Remove workaround in tun_aesgcm_defs.sh and trs_aesgcm_defs.sh
to get around the bug where the first inbound packet is dropped
for inline crypto.

Fixes: 929784452094 ("examples/ipsec-secgw: add scripts for functional test")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh | 10 ----------
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh | 10 ----------
 2 files changed, 20 deletions(-)

diff --git a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
index a4d902b..8382d3d 100644
--- a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
@@ -33,11 +33,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -68,9 +63,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV6}
 }
diff --git a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
index 1764ef6..8ae6532 100644
--- a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
@@ -35,11 +35,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -72,9 +67,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping6 -c 1 ${LOCAL_IPV6}
 }
-- 
2.7.4

[dpdk-dev] [PATCH v3 2/2] examples/ipsec-secgw/test: fix inline test scripts

[Bernard Iremonger]

Remove workaround in tun_aesgcm_defs.sh and trs_aesgcm_defs.sh
to get around the bug where the first inbound packet is dropped
for inline crypto.

Fixes: 929784452094 ("examples/ipsec-secgw: add scripts for functional test")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh | 10 ----------
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh | 10 ----------
 2 files changed, 20 deletions(-)

diff --git a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
index a4d902b..8382d3d 100644
--- a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
@@ -33,11 +33,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -68,9 +63,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV6}
 }
diff --git a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
index 1764ef6..8ae6532 100644
--- a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
@@ -35,11 +35,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -72,9 +67,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping6 -c 1 ${LOCAL_IPV6}
 }
-- 
2.7.4

Re: [dpdk-dev] [PATCH v3 0/2] examples/ipsec-secgw: fix 1st pkt dropped

[Ananyev, Konstantin]

> 
> This patchset fixes the issue of the first inbound packet
> being dropped for inline crypto.
> 
> Changes in v3:
> --------------
> The previous refactoring of the create_session() function has been dropped.
> The create_session() function is now called from sa_init() at startup.
> 
> The following functions have been added:
> crypto_devid_fill() in ipsec-secgw.c
> check_cryptodev_capability() in ipsec.c
> check_cryptodev_aead_capability() in ipsec.c
> create_sec_session() and create_crypto_session() in ipsec.c
> 
> The create_session() function has been refactored to call
> the create_sec_session() and create_crypto_session() functions.
> 
> 
> Changes in v2:
> --------------
> The first three patches of the v1 have been squashed.
> The commit message for the squashed patch has been updated.
> Patches 4,5 and 6 of the v1 have been dropped from this patchset.
> A patch to fix the test scripts has been added.
> 
> Bernard Iremonger (2):
>   examples/ipsec-secgw: fix 1st packet dropped for inline crypto
>   examples/ipsec-secgw/test: fix inline test scripts
> 
>  examples/ipsec-secgw/ipsec-secgw.c           | 271 +++++++------
>  examples/ipsec-secgw/ipsec.c                 | 569 ++++++++++++++-------------
>  examples/ipsec-secgw/ipsec.h                 |  10 +-
>  examples/ipsec-secgw/ipsec_process.c         |  38 +-
>  examples/ipsec-secgw/sa.c                    |  42 +-
>  examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 -
>  examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 -
>  7 files changed, 495 insertions(+), 455 deletions(-)
> 
> --

As ipsec lib and these tests were introduced in 19.02,
I don't think you need to cc stable.
Apart from that:
Series Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>

> 2.7.4

Re: [dpdk-dev] [PATCH v3 0/2] examples/ipsec-secgw: fix 1st pkt dropped

[Ananyev, Konstantin]

> 
> This patchset fixes the issue of the first inbound packet
> being dropped for inline crypto.
> 
> Changes in v3:
> --------------
> The previous refactoring of the create_session() function has been dropped.
> The create_session() function is now called from sa_init() at startup.
> 
> The following functions have been added:
> crypto_devid_fill() in ipsec-secgw.c
> check_cryptodev_capability() in ipsec.c
> check_cryptodev_aead_capability() in ipsec.c
> create_sec_session() and create_crypto_session() in ipsec.c
> 
> The create_session() function has been refactored to call
> the create_sec_session() and create_crypto_session() functions.
> 
> 
> Changes in v2:
> --------------
> The first three patches of the v1 have been squashed.
> The commit message for the squashed patch has been updated.
> Patches 4,5 and 6 of the v1 have been dropped from this patchset.
> A patch to fix the test scripts has been added.
> 
> Bernard Iremonger (2):
>   examples/ipsec-secgw: fix 1st packet dropped for inline crypto
>   examples/ipsec-secgw/test: fix inline test scripts
> 
>  examples/ipsec-secgw/ipsec-secgw.c           | 271 +++++++------
>  examples/ipsec-secgw/ipsec.c                 | 569 ++++++++++++++-------------
>  examples/ipsec-secgw/ipsec.h                 |  10 +-
>  examples/ipsec-secgw/ipsec_process.c         |  38 +-
>  examples/ipsec-secgw/sa.c                    |  42 +-
>  examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 -
>  examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 -
>  7 files changed, 495 insertions(+), 455 deletions(-)
> 
> --

As ipsec lib and these tests were introduced in 19.02,
I don't think you need to cc stable.
Apart from that:
Series Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>

> 2.7.4

Re: [dpdk-dev] [PATCH v3 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Akhil Goyal]

> 
> Inline crypto installs a flow rule in the NIC. This flow
> rule must be installed before the first inbound packet is
> received.
> 
> The create_session() function installs the flow rule.
> 
> Refactor ipsec-secgw.c, sa.c, ipsec.h and ipsec.c to create
> sessions at startup to fix the issue of the first packet
> being dropped for inline crypto.
> 
> The create_session() function is now called at initialisation in
> sa_add_rules() which is called from sa_init().
> The return code for add_rules is checked.
> Calls to create_session() in other functions are dropped.
> 
> Add crypto_devid_fill() in ipsec-secgw.c
> Add max_session_size() in ipsec-secgw.c
> Add check_cryptodev_capability() in ipsec.c
> Add check_cryptodev_aead_capability() in ipsec.c
> Add create_sec_session() and create_crypto_session() in ipsec.c
> 
> The crypto_dev_fill() function has been added to find the
> enabled crypto devices.
> 
> The max_session_size() function has been added to calculate memory
> requirements.
> 
> The check_cryptodev_capability() and check_cryptodev_aead_capability()
> functions have been added to check that the SA is supported by the
> crypto device.
> 
> The create_session() function is refactored to use the
> create_sec_session() and create_crypto_session() functions.
> 
> The cryprodev_init() function has been refactored to drop calls to
> rte_mempool_create() and to drop calculation of memory requirements.
> 
> The main() function has been refactored to call crypto_devid_fill()
> and max_session_size() and to call session_pool_init() and
> session_priv_pool_init().
> The ports are started now before adding a flow rule in main().
> The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
> now called after the ports have been started.
> 
> Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
> Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
> Cc: stable@dpdk.org
> 
> Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> ---
>  examples/ipsec-secgw/ipsec-secgw.c   | 271 +++++++++--------
>  examples/ipsec-secgw/ipsec.c         | 569 +++++++++++++++++++----------------
>  examples/ipsec-secgw/ipsec.h         |  10 +-
>  examples/ipsec-secgw/ipsec_process.c |  38 +--
>  examples/ipsec-secgw/sa.c            |  42 ++-
>  5 files changed, 495 insertions(+), 435 deletions(-)
> 
> diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-
> secgw/ipsec-secgw.c
> index ffbd00b..cc8bb57 100644
> --- a/examples/ipsec-secgw/ipsec-secgw.c
> +++ b/examples/ipsec-secgw/ipsec-secgw.c
> @@ -182,6 +182,14 @@ struct lcore_params {
>  	uint8_t lcore_id;
>  } __rte_cache_aligned;
> 
> +/*
> + * Number of enabled crypto devices
> + * This number is needed when checking crypto device capabilities
> + */
> +uint8_t crypto_dev_num;
> +/* array of crypto device ID's */
> +uint8_t crypto_devid[RTE_CRYPTO_MAX_DEVS];
> +
>  static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS];
> 
>  static struct lcore_params *lcore_params;
> @@ -1623,13 +1631,27 @@ check_cryptodev_mask(uint8_t cdev_id)
>  	return -1;
>  }
> 
> +static void
> +crypto_devid_fill(void)
> +{
> +	uint32_t i, n;
> +
> +	n = rte_cryptodev_count();
> +
> +	for (i = 0; i != n; i++) {
> +		if (check_cryptodev_mask(i) == 0)
> +			crypto_devid[crypto_dev_num++] = i;
> +	}
> +}
> +
>  static int32_t
>  cryptodevs_init(void)
>  {
>  	struct rte_cryptodev_config dev_conf;
>  	struct rte_cryptodev_qp_conf qp_conf;
>  	uint16_t idx, max_nb_qps, qp, i;
> -	int16_t cdev_id, port_id;
> +	int16_t cdev_id;
> +	uint32_t dev_max_sess;
>  	struct rte_hash_parameters params = { 0 };
> 
>  	params.entries = CDEV_MAP_ENTRIES;
> @@ -1652,45 +1674,6 @@ cryptodevs_init(void)
> 
>  	printf("lcore/cryptodev/qp mappings:\n");
> 
> -	uint32_t max_sess_sz = 0, sess_sz;
> -	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
> -		void *sec_ctx;
> -
> -		/* Get crypto priv session size */
> -		sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
> -		if (sess_sz > max_sess_sz)
> -			max_sess_sz = sess_sz;
> -
> -		/*
> -		 * If crypto device is security capable, need to check the
> -		 * size of security session as well.
> -		 */
> -
> -		/* Get security context of the crypto device */
> -		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
> -		if (sec_ctx == NULL)
> -			continue;
> -
> -		/* Get size of security session */
> -		sess_sz = rte_security_session_get_size(sec_ctx);
> -		if (sess_sz > max_sess_sz)
> -			max_sess_sz = sess_sz;
> -	}
> -	RTE_ETH_FOREACH_DEV(port_id) {
> -		void *sec_ctx;
> -
> -		if ((enabled_port_mask & (1 << port_id)) == 0)
> -			continue;
> -
> -		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
> -		if (sec_ctx == NULL)
> -			continue;
> -
> -		sess_sz = rte_security_session_get_size(sec_ctx);
> -		if (sess_sz > max_sess_sz)
> -			max_sess_sz = sess_sz;
> -	}
> -
>  	idx = 0;
>  	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
>  		struct rte_cryptodev_info cdev_info;
> @@ -1722,51 +1705,12 @@ cryptodevs_init(void)
>  		dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id);
>  		dev_conf.nb_queue_pairs = qp;
> 
> -		uint32_t dev_max_sess = cdev_info.sym.max_nb_sessions;
> +		dev_max_sess = cdev_info.sym.max_nb_sessions;
>  		if (dev_max_sess != 0 && dev_max_sess < CDEV_MP_NB_OBJS)
>  			rte_exit(EXIT_FAILURE,
>  				"Device does not support at least %u "
>  				"sessions", CDEV_MP_NB_OBJS);
> 
> -		if (!socket_ctx[dev_conf.socket_id].session_pool) {
> -			char mp_name[RTE_MEMPOOL_NAMESIZE];
> -			struct rte_mempool *sess_mp;
> -
> -			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> -					"sess_mp_%u", dev_conf.socket_id);
> -			sess_mp = rte_cryptodev_sym_session_pool_create(
> -					mp_name, CDEV_MP_NB_OBJS,
> -					0, CDEV_MP_CACHE_SZ, 0,
> -					dev_conf.socket_id);
> -			socket_ctx[dev_conf.socket_id].session_pool =
> sess_mp;
> -		}
> -
> -		if (!socket_ctx[dev_conf.socket_id].session_priv_pool) {
> -			char mp_name[RTE_MEMPOOL_NAMESIZE];
> -			struct rte_mempool *sess_mp;
> -
> -			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> -					"sess_mp_priv_%u",
> dev_conf.socket_id);
> -			sess_mp = rte_mempool_create(mp_name,
> -					CDEV_MP_NB_OBJS,
> -					max_sess_sz,
> -					CDEV_MP_CACHE_SZ,
> -					0, NULL, NULL, NULL,
> -					NULL, dev_conf.socket_id,
> -					0);
> -			socket_ctx[dev_conf.socket_id].session_priv_pool =
> -					sess_mp;
> -		}
> -
> -		if (!socket_ctx[dev_conf.socket_id].session_priv_pool ||
> -				!socket_ctx[dev_conf.socket_id].session_pool)
> -			rte_exit(EXIT_FAILURE,
> -				"Cannot create session pool on socket %d\n",
> -				dev_conf.socket_id);
> -		else
> -			printf("Allocated session pool on socket %d\n",
> -					dev_conf.socket_id);
> -
>  		if (rte_cryptodev_configure(cdev_id, &dev_conf))
>  			rte_panic("Failed to initialize cryptodev %u\n",
>  					cdev_id);
> @@ -1787,38 +1731,6 @@ cryptodevs_init(void)
>  					cdev_id);
>  	}
> 
> -	/* create session pools for eth devices that implement security */
> -	RTE_ETH_FOREACH_DEV(port_id) {
> -		if ((enabled_port_mask & (1 << port_id)) &&
> -				rte_eth_dev_get_sec_ctx(port_id)) {
> -			int socket_id = rte_eth_dev_socket_id(port_id);
> -
> -			if (!socket_ctx[socket_id].session_pool) {
> -				char mp_name[RTE_MEMPOOL_NAMESIZE];
> -				struct rte_mempool *sess_mp;
> -
> -				snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> -						"sess_mp_%u", socket_id);
> -				sess_mp = rte_mempool_create(mp_name,
> -						(CDEV_MP_NB_OBJS * 2),
> -						max_sess_sz,
> -						CDEV_MP_CACHE_SZ,
> -						0, NULL, NULL, NULL,
> -						NULL, socket_id,
> -						0);
> -				if (sess_mp == NULL)
> -					rte_exit(EXIT_FAILURE,
> -						"Cannot create session pool "
> -						"on socket %d\n", socket_id);
> -				else
> -					printf("Allocated session pool "
> -						"on socket %d\n", socket_id);
> -				socket_ctx[socket_id].session_pool = sess_mp;
> -			}
> -		}
> -	}
> -
> -
>  	printf("\n");
> 
>  	return 0;
> @@ -1984,6 +1896,98 @@ port_init(uint16_t portid, uint64_t req_rx_offloads,
> uint64_t req_tx_offloads)
>  	printf("\n");
>  }
> 
> +static size_t
> +max_session_size(void)
> +{
> +	size_t max_sz, sz;
> +	void *sec_ctx;
> +	int16_t cdev_id, port_id, n;
> +
> +	max_sz = 0;
> +	n =  rte_cryptodev_count();
> +	for (cdev_id = 0; cdev_id != n; cdev_id++) {
> +		sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
> +		if (sz > max_sz)
> +			max_sz = sz;
> +		/*
> +		 * If crypto device is security capable, need to check the
> +		 * size of security session as well.
> +		 */
> +
> +		/* Get security context of the crypto device */
> +		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
> +		if (sec_ctx == NULL)
> +			continue;
> +
> +		/* Get size of security session */
> +		sz = rte_security_session_get_size(sec_ctx);
> +		if (sz > max_sz)
> +			max_sz = sz;
> +	}
> +
> +	RTE_ETH_FOREACH_DEV(port_id) {
> +		if ((enabled_port_mask & (1 << port_id)) == 0)
> +			continue;
> +
> +		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
> +		if (sec_ctx == NULL)
> +			continue;
> +
> +		sz = rte_security_session_get_size(sec_ctx);
> +		if (sz > max_sz)
> +			max_sz = sz;
> +	}
> +
> +	return max_sz;
> +}
> +
> +static void
> +session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz)
> +{
> +	char mp_name[RTE_MEMPOOL_NAMESIZE];
> +	struct rte_mempool *sess_mp;
> +
> +	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> +			"sess_mp_%u", socket_id);
> +	sess_mp = rte_cryptodev_sym_session_pool_create(
> +			mp_name, CDEV_MP_NB_OBJS,
> +			sess_sz, CDEV_MP_CACHE_SZ, 0,
> +			socket_id);
> +	ctx->session_pool = sess_mp;
> +
> +	if (ctx->session_pool == NULL)
> +		rte_exit(EXIT_FAILURE,
> +			"Cannot init session pool on socket %d\n", socket_id);
> +	else
> +		printf("Allocated session pool on socket %d\n",	socket_id);
> +}
> +
> +static void
> +session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
> +	size_t sess_sz)
> +{
> +	char mp_name[RTE_MEMPOOL_NAMESIZE];
> +	struct rte_mempool *sess_mp;
> +
> +	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> +			"sess_mp_priv_%u", socket_id);
> +	sess_mp = rte_mempool_create(mp_name,
> +			CDEV_MP_NB_OBJS,
> +			sess_sz,
> +			CDEV_MP_CACHE_SZ,
> +			0, NULL, NULL, NULL,
> +			NULL, socket_id,
> +			0);
> +	ctx->session_priv_pool = sess_mp;
> +	if (ctx->session_priv_pool == NULL)
> +		rte_exit(EXIT_FAILURE,
> +			"Cannot init session priv pool on socket %d\n",
> +			socket_id);
> +	else
> +		printf("Allocated session priv pool on socket %d\n",
> +			socket_id);
> +}
> +
>  static void
>  pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf)
>  {
> @@ -2064,9 +2068,11 @@ main(int32_t argc, char **argv)
>  {
>  	int32_t ret;
>  	uint32_t lcore_id;
> +	uint32_t i;
>  	uint8_t socket_id;
>  	uint16_t portid;
>  	uint64_t req_rx_offloads, req_tx_offloads;
> +	size_t sess_sz;
> 
>  	/* init EAL */
>  	ret = rte_eal_init(argc, argv);
> @@ -2094,7 +2100,10 @@ main(int32_t argc, char **argv)
> 
>  	nb_lcores = rte_lcore_count();
> 
> -	/* Replicate each context per socket */
> +	crypto_devid_fill();
> +
> +	sess_sz = max_session_size();
> +
>  	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
>  		if (rte_lcore_is_enabled(lcore_id) == 0)
>  			continue;
> @@ -2104,20 +2113,17 @@ main(int32_t argc, char **argv)
>  		else
>  			socket_id = 0;
> 
> +		/* mbuf_pool is initialised by the pool_init() function*/
>  		if (socket_ctx[socket_id].mbuf_pool)
>  			continue;
> 
> -		/* initilaze SPD */
> -		sp4_init(&socket_ctx[socket_id], socket_id);
> -
> -		sp6_init(&socket_ctx[socket_id], socket_id);
> -
> -		/* initilaze SAD */
> -		sa_init(&socket_ctx[socket_id], socket_id);
> -
> -		rt_init(&socket_ctx[socket_id], socket_id);
> -
>  		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
> +		session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
> +		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
> +			sess_sz);
> +
> +		if (!numa_on)
> +			break;
>  	}
> 
>  	RTE_ETH_FOREACH_DEV(portid) {
> @@ -2135,7 +2141,11 @@ main(int32_t argc, char **argv)
>  		if ((enabled_port_mask & (1 << portid)) == 0)
>  			continue;
> 
> -		/* Start device */
> +		/*
> +		 * Start device
> +		 * note: device must be started before a flow rule
> +		 * can be installed.
> +		 */
>  		ret = rte_eth_dev_start(portid);
>  		if (ret < 0)
>  			rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
> @@ -2153,6 +2163,19 @@ main(int32_t argc, char **argv)
>  			RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback,
> NULL);
>  	}
> 
> +	/* Replicate each context per socket */
> +	for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
> +		socket_id = rte_socket_id_by_idx(i);
> +		if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
> +			(socket_ctx[socket_id].sa_in == NULL) &&
> +			(socket_ctx[socket_id].sa_out == NULL)) {
> +			sa_init(&socket_ctx[socket_id], socket_id);
> +			sp4_init(&socket_ctx[socket_id], socket_id);
> +			sp6_init(&socket_ctx[socket_id], socket_id);
> +			rt_init(&socket_ctx[socket_id], socket_id);
> +		}
> +	}
> +
>  	check_all_ports_link_status(enabled_port_mask);
> 
>  	/* launch per-lcore init on every lcore */
> diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
> index 4352cb8..e31d472 100644
> --- a/examples/ipsec-secgw/ipsec.c
> +++ b/examples/ipsec-secgw/ipsec.c
> @@ -39,42 +39,17 @@ set_ipsec_conf(struct ipsec_sa *sa, struct
> rte_security_ipsec_xform *ipsec)
>  	ipsec->esn_soft_limit = IPSEC_OFFLOAD_ESN_SOFTLIMIT;
>  }
> 
> -int
> -create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
> +static int
> +create_sec_session(struct ipsec_sa *sa, struct rte_mempool *pool)
>  {
> -	struct rte_cryptodev_info cdev_info;
> -	unsigned long cdev_id_qp = 0;
> +	const struct rte_security_capability *sec_cap;
> +	struct rte_security_ctx *ctx;
>  	int32_t ret = 0;
> -	struct cdev_key key = { 0 };
> -
> -	key.lcore_id = (uint8_t)rte_lcore_id();
> -
> -	key.cipher_algo = (uint8_t)sa->cipher_algo;
> -	key.auth_algo = (uint8_t)sa->auth_algo;
> -	key.aead_algo = (uint8_t)sa->aead_algo;
> -
> -	if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {
> -		ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
> -				(void **)&cdev_id_qp);
> -		if (ret < 0) {
> -			RTE_LOG(ERR, IPSEC,
> -				"No cryptodev: core %u, cipher_algo %u, "
> -				"auth_algo %u, aead_algo %u\n",
> -				key.lcore_id,
> -				key.cipher_algo,
> -				key.auth_algo,
> -				key.aead_algo);
> -			return -1;
> -		}
> -	}
> 
> -	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on cryptodev
> "
> -			"%u qp %u\n", sa->spi,
> -			ipsec_ctx->tbl[cdev_id_qp].id,
> -			ipsec_ctx->tbl[cdev_id_qp].qp);
> +	if ((sa == NULL) || (pool == NULL))
> +		return -EINVAL;
> 
> -	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
> -		struct rte_security_session_conf sess_conf = {
> +	struct rte_security_session_conf sess_conf = {
>  			.action_type = sa->type,
>  			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
>  			{.ipsec = {
> @@ -90,247 +65,340 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct
> ipsec_sa *sa)
>  			} },
>  			.crypto_xform = sa->xforms,
>  			.userdata = NULL,
> -
>  		};
> 
> -		if (sa->type ==
> RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> -			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> -
> 	rte_cryptodev_get_sec_ctx(
> -							ipsec_ctx-
> >tbl[cdev_id_qp].id);
> -
> -			/* Set IPsec parameters in conf */
> -			set_ipsec_conf(sa, &(sess_conf.ipsec));
> -
> -			sa->sec_session = rte_security_session_create(ctx,
> -					&sess_conf, ipsec_ctx->session_pool);
> -			if (sa->sec_session == NULL) {
> -				RTE_LOG(ERR, IPSEC,
> -				"SEC Session init failed: err: %d\n", ret);
> -				return -1;
> -			}
> -		} else if (sa->type ==
> RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
> -			struct rte_flow_error err;
> -			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> -
> 	rte_eth_dev_get_sec_ctx(
> -							sa->portid);
> -			const struct rte_security_capability *sec_cap;
> -			int ret = 0;
> -
> -			sa->sec_session = rte_security_session_create(ctx,
> -					&sess_conf, ipsec_ctx->session_pool);
> -			if (sa->sec_session == NULL) {
> -				RTE_LOG(ERR, IPSEC,
> -				"SEC Session init failed: err: %d\n", ret);
> -				return -1;
> -			}
> +	if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> +		ctx = (struct rte_security_ctx *)
> +				rte_eth_dev_get_sec_ctx(sa->portid);
> 
> -			sec_cap = rte_security_capabilities_get(ctx);
> +		/* Set IPsec parameters in conf */
> +		set_ipsec_conf(sa, &(sess_conf.ipsec));
> 
> -			/* iterate until ESP tunnel*/
> -			while (sec_cap->action !=
> -					RTE_SECURITY_ACTION_TYPE_NONE)
> {
> +		sa->sec_session = rte_security_session_create(ctx,
> +				&sess_conf, pool);
> +		if (sa->sec_session == NULL) {
> +			RTE_LOG(ERR, IPSEC,
> +				"SEC Session init failed: err: %d\n",
> +				ret);
> +			return -1;
> +		}
> +	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
> +		struct rte_flow_error err;
> +		ctx = (struct rte_security_ctx *)
> +				rte_eth_dev_get_sec_ctx(sa->portid);
> +		sa->sec_session = rte_security_session_create(ctx,
> +				&sess_conf, pool);
> +		if (sa->sec_session == NULL) {
> +			RTE_LOG(ERR, IPSEC, "SEC Session init failed\n");
> +			return -1;
> +		}
> 
> -				if (sec_cap->action == sa->type &&
> -				    sec_cap->protocol ==
> -					RTE_SECURITY_PROTOCOL_IPSEC &&
> -				    sec_cap->ipsec.mode ==
> -
> 	RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
> -				    sec_cap->ipsec.direction == sa->direction)
> -					break;
> -				sec_cap++;
> -			}
> +		sec_cap = rte_security_capabilities_get(ctx);
> +
> +		/* iterate until ESP tunnel*/
> +		while (sec_cap->action != RTE_SECURITY_ACTION_TYPE_NONE)
> {
> +			if (sec_cap->action == sa->type &&
> +				sec_cap->protocol ==
> +				RTE_SECURITY_PROTOCOL_IPSEC &&
> +				sec_cap->ipsec.mode ==
> +				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
> +				sec_cap->ipsec.direction == sa->direction)
> +				break;
> +			sec_cap++;
> +		}
> 
> -			if (sec_cap->action ==
> RTE_SECURITY_ACTION_TYPE_NONE) {
> -				RTE_LOG(ERR, IPSEC,
> +		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
> +			RTE_LOG(ERR, IPSEC,
>  				"No suitable security capability found\n");
>  				return -1;
> -			}
> +		}
> 
> -			sa->ol_flags = sec_cap->ol_flags;
> -			sa->security_ctx = ctx;
> -			sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
> -
> -			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
> -			sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
> -			if (sa->flags & IP6_TUNNEL) {
> -				sa->pattern[1].spec = &sa->ipv6_spec;
> -				memcpy(sa->ipv6_spec.hdr.dst_addr,
> -					sa->dst.ip.ip6.ip6_b, 16);
> -				memcpy(sa->ipv6_spec.hdr.src_addr,
> -				       sa->src.ip.ip6.ip6_b, 16);
> -			} else {
> -				sa->pattern[1].spec = &sa->ipv4_spec;
> -				sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
> -				sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
> -			}
> +		sa->ol_flags = sec_cap->ol_flags;
> +		sa->security_ctx = ctx;
> +		sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
> +
> +		sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
> +		sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
> +		if (sa->flags & IP6_TUNNEL) {
> +			sa->pattern[1].spec = &sa->ipv6_spec;
> +			memcpy(sa->ipv6_spec.hdr.dst_addr,
> +				sa->dst.ip.ip6.ip6_b, 16);
> +			memcpy(sa->ipv6_spec.hdr.src_addr,
> +			       sa->src.ip.ip6.ip6_b, 16);
> +		} else {
> +			sa->pattern[1].spec = &sa->ipv4_spec;
> +			sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
> +			sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
> +		}
> 
> -			sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
> -			sa->pattern[2].spec = &sa->esp_spec;
> -			sa->pattern[2].mask = &rte_flow_item_esp_mask;
> -			sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
> +		sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
> +		sa->pattern[2].spec = &sa->esp_spec;
> +		sa->pattern[2].mask = &rte_flow_item_esp_mask;
> +		sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
> 
> -			sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
> +		sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
> 
> -			sa->action[0].type =
> RTE_FLOW_ACTION_TYPE_SECURITY;
> -			sa->action[0].conf = sa->sec_session;
> +		sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
> +		sa->action[0].conf = sa->sec_session;
> 
> -			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
> +		sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
> 
> -			sa->attr.egress = (sa->direction ==
> +		sa->attr.egress = (sa->direction ==
> 
> 	RTE_SECURITY_IPSEC_SA_DIR_EGRESS);
> -			sa->attr.ingress = (sa->direction ==
> +		sa->attr.ingress = (sa->direction ==
> 
> 	RTE_SECURITY_IPSEC_SA_DIR_INGRESS);
> -			if (sa->attr.ingress) {
> -				uint8_t rss_key[40];
> -				struct rte_eth_rss_conf rss_conf = {
> -					.rss_key = rss_key,
> -					.rss_key_len = 40,
> -				};
> -				struct rte_eth_dev *eth_dev;
> -				uint16_t
> queue[RTE_MAX_QUEUES_PER_PORT];
> -				struct rte_flow_action_rss action_rss;
> -				unsigned int i;
> -				unsigned int j;
> -
> -				sa->action[2].type =
> RTE_FLOW_ACTION_TYPE_END;
> -				/* Try RSS. */
> -				sa->action[1].type =
> RTE_FLOW_ACTION_TYPE_RSS;
> -				sa->action[1].conf = &action_rss;
> -				eth_dev = ctx->device;
> -				rte_eth_dev_rss_hash_conf_get(sa->portid,
> -							      &rss_conf);
> -				for (i = 0, j = 0;
> -				     i < eth_dev->data->nb_rx_queues; ++i)
> -					if (eth_dev->data->rx_queues[i])
> -						queue[j++] = i;
> +		if (sa->attr.ingress) {
> +			uint8_t rss_key[40];
> +			struct rte_eth_rss_conf rss_conf = {
> +				.rss_key = rss_key,
> +				.rss_key_len = 40,
> +			};
> +			struct rte_eth_dev *eth_dev;
> +			uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
> +			struct rte_flow_action_rss action_rss;
> +			unsigned int i;
> +			unsigned int j;
> +
> +			sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
> +			/* Try RSS. */
> +			sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
> +			sa->action[1].conf = &action_rss;
> +			eth_dev = ctx->device;
> +			rte_eth_dev_rss_hash_conf_get(sa->portid,
> +						&rss_conf);
> +			for (i = 0, j = 0; i < eth_dev->data->nb_rx_queues;
> +				++i)
> +				if (eth_dev->data->rx_queues[i])
> +					queue[j++] = i;
Compilation error

/home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c: In function 'create_sec_session':
/home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c:169:4: error: this 'for' clause does not guard... [-Werror=misleading-indentation]
    for (i = 0, j = 0; i < eth_dev->data->nb_rx_queues;
    ^~~
/home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c:173:5: note: ...this statement, but the latter is misleadingly indented as if it is guarded by the 'for'
     action_rss = (struct rte_flow_action_rss){
     ^~~~~~~~~~

>  				action_rss = (struct rte_flow_action_rss){
>  					.types = rss_conf.rss_hf,
>  					.key_len = rss_conf.rss_key_len,
>  					.queue_num = j,
>  					.key = rss_key,
>  					.queue = queue,
> -				};
> -				ret = rte_flow_validate(sa->portid, &sa->attr,
> -							sa->pattern, sa-
> >action,
> -							&err);
> -				if (!ret)
> -					goto flow_create;
> -				/* Try Queue. */
> -				sa->action[1].type =
> RTE_FLOW_ACTION_TYPE_QUEUE;
> -				sa->action[1].conf =
> -					&(struct rte_flow_action_queue){
> -					.index = 0,
> -				};
> -				ret = rte_flow_validate(sa->portid, &sa->attr,
> -							sa->pattern, sa-
> >action,
> -							&err);
> -				/* Try End. */
> -				sa->action[1].type =
> RTE_FLOW_ACTION_TYPE_END;
> -				sa->action[1].conf = NULL;
> -				ret = rte_flow_validate(sa->portid, &sa->attr,
> +			};
> +			ret = rte_flow_validate(sa->portid, &sa->attr,
> +						sa->pattern, sa->action,
> +						&err);
> +			if (!ret)
> +				goto flow_create;
> +			/* Try Queue. */
> +			sa->action[1].type = RTE_FLOW_ACTION_TYPE_QUEUE;
> +			sa->action[1].conf =
> +				&(struct rte_flow_action_queue){
> +				.index = 0,
> +			};
> +			ret = rte_flow_validate(sa->portid, &sa->attr,
> +						sa->pattern, sa->action,
> +						&err);
> +			/* Try End. */
> +			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
> +			sa->action[1].conf = NULL;
> +			ret = rte_flow_validate(sa->portid, &sa->attr,
>  							sa->pattern, sa-
> >action,
>  							&err);
> -				if (ret)
> -					goto flow_create_failure;
> -			} else if (sa->attr.egress &&
> -				   (sa->ol_flags &
> -				    RTE_SECURITY_TX_HW_TRAILER_OFFLOAD))
> {
> -				sa->action[1].type =
> -					RTE_FLOW_ACTION_TYPE_PASSTHRU;
> -				sa->action[2].type =
> -					RTE_FLOW_ACTION_TYPE_END;
> -			}
> +			if (ret)
> +				goto flow_create_failure;
> +		} else if (sa->attr.egress &&
> +				(sa->ol_flags &
> +				RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
> +			sa->action[1].type =
> +				RTE_FLOW_ACTION_TYPE_PASSTHRU;
> +			sa->action[2].type =
> +				RTE_FLOW_ACTION_TYPE_END;
> +		}
>  flow_create:
> -			sa->flow = rte_flow_create(sa->portid,
> -				&sa->attr, sa->pattern, sa->action, &err);
> -			if (sa->flow == NULL) {
> +		sa->flow = rte_flow_create(sa->portid,
> +			&sa->attr, sa->pattern, sa->action, &err);
> +		if (sa->flow == NULL) {
>  flow_create_failure:
> -				RTE_LOG(ERR, IPSEC,
> -					"Failed to create ipsec flow msg: %s\n",
> -					err.message);
> -				return -1;
> -			}
> -		} else if (sa->type ==
> -
> 	RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
> -			struct rte_security_ctx *ctx =
> -					(struct rte_security_ctx *)
> -					rte_eth_dev_get_sec_ctx(sa->portid);
> -			const struct rte_security_capability *sec_cap;
> -
> -			if (ctx == NULL) {
> -				RTE_LOG(ERR, IPSEC,
> -				"Ethernet device doesn't have security features
> registered\n");
> -				return -1;
> -			}
> +			RTE_LOG(ERR, IPSEC,
> +				"Failed to create ipsec flow msg: %s\n",
> +				err.message);
> +			return -1;
> +		}
> +	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
> {
> +		struct rte_security_ctx *ctx =
> +				(struct rte_security_ctx *)
> +				rte_eth_dev_get_sec_ctx(sa->portid);
> +		const struct rte_security_capability *sec_cap;
> 
> -			/* Set IPsec parameters in conf */
> -			set_ipsec_conf(sa, &(sess_conf.ipsec));
> -
> -			/* Save SA as userdata for the security session. When
> -			 * the packet is received, this userdata will be
> -			 * retrieved using the metadata from the packet.
> -			 *
> -			 * The PMD is expected to set similar metadata for
> other
> -			 * operations, like rte_eth_event, which are tied to
> -			 * security session. In such cases, the userdata could
> -			 * be obtained to uniquely identify the security
> -			 * parameters denoted.
> -			 */
> -
> -			sess_conf.userdata = (void *) sa;
> -
> -			sa->sec_session = rte_security_session_create(ctx,
> -					&sess_conf, ipsec_ctx->session_pool);
> -			if (sa->sec_session == NULL) {
> -				RTE_LOG(ERR, IPSEC,
> -				"SEC Session init failed: err: %d\n", ret);
> -				return -1;
> -			}
> +		if (ctx == NULL) {
> +			RTE_LOG(ERR, IPSEC,
> +			"Ethernet device doesn't have security features
> registered\n");
> +			return -1;
> +		}
> +
> +		/* Set IPsec parameters in conf */
> +		set_ipsec_conf(sa, &(sess_conf.ipsec));
> +
> +		/* Save SA as userdata for the security session. When
> +		 * the packet is received, this userdata will be
> +		 * retrieved using the metadata from the packet.
> +		 *
> +		 * The PMD is expected to set similar metadata for other
> +		 * operations, like rte_eth_event, which are tied to
> +		 * security session. In such cases, the userdata could
> +		 * be obtained to uniquely identify the security
> +		 * parameters denoted.
> +		 */
> +
> +		sess_conf.userdata = (void *) sa;
> +
> +		sa->sec_session = rte_security_session_create(ctx,
> +			&sess_conf, pool);
> +		if (sa->sec_session == NULL) {
> +			RTE_LOG(ERR, IPSEC,
> +			"SEC Session init failed: err: %d\n", ret);
> +			return -1;
> +		}
> 
> -			sec_cap = rte_security_capabilities_get(ctx);
> +		sec_cap = rte_security_capabilities_get(ctx);
> 
> -			if (sec_cap == NULL) {
> -				RTE_LOG(ERR, IPSEC,
> -				"No capabilities registered\n");
> -				return -1;
> -			}
> +		if (sec_cap == NULL) {
> +			RTE_LOG(ERR, IPSEC,
> +			"No capabilities registered\n");
> +			return -1;
> +		}
> 
> -			/* iterate until ESP tunnel*/
> -			while (sec_cap->action !=
> +		/* iterate until ESP tunnel*/
> +		while (sec_cap->action !=
>  					RTE_SECURITY_ACTION_TYPE_NONE)
> {
> 
> -				if (sec_cap->action == sa->type &&
> -				    sec_cap->protocol ==
> -					RTE_SECURITY_PROTOCOL_IPSEC &&
> -				    sec_cap->ipsec.mode ==
> -
> 	RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
> -				    sec_cap->ipsec.direction == sa->direction)
> -					break;
> -				sec_cap++;
> -			}
> +			if (sec_cap->action == sa->type &&
> +				sec_cap->protocol ==
> +				RTE_SECURITY_PROTOCOL_IPSEC &&
> +				sec_cap->ipsec.mode ==
> +				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
> +				sec_cap->ipsec.direction == sa->direction)
> +				break;
> +			sec_cap++;
> +		}
> 
> -			if (sec_cap->action ==
> RTE_SECURITY_ACTION_TYPE_NONE) {
> -				RTE_LOG(ERR, IPSEC,
> -				"No suitable security capability found\n");
> -				return -1;
> -			}
> +		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
> +			RTE_LOG(ERR, IPSEC,
> +			"No suitable security capability found\n");
> +			return -1;
> +		}
> +
> +		sa->ol_flags = sec_cap->ol_flags;
> +		sa->security_ctx = ctx;
> +	} else
> +		return -EINVAL;
> +	return 0;
> +}
> +
> +#define CDEV_IV_SIZE 12
> +
> +static int
> +check_cryptodev_aead_capablity(const struct ipsec_sa *ss, uint8_t dev_id)
> +{
> +	struct rte_cryptodev_sym_capability_idx cap_idx;
> +	const struct rte_cryptodev_symmetric_capability *cap;
> 
> -			sa->ol_flags = sec_cap->ol_flags;
> -			sa->security_ctx = ctx;
> +	cap_idx.type = RTE_CRYPTO_SYM_XFORM_AEAD;
> +	cap_idx.algo.aead = ss->aead_algo;
> +
> +	cap = rte_cryptodev_sym_capability_get(dev_id, &cap_idx);
> +	if (cap == NULL)
> +		return -ENOENT;
> +
> +	return rte_cryptodev_sym_capability_check_aead(cap,
> +					ss->cipher_key_len,
> +					ss->digest_len,
> +					ss->aad_len,
> +					CDEV_IV_SIZE);
> +}
> +
> +static int
> +check_cryptodev_capablity(const struct ipsec_sa *ss, uint8_t dev_id)
> +{
> +	struct rte_cryptodev_sym_capability_idx cap_idx;
> +	const struct rte_cryptodev_symmetric_capability *cap;
> +	uint16_t auth_iv_len;
> +	int rc = -1;
> +
> +	if (ss == NULL)
> +		return rc;
> +
> +	if (ss->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
> +		return check_cryptodev_aead_capablity(ss, dev_id);
> +
> +	auth_iv_len = 0;
> +
> +	cap_idx.type = RTE_CRYPTO_SYM_XFORM_AUTH;
> +	cap_idx.algo.auth = ss->auth_algo;
> +	cap = rte_cryptodev_sym_capability_get(dev_id, &cap_idx);
> +	if (cap != NULL) {
> +		rc = rte_cryptodev_sym_capability_check_auth(
> +				cap, ss->auth_key_len, ss->digest_len,
> +				auth_iv_len);
> +		if (rc == 0) {
> +			cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
> +			cap_idx.algo.cipher = ss->cipher_algo;
> +			cap = rte_cryptodev_sym_capability_get(dev_id,
> +					&cap_idx);
> +			if (cap != NULL)
> +				rc =
> rte_cryptodev_sym_capability_check_cipher(
> +						cap, ss->cipher_key_len,
> +						ss->iv_len);
>  		}
> -	} else {
> -		sa->crypto_session = rte_cryptodev_sym_session_create(
> -				ipsec_ctx->session_pool);
> -		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
> -				sa->crypto_session, sa->xforms,
> -				ipsec_ctx->session_priv_pool);
> -
> -		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
> -				&cdev_info);
>  	}
> -	sa->cdev_id_qp = cdev_id_qp;
> 
> -	return 0;
> +	return rc;
> +}
> +
> +static int
> +create_crypto_session(struct ipsec_sa *sa, struct rte_mempool *pool)
> +{
> +	int32_t rc;
> +	uint32_t devnum, i;
> +	struct rte_cryptodev_sym_session *s;
> +	uint8_t devid[RTE_CRYPTO_MAX_DEVS];
> +
> +	/* check which cryptodevs support SA */
> +	devnum = 0;
> +	for (i = 0; i < crypto_dev_num; i++) {
> +		rc = check_cryptodev_capablity(sa, crypto_devid[i]);
> +		if (rc == 0)
> +			devid[devnum++] = crypto_devid[i];
> +	}
> +
> +	if (devnum == 0)
> +		return -ENODEV;
> +
> +	s = rte_cryptodev_sym_session_create(pool);
> +	if (s == NULL)
> +		return -ENOMEM;
> +
> +	/* initialize SA crypto session for all supported devices */
> +	for (i = 0; i != devnum; i++) {
> +		rc = rte_cryptodev_sym_session_init(devid[i], s, sa->xforms,
> +			pool);
> +		if (rc != 0)
> +			break;
> +	}
> +
> +	if (i == devnum) {
> +		sa->crypto_session = s;
> +		return 0;
> +	}
> +
> +	/* failure, do cleanup */
> +	while (i-- != 0)
> +		rte_cryptodev_sym_session_clear(devid[i], s);
> +
> +	rte_cryptodev_sym_session_free(s);
> +	return rc;
> +}
> +
> +int
> +create_session(struct ipsec_sa *sa, struct rte_mempool *pool)
> +{
> +	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE)
> +		return create_sec_session(sa, pool);
> +	else
> +		return create_crypto_session(sa, pool);
>  }
> 
>  /*
> @@ -393,13 +461,6 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
> ipsec_ctx *ipsec_ctx,
>  			priv->cop.status =
> RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
> 
>  			rte_prefetch0(&priv->sym_cop);
> -
> -			if ((unlikely(sa->sec_session == NULL)) &&
> -					create_session(ipsec_ctx, sa)) {
> -				rte_pktmbuf_free(pkts[i]);
> -				continue;
> -			}
> -
>  			sym_cop = get_sym_cop(&priv->cop);
>  			sym_cop->m_src = pkts[i];
> 
> @@ -412,13 +473,6 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
> ipsec_ctx *ipsec_ctx,
>  			priv->cop.status =
> RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
> 
>  			rte_prefetch0(&priv->sym_cop);
> -
> -			if ((unlikely(sa->crypto_session == NULL)) &&
> -					create_session(ipsec_ctx, sa)) {
> -				rte_pktmbuf_free(pkts[i]);
> -				continue;
> -			}
> -
>  			rte_crypto_op_attach_sym_session(&priv->cop,
>  					sa->crypto_session);
> 
> @@ -429,12 +483,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
> ipsec_ctx *ipsec_ctx,
>  			}
>  			break;
>  		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
> -			if ((unlikely(sa->sec_session == NULL)) &&
> -					create_session(ipsec_ctx, sa)) {
> -				rte_pktmbuf_free(pkts[i]);
> -				continue;
> -			}
> -
> +			RTE_ASSERT(sa->sec_session != NULL);
>  			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
>  			if (sa->ol_flags &
> RTE_SECURITY_TX_OLOAD_NEED_MDATA)
>  				rte_security_set_pkt_metadata(
> @@ -442,17 +491,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
> ipsec_ctx *ipsec_ctx,
>  						sa->sec_session, pkts[i], NULL);
>  			continue;
>  		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
> +			RTE_ASSERT(sa->sec_session != NULL);
>  			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
>  			priv->cop.status =
> RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
> 
>  			rte_prefetch0(&priv->sym_cop);
> -
> -			if ((unlikely(sa->sec_session == NULL)) &&
> -					create_session(ipsec_ctx, sa)) {
> -				rte_pktmbuf_free(pkts[i]);
> -				continue;
> -			}
> -
>  			rte_security_attach_session(&priv->cop,
>  					sa->sec_session);
> 
> diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
> index 99f49d6..804330c 100644
> --- a/examples/ipsec-secgw/ipsec.h
> +++ b/examples/ipsec-secgw/ipsec.h
> @@ -83,6 +83,14 @@ struct app_sa_prm {
> 
>  extern struct app_sa_prm app_sa_prm;
> 
> +/*
> + * Number of enabled crypto devices
> + * This number is needed when checking crypto device capabilities
> + */
> +extern uint8_t crypto_dev_num;
> +/* array of crypto device ID's */
> +extern uint8_t crypto_devid[RTE_CRYPTO_MAX_DEVS];
> +
>  struct ipsec_sa {
>  	struct rte_ipsec_session ips; /* one session per sa for now */
>  	uint32_t spi;
> @@ -306,6 +314,6 @@ void
>  enqueue_cop_burst(struct cdev_qp *cqp);
> 
>  int
> -create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
> +create_session(struct ipsec_sa *sa, struct rte_mempool *pool);
> 
>  #endif /* __IPSEC_H__ */
> diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-
> secgw/ipsec_process.c
> index 3f9cacb..0df6969 100644
> --- a/examples/ipsec-secgw/ipsec_process.c
> +++ b/examples/ipsec-secgw/ipsec_process.c
> @@ -86,39 +86,6 @@ enqueue_cop_bulk(struct cdev_qp *cqp, struct
> rte_crypto_op *cop[], uint32_t num)
>  	cqp->len = len;
>  }
> 
> -static inline int
> -fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
> -	struct ipsec_sa *sa)
> -{
> -	int32_t rc;
> -
> -	/* setup crypto section */
> -	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
> -		if (sa->crypto_session == NULL) {
> -			rc = create_session(ctx, sa);
> -			if (rc != 0)
> -				return rc;
> -		}
> -		ss->crypto.ses = sa->crypto_session;
> -	/* setup session action type */
> -	} else {
> -		if (sa->sec_session == NULL) {
> -			rc = create_session(ctx, sa);
> -			if (rc != 0)
> -				return rc;
> -		}
> -		ss->security.ses = sa->sec_session;
> -		ss->security.ctx = sa->security_ctx;
> -		ss->security.ol_flags = sa->ol_flags;
> -	}
> -
> -	rc = rte_ipsec_session_prepare(ss);
> -	if (rc != 0)
> -		memset(ss, 0, sizeof(*ss));
> -
> -	return rc;
> -}
> -
>  /*
>   * group input packets byt the SA they belong to.
>   */
> @@ -219,9 +186,8 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic
> *trf)
> 
>  		ips = &sa->ips;
> 
> -		/* no valid HW session for that SA, try to create one */
> -		if (sa == NULL || (ips->crypto.ses == NULL &&
> -				fill_ipsec_session(ips, ctx, sa) != 0))
> +		/* no valid HW session for that SA */
> +		if (sa == NULL || ips->crypto.ses == NULL)
>  			k = 0;
> 
>  		/* process packets inline */
> diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
> index a7298a3..0f36f5b 100644
> --- a/examples/ipsec-secgw/sa.c
> +++ b/examples/ipsec-secgw/sa.c
> @@ -774,14 +774,14 @@ check_eth_dev_caps(uint16_t portid, uint32_t
> inbound)
>  	return 0;
>  }
> 
> -
>  static int
>  sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> -		uint32_t nb_entries, uint32_t inbound)
> +	uint32_t nb_entries, uint32_t inbound, struct socket_ctx *skt_ctx)
>  {
>  	struct ipsec_sa *sa;
>  	uint32_t i, idx;
>  	uint16_t iv_length, aad_length;
> +	int32_t rc;
> 
>  	/* for ESN upper 32 bits of SQN also need to be part of AAD */
>  	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
> @@ -902,6 +902,12 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct
> ipsec_sa entries[],
> 
>  			print_one_sa_rule(sa, inbound);
>  		}
> +		rc = create_session(sa, skt_ctx->session_pool);
> +		if (rc != 0) {
> +			RTE_LOG(ERR, IPSEC_ESP,
> +					"create_session() failed\n");
> +			return -EINVAL;
> +		}
>  	}
> 
>  	return 0;
> @@ -909,16 +915,16 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct
> ipsec_sa entries[],
> 
>  static inline int
>  sa_out_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> -		uint32_t nb_entries)
> +		uint32_t nb_entries, struct socket_ctx *skt_ctx)
>  {
> -	return sa_add_rules(sa_ctx, entries, nb_entries, 0);
> +	return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx);
>  }
> 
>  static inline int
>  sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> -		uint32_t nb_entries)
> +		uint32_t nb_entries, struct socket_ctx *skt_ctx)
>  {
> -	return sa_add_rules(sa_ctx, entries, nb_entries, 1);
> +	return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx);
>  }
> 
>  /*
> @@ -1012,10 +1018,12 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm,
> const struct ipsec_sa *ss,
>  	return 0;
>  }
> 
> -static void
> +static int
>  fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
>  	const struct ipsec_sa *lsa)
>  {
> +	int32_t rc = 0;
> +
>  	ss->sa = sa;
>  	ss->type = lsa->type;
> 
> @@ -1028,6 +1036,12 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct
> rte_ipsec_sa *sa,
>  		ss->security.ctx = lsa->security_ctx;
>  		ss->security.ol_flags = lsa->ol_flags;
>  	}
> +
> +	rc = rte_ipsec_session_prepare(ss);
> +	if (rc != 0)
> +		memset(ss, 0, sizeof(*ss));
> +
> +	return rc;
>  }
> 
>  /*
> @@ -1062,8 +1076,8 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa
> *sa, uint32_t sa_size)
>  	if (rc < 0)
>  		return rc;
> 
> -	fill_ipsec_session(&lsa->ips, sa, lsa);
> -	return 0;
> +	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
> +	return rc;
>  }
> 
>  /*
> @@ -1141,7 +1155,10 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
>  				"context %s in socket %d\n", rte_errno,
>  				name, socket_id);
> 
> -		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in);
> +		rc = sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx);
> +		if (rc != 0)
> +			rte_exit(EXIT_FAILURE,
> +				"failed to add inbound rules\n");
> 
>  		if (app_sa_prm.enable != 0) {
>  			rc = ipsec_satbl_init(ctx->sa_in, sa_in, nb_sa_in,
> @@ -1161,7 +1178,10 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
>  				"context %s in socket %d\n", rte_errno,
>  				name, socket_id);
> 
> -		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out);
> +		rc = sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx);
> +		if (rc != 0)
> +			rte_exit(EXIT_FAILURE,
> +				"failed to add outbound rules\n");
> 
>  		if (app_sa_prm.enable != 0) {
>  			rc = ipsec_satbl_init(ctx->sa_out, sa_out, nb_sa_out,
> --
> 2.7.4

Re: [dpdk-dev] [PATCH v3 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Akhil Goyal]

> 
> Inline crypto installs a flow rule in the NIC. This flow
> rule must be installed before the first inbound packet is
> received.
> 
> The create_session() function installs the flow rule.
> 
> Refactor ipsec-secgw.c, sa.c, ipsec.h and ipsec.c to create
> sessions at startup to fix the issue of the first packet
> being dropped for inline crypto.
> 
> The create_session() function is now called at initialisation in
> sa_add_rules() which is called from sa_init().
> The return code for add_rules is checked.
> Calls to create_session() in other functions are dropped.
> 
> Add crypto_devid_fill() in ipsec-secgw.c
> Add max_session_size() in ipsec-secgw.c
> Add check_cryptodev_capability() in ipsec.c
> Add check_cryptodev_aead_capability() in ipsec.c
> Add create_sec_session() and create_crypto_session() in ipsec.c
> 
> The crypto_dev_fill() function has been added to find the
> enabled crypto devices.
> 
> The max_session_size() function has been added to calculate memory
> requirements.
> 
> The check_cryptodev_capability() and check_cryptodev_aead_capability()
> functions have been added to check that the SA is supported by the
> crypto device.
> 
> The create_session() function is refactored to use the
> create_sec_session() and create_crypto_session() functions.
> 
> The cryprodev_init() function has been refactored to drop calls to
> rte_mempool_create() and to drop calculation of memory requirements.
> 
> The main() function has been refactored to call crypto_devid_fill()
> and max_session_size() and to call session_pool_init() and
> session_priv_pool_init().
> The ports are started now before adding a flow rule in main().
> The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
> now called after the ports have been started.
> 
> Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
> Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
> Cc: stable@dpdk.org
> 
> Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> ---
>  examples/ipsec-secgw/ipsec-secgw.c   | 271 +++++++++--------
>  examples/ipsec-secgw/ipsec.c         | 569 +++++++++++++++++++----------------
>  examples/ipsec-secgw/ipsec.h         |  10 +-
>  examples/ipsec-secgw/ipsec_process.c |  38 +--
>  examples/ipsec-secgw/sa.c            |  42 ++-
>  5 files changed, 495 insertions(+), 435 deletions(-)
> 
> diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-
> secgw/ipsec-secgw.c
> index ffbd00b..cc8bb57 100644
> --- a/examples/ipsec-secgw/ipsec-secgw.c
> +++ b/examples/ipsec-secgw/ipsec-secgw.c
> @@ -182,6 +182,14 @@ struct lcore_params {
>  	uint8_t lcore_id;
>  } __rte_cache_aligned;
> 
> +/*
> + * Number of enabled crypto devices
> + * This number is needed when checking crypto device capabilities
> + */
> +uint8_t crypto_dev_num;
> +/* array of crypto device ID's */
> +uint8_t crypto_devid[RTE_CRYPTO_MAX_DEVS];
> +
>  static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS];
> 
>  static struct lcore_params *lcore_params;
> @@ -1623,13 +1631,27 @@ check_cryptodev_mask(uint8_t cdev_id)
>  	return -1;
>  }
> 
> +static void
> +crypto_devid_fill(void)
> +{
> +	uint32_t i, n;
> +
> +	n = rte_cryptodev_count();
> +
> +	for (i = 0; i != n; i++) {
> +		if (check_cryptodev_mask(i) == 0)
> +			crypto_devid[crypto_dev_num++] = i;
> +	}
> +}
> +
>  static int32_t
>  cryptodevs_init(void)
>  {
>  	struct rte_cryptodev_config dev_conf;
>  	struct rte_cryptodev_qp_conf qp_conf;
>  	uint16_t idx, max_nb_qps, qp, i;
> -	int16_t cdev_id, port_id;
> +	int16_t cdev_id;
> +	uint32_t dev_max_sess;
>  	struct rte_hash_parameters params = { 0 };
> 
>  	params.entries = CDEV_MAP_ENTRIES;
> @@ -1652,45 +1674,6 @@ cryptodevs_init(void)
> 
>  	printf("lcore/cryptodev/qp mappings:\n");
> 
> -	uint32_t max_sess_sz = 0, sess_sz;
> -	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
> -		void *sec_ctx;
> -
> -		/* Get crypto priv session size */
> -		sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
> -		if (sess_sz > max_sess_sz)
> -			max_sess_sz = sess_sz;
> -
> -		/*
> -		 * If crypto device is security capable, need to check the
> -		 * size of security session as well.
> -		 */
> -
> -		/* Get security context of the crypto device */
> -		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
> -		if (sec_ctx == NULL)
> -			continue;
> -
> -		/* Get size of security session */
> -		sess_sz = rte_security_session_get_size(sec_ctx);
> -		if (sess_sz > max_sess_sz)
> -			max_sess_sz = sess_sz;
> -	}
> -	RTE_ETH_FOREACH_DEV(port_id) {
> -		void *sec_ctx;
> -
> -		if ((enabled_port_mask & (1 << port_id)) == 0)
> -			continue;
> -
> -		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
> -		if (sec_ctx == NULL)
> -			continue;
> -
> -		sess_sz = rte_security_session_get_size(sec_ctx);
> -		if (sess_sz > max_sess_sz)
> -			max_sess_sz = sess_sz;
> -	}
> -
>  	idx = 0;
>  	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
>  		struct rte_cryptodev_info cdev_info;
> @@ -1722,51 +1705,12 @@ cryptodevs_init(void)
>  		dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id);
>  		dev_conf.nb_queue_pairs = qp;
> 
> -		uint32_t dev_max_sess = cdev_info.sym.max_nb_sessions;
> +		dev_max_sess = cdev_info.sym.max_nb_sessions;
>  		if (dev_max_sess != 0 && dev_max_sess < CDEV_MP_NB_OBJS)
>  			rte_exit(EXIT_FAILURE,
>  				"Device does not support at least %u "
>  				"sessions", CDEV_MP_NB_OBJS);
> 
> -		if (!socket_ctx[dev_conf.socket_id].session_pool) {
> -			char mp_name[RTE_MEMPOOL_NAMESIZE];
> -			struct rte_mempool *sess_mp;
> -
> -			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> -					"sess_mp_%u", dev_conf.socket_id);
> -			sess_mp = rte_cryptodev_sym_session_pool_create(
> -					mp_name, CDEV_MP_NB_OBJS,
> -					0, CDEV_MP_CACHE_SZ, 0,
> -					dev_conf.socket_id);
> -			socket_ctx[dev_conf.socket_id].session_pool =
> sess_mp;
> -		}
> -
> -		if (!socket_ctx[dev_conf.socket_id].session_priv_pool) {
> -			char mp_name[RTE_MEMPOOL_NAMESIZE];
> -			struct rte_mempool *sess_mp;
> -
> -			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> -					"sess_mp_priv_%u",
> dev_conf.socket_id);
> -			sess_mp = rte_mempool_create(mp_name,
> -					CDEV_MP_NB_OBJS,
> -					max_sess_sz,
> -					CDEV_MP_CACHE_SZ,
> -					0, NULL, NULL, NULL,
> -					NULL, dev_conf.socket_id,
> -					0);
> -			socket_ctx[dev_conf.socket_id].session_priv_pool =
> -					sess_mp;
> -		}
> -
> -		if (!socket_ctx[dev_conf.socket_id].session_priv_pool ||
> -				!socket_ctx[dev_conf.socket_id].session_pool)
> -			rte_exit(EXIT_FAILURE,
> -				"Cannot create session pool on socket %d\n",
> -				dev_conf.socket_id);
> -		else
> -			printf("Allocated session pool on socket %d\n",
> -					dev_conf.socket_id);
> -
>  		if (rte_cryptodev_configure(cdev_id, &dev_conf))
>  			rte_panic("Failed to initialize cryptodev %u\n",
>  					cdev_id);
> @@ -1787,38 +1731,6 @@ cryptodevs_init(void)
>  					cdev_id);
>  	}
> 
> -	/* create session pools for eth devices that implement security */
> -	RTE_ETH_FOREACH_DEV(port_id) {
> -		if ((enabled_port_mask & (1 << port_id)) &&
> -				rte_eth_dev_get_sec_ctx(port_id)) {
> -			int socket_id = rte_eth_dev_socket_id(port_id);
> -
> -			if (!socket_ctx[socket_id].session_pool) {
> -				char mp_name[RTE_MEMPOOL_NAMESIZE];
> -				struct rte_mempool *sess_mp;
> -
> -				snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> -						"sess_mp_%u", socket_id);
> -				sess_mp = rte_mempool_create(mp_name,
> -						(CDEV_MP_NB_OBJS * 2),
> -						max_sess_sz,
> -						CDEV_MP_CACHE_SZ,
> -						0, NULL, NULL, NULL,
> -						NULL, socket_id,
> -						0);
> -				if (sess_mp == NULL)
> -					rte_exit(EXIT_FAILURE,
> -						"Cannot create session pool "
> -						"on socket %d\n", socket_id);
> -				else
> -					printf("Allocated session pool "
> -						"on socket %d\n", socket_id);
> -				socket_ctx[socket_id].session_pool = sess_mp;
> -			}
> -		}
> -	}
> -
> -
>  	printf("\n");
> 
>  	return 0;
> @@ -1984,6 +1896,98 @@ port_init(uint16_t portid, uint64_t req_rx_offloads,
> uint64_t req_tx_offloads)
>  	printf("\n");
>  }
> 
> +static size_t
> +max_session_size(void)
> +{
> +	size_t max_sz, sz;
> +	void *sec_ctx;
> +	int16_t cdev_id, port_id, n;
> +
> +	max_sz = 0;
> +	n =  rte_cryptodev_count();
> +	for (cdev_id = 0; cdev_id != n; cdev_id++) {
> +		sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
> +		if (sz > max_sz)
> +			max_sz = sz;
> +		/*
> +		 * If crypto device is security capable, need to check the
> +		 * size of security session as well.
> +		 */
> +
> +		/* Get security context of the crypto device */
> +		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
> +		if (sec_ctx == NULL)
> +			continue;
> +
> +		/* Get size of security session */
> +		sz = rte_security_session_get_size(sec_ctx);
> +		if (sz > max_sz)
> +			max_sz = sz;
> +	}
> +
> +	RTE_ETH_FOREACH_DEV(port_id) {
> +		if ((enabled_port_mask & (1 << port_id)) == 0)
> +			continue;
> +
> +		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
> +		if (sec_ctx == NULL)
> +			continue;
> +
> +		sz = rte_security_session_get_size(sec_ctx);
> +		if (sz > max_sz)
> +			max_sz = sz;
> +	}
> +
> +	return max_sz;
> +}
> +
> +static void
> +session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz)
> +{
> +	char mp_name[RTE_MEMPOOL_NAMESIZE];
> +	struct rte_mempool *sess_mp;
> +
> +	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> +			"sess_mp_%u", socket_id);
> +	sess_mp = rte_cryptodev_sym_session_pool_create(
> +			mp_name, CDEV_MP_NB_OBJS,
> +			sess_sz, CDEV_MP_CACHE_SZ, 0,
> +			socket_id);
> +	ctx->session_pool = sess_mp;
> +
> +	if (ctx->session_pool == NULL)
> +		rte_exit(EXIT_FAILURE,
> +			"Cannot init session pool on socket %d\n", socket_id);
> +	else
> +		printf("Allocated session pool on socket %d\n",	socket_id);
> +}
> +
> +static void
> +session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
> +	size_t sess_sz)
> +{
> +	char mp_name[RTE_MEMPOOL_NAMESIZE];
> +	struct rte_mempool *sess_mp;
> +
> +	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> +			"sess_mp_priv_%u", socket_id);
> +	sess_mp = rte_mempool_create(mp_name,
> +			CDEV_MP_NB_OBJS,
> +			sess_sz,
> +			CDEV_MP_CACHE_SZ,
> +			0, NULL, NULL, NULL,
> +			NULL, socket_id,
> +			0);
> +	ctx->session_priv_pool = sess_mp;
> +	if (ctx->session_priv_pool == NULL)
> +		rte_exit(EXIT_FAILURE,
> +			"Cannot init session priv pool on socket %d\n",
> +			socket_id);
> +	else
> +		printf("Allocated session priv pool on socket %d\n",
> +			socket_id);
> +}
> +
>  static void
>  pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf)
>  {
> @@ -2064,9 +2068,11 @@ main(int32_t argc, char **argv)
>  {
>  	int32_t ret;
>  	uint32_t lcore_id;
> +	uint32_t i;
>  	uint8_t socket_id;
>  	uint16_t portid;
>  	uint64_t req_rx_offloads, req_tx_offloads;
> +	size_t sess_sz;
> 
>  	/* init EAL */
>  	ret = rte_eal_init(argc, argv);
> @@ -2094,7 +2100,10 @@ main(int32_t argc, char **argv)
> 
>  	nb_lcores = rte_lcore_count();
> 
> -	/* Replicate each context per socket */
> +	crypto_devid_fill();
> +
> +	sess_sz = max_session_size();
> +
>  	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
>  		if (rte_lcore_is_enabled(lcore_id) == 0)
>  			continue;
> @@ -2104,20 +2113,17 @@ main(int32_t argc, char **argv)
>  		else
>  			socket_id = 0;
> 
> +		/* mbuf_pool is initialised by the pool_init() function*/
>  		if (socket_ctx[socket_id].mbuf_pool)
>  			continue;
> 
> -		/* initilaze SPD */
> -		sp4_init(&socket_ctx[socket_id], socket_id);
> -
> -		sp6_init(&socket_ctx[socket_id], socket_id);
> -
> -		/* initilaze SAD */
> -		sa_init(&socket_ctx[socket_id], socket_id);
> -
> -		rt_init(&socket_ctx[socket_id], socket_id);
> -
>  		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
> +		session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
> +		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
> +			sess_sz);
> +
> +		if (!numa_on)
> +			break;
>  	}
> 
>  	RTE_ETH_FOREACH_DEV(portid) {
> @@ -2135,7 +2141,11 @@ main(int32_t argc, char **argv)
>  		if ((enabled_port_mask & (1 << portid)) == 0)
>  			continue;
> 
> -		/* Start device */
> +		/*
> +		 * Start device
> +		 * note: device must be started before a flow rule
> +		 * can be installed.
> +		 */
>  		ret = rte_eth_dev_start(portid);
>  		if (ret < 0)
>  			rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
> @@ -2153,6 +2163,19 @@ main(int32_t argc, char **argv)
>  			RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback,
> NULL);
>  	}
> 
> +	/* Replicate each context per socket */
> +	for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
> +		socket_id = rte_socket_id_by_idx(i);
> +		if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
> +			(socket_ctx[socket_id].sa_in == NULL) &&
> +			(socket_ctx[socket_id].sa_out == NULL)) {
> +			sa_init(&socket_ctx[socket_id], socket_id);
> +			sp4_init(&socket_ctx[socket_id], socket_id);
> +			sp6_init(&socket_ctx[socket_id], socket_id);
> +			rt_init(&socket_ctx[socket_id], socket_id);
> +		}
> +	}
> +
>  	check_all_ports_link_status(enabled_port_mask);
> 
>  	/* launch per-lcore init on every lcore */
> diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
> index 4352cb8..e31d472 100644
> --- a/examples/ipsec-secgw/ipsec.c
> +++ b/examples/ipsec-secgw/ipsec.c
> @@ -39,42 +39,17 @@ set_ipsec_conf(struct ipsec_sa *sa, struct
> rte_security_ipsec_xform *ipsec)
>  	ipsec->esn_soft_limit = IPSEC_OFFLOAD_ESN_SOFTLIMIT;
>  }
> 
> -int
> -create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
> +static int
> +create_sec_session(struct ipsec_sa *sa, struct rte_mempool *pool)
>  {
> -	struct rte_cryptodev_info cdev_info;
> -	unsigned long cdev_id_qp = 0;
> +	const struct rte_security_capability *sec_cap;
> +	struct rte_security_ctx *ctx;
>  	int32_t ret = 0;
> -	struct cdev_key key = { 0 };
> -
> -	key.lcore_id = (uint8_t)rte_lcore_id();
> -
> -	key.cipher_algo = (uint8_t)sa->cipher_algo;
> -	key.auth_algo = (uint8_t)sa->auth_algo;
> -	key.aead_algo = (uint8_t)sa->aead_algo;
> -
> -	if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {
> -		ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
> -				(void **)&cdev_id_qp);
> -		if (ret < 0) {
> -			RTE_LOG(ERR, IPSEC,
> -				"No cryptodev: core %u, cipher_algo %u, "
> -				"auth_algo %u, aead_algo %u\n",
> -				key.lcore_id,
> -				key.cipher_algo,
> -				key.auth_algo,
> -				key.aead_algo);
> -			return -1;
> -		}
> -	}
> 
> -	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on cryptodev
> "
> -			"%u qp %u\n", sa->spi,
> -			ipsec_ctx->tbl[cdev_id_qp].id,
> -			ipsec_ctx->tbl[cdev_id_qp].qp);
> +	if ((sa == NULL) || (pool == NULL))
> +		return -EINVAL;
> 
> -	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
> -		struct rte_security_session_conf sess_conf = {
> +	struct rte_security_session_conf sess_conf = {
>  			.action_type = sa->type,
>  			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
>  			{.ipsec = {
> @@ -90,247 +65,340 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct
> ipsec_sa *sa)
>  			} },
>  			.crypto_xform = sa->xforms,
>  			.userdata = NULL,
> -
>  		};
> 
> -		if (sa->type ==
> RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> -			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> -
> 	rte_cryptodev_get_sec_ctx(
> -							ipsec_ctx-
> >tbl[cdev_id_qp].id);
> -
> -			/* Set IPsec parameters in conf */
> -			set_ipsec_conf(sa, &(sess_conf.ipsec));
> -
> -			sa->sec_session = rte_security_session_create(ctx,
> -					&sess_conf, ipsec_ctx->session_pool);
> -			if (sa->sec_session == NULL) {
> -				RTE_LOG(ERR, IPSEC,
> -				"SEC Session init failed: err: %d\n", ret);
> -				return -1;
> -			}
> -		} else if (sa->type ==
> RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
> -			struct rte_flow_error err;
> -			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> -
> 	rte_eth_dev_get_sec_ctx(
> -							sa->portid);
> -			const struct rte_security_capability *sec_cap;
> -			int ret = 0;
> -
> -			sa->sec_session = rte_security_session_create(ctx,
> -					&sess_conf, ipsec_ctx->session_pool);
> -			if (sa->sec_session == NULL) {
> -				RTE_LOG(ERR, IPSEC,
> -				"SEC Session init failed: err: %d\n", ret);
> -				return -1;
> -			}
> +	if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> +		ctx = (struct rte_security_ctx *)
> +				rte_eth_dev_get_sec_ctx(sa->portid);
> 
> -			sec_cap = rte_security_capabilities_get(ctx);
> +		/* Set IPsec parameters in conf */
> +		set_ipsec_conf(sa, &(sess_conf.ipsec));
> 
> -			/* iterate until ESP tunnel*/
> -			while (sec_cap->action !=
> -					RTE_SECURITY_ACTION_TYPE_NONE)
> {
> +		sa->sec_session = rte_security_session_create(ctx,
> +				&sess_conf, pool);
> +		if (sa->sec_session == NULL) {
> +			RTE_LOG(ERR, IPSEC,
> +				"SEC Session init failed: err: %d\n",
> +				ret);
> +			return -1;
> +		}
> +	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
> +		struct rte_flow_error err;
> +		ctx = (struct rte_security_ctx *)
> +				rte_eth_dev_get_sec_ctx(sa->portid);
> +		sa->sec_session = rte_security_session_create(ctx,
> +				&sess_conf, pool);
> +		if (sa->sec_session == NULL) {
> +			RTE_LOG(ERR, IPSEC, "SEC Session init failed\n");
> +			return -1;
> +		}
> 
> -				if (sec_cap->action == sa->type &&
> -				    sec_cap->protocol ==
> -					RTE_SECURITY_PROTOCOL_IPSEC &&
> -				    sec_cap->ipsec.mode ==
> -
> 	RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
> -				    sec_cap->ipsec.direction == sa->direction)
> -					break;
> -				sec_cap++;
> -			}
> +		sec_cap = rte_security_capabilities_get(ctx);
> +
> +		/* iterate until ESP tunnel*/
> +		while (sec_cap->action != RTE_SECURITY_ACTION_TYPE_NONE)
> {
> +			if (sec_cap->action == sa->type &&
> +				sec_cap->protocol ==
> +				RTE_SECURITY_PROTOCOL_IPSEC &&
> +				sec_cap->ipsec.mode ==
> +				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
> +				sec_cap->ipsec.direction == sa->direction)
> +				break;
> +			sec_cap++;
> +		}
> 
> -			if (sec_cap->action ==
> RTE_SECURITY_ACTION_TYPE_NONE) {
> -				RTE_LOG(ERR, IPSEC,
> +		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
> +			RTE_LOG(ERR, IPSEC,
>  				"No suitable security capability found\n");
>  				return -1;
> -			}
> +		}
> 
> -			sa->ol_flags = sec_cap->ol_flags;
> -			sa->security_ctx = ctx;
> -			sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
> -
> -			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
> -			sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
> -			if (sa->flags & IP6_TUNNEL) {
> -				sa->pattern[1].spec = &sa->ipv6_spec;
> -				memcpy(sa->ipv6_spec.hdr.dst_addr,
> -					sa->dst.ip.ip6.ip6_b, 16);
> -				memcpy(sa->ipv6_spec.hdr.src_addr,
> -				       sa->src.ip.ip6.ip6_b, 16);
> -			} else {
> -				sa->pattern[1].spec = &sa->ipv4_spec;
> -				sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
> -				sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
> -			}
> +		sa->ol_flags = sec_cap->ol_flags;
> +		sa->security_ctx = ctx;
> +		sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
> +
> +		sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
> +		sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
> +		if (sa->flags & IP6_TUNNEL) {
> +			sa->pattern[1].spec = &sa->ipv6_spec;
> +			memcpy(sa->ipv6_spec.hdr.dst_addr,
> +				sa->dst.ip.ip6.ip6_b, 16);
> +			memcpy(sa->ipv6_spec.hdr.src_addr,
> +			       sa->src.ip.ip6.ip6_b, 16);
> +		} else {
> +			sa->pattern[1].spec = &sa->ipv4_spec;
> +			sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
> +			sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
> +		}
> 
> -			sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
> -			sa->pattern[2].spec = &sa->esp_spec;
> -			sa->pattern[2].mask = &rte_flow_item_esp_mask;
> -			sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
> +		sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
> +		sa->pattern[2].spec = &sa->esp_spec;
> +		sa->pattern[2].mask = &rte_flow_item_esp_mask;
> +		sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
> 
> -			sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
> +		sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
> 
> -			sa->action[0].type =
> RTE_FLOW_ACTION_TYPE_SECURITY;
> -			sa->action[0].conf = sa->sec_session;
> +		sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
> +		sa->action[0].conf = sa->sec_session;
> 
> -			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
> +		sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
> 
> -			sa->attr.egress = (sa->direction ==
> +		sa->attr.egress = (sa->direction ==
> 
> 	RTE_SECURITY_IPSEC_SA_DIR_EGRESS);
> -			sa->attr.ingress = (sa->direction ==
> +		sa->attr.ingress = (sa->direction ==
> 
> 	RTE_SECURITY_IPSEC_SA_DIR_INGRESS);
> -			if (sa->attr.ingress) {
> -				uint8_t rss_key[40];
> -				struct rte_eth_rss_conf rss_conf = {
> -					.rss_key = rss_key,
> -					.rss_key_len = 40,
> -				};
> -				struct rte_eth_dev *eth_dev;
> -				uint16_t
> queue[RTE_MAX_QUEUES_PER_PORT];
> -				struct rte_flow_action_rss action_rss;
> -				unsigned int i;
> -				unsigned int j;
> -
> -				sa->action[2].type =
> RTE_FLOW_ACTION_TYPE_END;
> -				/* Try RSS. */
> -				sa->action[1].type =
> RTE_FLOW_ACTION_TYPE_RSS;
> -				sa->action[1].conf = &action_rss;
> -				eth_dev = ctx->device;
> -				rte_eth_dev_rss_hash_conf_get(sa->portid,
> -							      &rss_conf);
> -				for (i = 0, j = 0;
> -				     i < eth_dev->data->nb_rx_queues; ++i)
> -					if (eth_dev->data->rx_queues[i])
> -						queue[j++] = i;
> +		if (sa->attr.ingress) {
> +			uint8_t rss_key[40];
> +			struct rte_eth_rss_conf rss_conf = {
> +				.rss_key = rss_key,
> +				.rss_key_len = 40,
> +			};
> +			struct rte_eth_dev *eth_dev;
> +			uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
> +			struct rte_flow_action_rss action_rss;
> +			unsigned int i;
> +			unsigned int j;
> +
> +			sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
> +			/* Try RSS. */
> +			sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
> +			sa->action[1].conf = &action_rss;
> +			eth_dev = ctx->device;
> +			rte_eth_dev_rss_hash_conf_get(sa->portid,
> +						&rss_conf);
> +			for (i = 0, j = 0; i < eth_dev->data->nb_rx_queues;
> +				++i)
> +				if (eth_dev->data->rx_queues[i])
> +					queue[j++] = i;
Compilation error

/home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c: In function 'create_sec_session':
/home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c:169:4: error: this 'for' clause does not guard... [-Werror=misleading-indentation]
    for (i = 0, j = 0; i < eth_dev->data->nb_rx_queues;
    ^~~
/home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c:173:5: note: ...this statement, but the latter is misleadingly indented as if it is guarded by the 'for'
     action_rss = (struct rte_flow_action_rss){
     ^~~~~~~~~~

>  				action_rss = (struct rte_flow_action_rss){
>  					.types = rss_conf.rss_hf,
>  					.key_len = rss_conf.rss_key_len,
>  					.queue_num = j,
>  					.key = rss_key,
>  					.queue = queue,
> -				};
> -				ret = rte_flow_validate(sa->portid, &sa->attr,
> -							sa->pattern, sa-
> >action,
> -							&err);
> -				if (!ret)
> -					goto flow_create;
> -				/* Try Queue. */
> -				sa->action[1].type =
> RTE_FLOW_ACTION_TYPE_QUEUE;
> -				sa->action[1].conf =
> -					&(struct rte_flow_action_queue){
> -					.index = 0,
> -				};
> -				ret = rte_flow_validate(sa->portid, &sa->attr,
> -							sa->pattern, sa-
> >action,
> -							&err);
> -				/* Try End. */
> -				sa->action[1].type =
> RTE_FLOW_ACTION_TYPE_END;
> -				sa->action[1].conf = NULL;
> -				ret = rte_flow_validate(sa->portid, &sa->attr,
> +			};
> +			ret = rte_flow_validate(sa->portid, &sa->attr,
> +						sa->pattern, sa->action,
> +						&err);
> +			if (!ret)
> +				goto flow_create;
> +			/* Try Queue. */
> +			sa->action[1].type = RTE_FLOW_ACTION_TYPE_QUEUE;
> +			sa->action[1].conf =
> +				&(struct rte_flow_action_queue){
> +				.index = 0,
> +			};
> +			ret = rte_flow_validate(sa->portid, &sa->attr,
> +						sa->pattern, sa->action,
> +						&err);
> +			/* Try End. */
> +			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
> +			sa->action[1].conf = NULL;
> +			ret = rte_flow_validate(sa->portid, &sa->attr,
>  							sa->pattern, sa-
> >action,
>  							&err);
> -				if (ret)
> -					goto flow_create_failure;
> -			} else if (sa->attr.egress &&
> -				   (sa->ol_flags &
> -				    RTE_SECURITY_TX_HW_TRAILER_OFFLOAD))
> {
> -				sa->action[1].type =
> -					RTE_FLOW_ACTION_TYPE_PASSTHRU;
> -				sa->action[2].type =
> -					RTE_FLOW_ACTION_TYPE_END;
> -			}
> +			if (ret)
> +				goto flow_create_failure;
> +		} else if (sa->attr.egress &&
> +				(sa->ol_flags &
> +				RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
> +			sa->action[1].type =
> +				RTE_FLOW_ACTION_TYPE_PASSTHRU;
> +			sa->action[2].type =
> +				RTE_FLOW_ACTION_TYPE_END;
> +		}
>  flow_create:
> -			sa->flow = rte_flow_create(sa->portid,
> -				&sa->attr, sa->pattern, sa->action, &err);
> -			if (sa->flow == NULL) {
> +		sa->flow = rte_flow_create(sa->portid,
> +			&sa->attr, sa->pattern, sa->action, &err);
> +		if (sa->flow == NULL) {
>  flow_create_failure:
> -				RTE_LOG(ERR, IPSEC,
> -					"Failed to create ipsec flow msg: %s\n",
> -					err.message);
> -				return -1;
> -			}
> -		} else if (sa->type ==
> -
> 	RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
> -			struct rte_security_ctx *ctx =
> -					(struct rte_security_ctx *)
> -					rte_eth_dev_get_sec_ctx(sa->portid);
> -			const struct rte_security_capability *sec_cap;
> -
> -			if (ctx == NULL) {
> -				RTE_LOG(ERR, IPSEC,
> -				"Ethernet device doesn't have security features
> registered\n");
> -				return -1;
> -			}
> +			RTE_LOG(ERR, IPSEC,
> +				"Failed to create ipsec flow msg: %s\n",
> +				err.message);
> +			return -1;
> +		}
> +	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
> {
> +		struct rte_security_ctx *ctx =
> +				(struct rte_security_ctx *)
> +				rte_eth_dev_get_sec_ctx(sa->portid);
> +		const struct rte_security_capability *sec_cap;
> 
> -			/* Set IPsec parameters in conf */
> -			set_ipsec_conf(sa, &(sess_conf.ipsec));
> -
> -			/* Save SA as userdata for the security session. When
> -			 * the packet is received, this userdata will be
> -			 * retrieved using the metadata from the packet.
> -			 *
> -			 * The PMD is expected to set similar metadata for
> other
> -			 * operations, like rte_eth_event, which are tied to
> -			 * security session. In such cases, the userdata could
> -			 * be obtained to uniquely identify the security
> -			 * parameters denoted.
> -			 */
> -
> -			sess_conf.userdata = (void *) sa;
> -
> -			sa->sec_session = rte_security_session_create(ctx,
> -					&sess_conf, ipsec_ctx->session_pool);
> -			if (sa->sec_session == NULL) {
> -				RTE_LOG(ERR, IPSEC,
> -				"SEC Session init failed: err: %d\n", ret);
> -				return -1;
> -			}
> +		if (ctx == NULL) {
> +			RTE_LOG(ERR, IPSEC,
> +			"Ethernet device doesn't have security features
> registered\n");
> +			return -1;
> +		}
> +
> +		/* Set IPsec parameters in conf */
> +		set_ipsec_conf(sa, &(sess_conf.ipsec));
> +
> +		/* Save SA as userdata for the security session. When
> +		 * the packet is received, this userdata will be
> +		 * retrieved using the metadata from the packet.
> +		 *
> +		 * The PMD is expected to set similar metadata for other
> +		 * operations, like rte_eth_event, which are tied to
> +		 * security session. In such cases, the userdata could
> +		 * be obtained to uniquely identify the security
> +		 * parameters denoted.
> +		 */
> +
> +		sess_conf.userdata = (void *) sa;
> +
> +		sa->sec_session = rte_security_session_create(ctx,
> +			&sess_conf, pool);
> +		if (sa->sec_session == NULL) {
> +			RTE_LOG(ERR, IPSEC,
> +			"SEC Session init failed: err: %d\n", ret);
> +			return -1;
> +		}
> 
> -			sec_cap = rte_security_capabilities_get(ctx);
> +		sec_cap = rte_security_capabilities_get(ctx);
> 
> -			if (sec_cap == NULL) {
> -				RTE_LOG(ERR, IPSEC,
> -				"No capabilities registered\n");
> -				return -1;
> -			}
> +		if (sec_cap == NULL) {
> +			RTE_LOG(ERR, IPSEC,
> +			"No capabilities registered\n");
> +			return -1;
> +		}
> 
> -			/* iterate until ESP tunnel*/
> -			while (sec_cap->action !=
> +		/* iterate until ESP tunnel*/
> +		while (sec_cap->action !=
>  					RTE_SECURITY_ACTION_TYPE_NONE)
> {
> 
> -				if (sec_cap->action == sa->type &&
> -				    sec_cap->protocol ==
> -					RTE_SECURITY_PROTOCOL_IPSEC &&
> -				    sec_cap->ipsec.mode ==
> -
> 	RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
> -				    sec_cap->ipsec.direction == sa->direction)
> -					break;
> -				sec_cap++;
> -			}
> +			if (sec_cap->action == sa->type &&
> +				sec_cap->protocol ==
> +				RTE_SECURITY_PROTOCOL_IPSEC &&
> +				sec_cap->ipsec.mode ==
> +				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
> +				sec_cap->ipsec.direction == sa->direction)
> +				break;
> +			sec_cap++;
> +		}
> 
> -			if (sec_cap->action ==
> RTE_SECURITY_ACTION_TYPE_NONE) {
> -				RTE_LOG(ERR, IPSEC,
> -				"No suitable security capability found\n");
> -				return -1;
> -			}
> +		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
> +			RTE_LOG(ERR, IPSEC,
> +			"No suitable security capability found\n");
> +			return -1;
> +		}
> +
> +		sa->ol_flags = sec_cap->ol_flags;
> +		sa->security_ctx = ctx;
> +	} else
> +		return -EINVAL;
> +	return 0;
> +}
> +
> +#define CDEV_IV_SIZE 12
> +
> +static int
> +check_cryptodev_aead_capablity(const struct ipsec_sa *ss, uint8_t dev_id)
> +{
> +	struct rte_cryptodev_sym_capability_idx cap_idx;
> +	const struct rte_cryptodev_symmetric_capability *cap;
> 
> -			sa->ol_flags = sec_cap->ol_flags;
> -			sa->security_ctx = ctx;
> +	cap_idx.type = RTE_CRYPTO_SYM_XFORM_AEAD;
> +	cap_idx.algo.aead = ss->aead_algo;
> +
> +	cap = rte_cryptodev_sym_capability_get(dev_id, &cap_idx);
> +	if (cap == NULL)
> +		return -ENOENT;
> +
> +	return rte_cryptodev_sym_capability_check_aead(cap,
> +					ss->cipher_key_len,
> +					ss->digest_len,
> +					ss->aad_len,
> +					CDEV_IV_SIZE);
> +}
> +
> +static int
> +check_cryptodev_capablity(const struct ipsec_sa *ss, uint8_t dev_id)
> +{
> +	struct rte_cryptodev_sym_capability_idx cap_idx;
> +	const struct rte_cryptodev_symmetric_capability *cap;
> +	uint16_t auth_iv_len;
> +	int rc = -1;
> +
> +	if (ss == NULL)
> +		return rc;
> +
> +	if (ss->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
> +		return check_cryptodev_aead_capablity(ss, dev_id);
> +
> +	auth_iv_len = 0;
> +
> +	cap_idx.type = RTE_CRYPTO_SYM_XFORM_AUTH;
> +	cap_idx.algo.auth = ss->auth_algo;
> +	cap = rte_cryptodev_sym_capability_get(dev_id, &cap_idx);
> +	if (cap != NULL) {
> +		rc = rte_cryptodev_sym_capability_check_auth(
> +				cap, ss->auth_key_len, ss->digest_len,
> +				auth_iv_len);
> +		if (rc == 0) {
> +			cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
> +			cap_idx.algo.cipher = ss->cipher_algo;
> +			cap = rte_cryptodev_sym_capability_get(dev_id,
> +					&cap_idx);
> +			if (cap != NULL)
> +				rc =
> rte_cryptodev_sym_capability_check_cipher(
> +						cap, ss->cipher_key_len,
> +						ss->iv_len);
>  		}
> -	} else {
> -		sa->crypto_session = rte_cryptodev_sym_session_create(
> -				ipsec_ctx->session_pool);
> -		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
> -				sa->crypto_session, sa->xforms,
> -				ipsec_ctx->session_priv_pool);
> -
> -		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
> -				&cdev_info);
>  	}
> -	sa->cdev_id_qp = cdev_id_qp;
> 
> -	return 0;
> +	return rc;
> +}
> +
> +static int
> +create_crypto_session(struct ipsec_sa *sa, struct rte_mempool *pool)
> +{
> +	int32_t rc;
> +	uint32_t devnum, i;
> +	struct rte_cryptodev_sym_session *s;
> +	uint8_t devid[RTE_CRYPTO_MAX_DEVS];
> +
> +	/* check which cryptodevs support SA */
> +	devnum = 0;
> +	for (i = 0; i < crypto_dev_num; i++) {
> +		rc = check_cryptodev_capablity(sa, crypto_devid[i]);
> +		if (rc == 0)
> +			devid[devnum++] = crypto_devid[i];
> +	}
> +
> +	if (devnum == 0)
> +		return -ENODEV;
> +
> +	s = rte_cryptodev_sym_session_create(pool);
> +	if (s == NULL)
> +		return -ENOMEM;
> +
> +	/* initialize SA crypto session for all supported devices */
> +	for (i = 0; i != devnum; i++) {
> +		rc = rte_cryptodev_sym_session_init(devid[i], s, sa->xforms,
> +			pool);
> +		if (rc != 0)
> +			break;
> +	}
> +
> +	if (i == devnum) {
> +		sa->crypto_session = s;
> +		return 0;
> +	}
> +
> +	/* failure, do cleanup */
> +	while (i-- != 0)
> +		rte_cryptodev_sym_session_clear(devid[i], s);
> +
> +	rte_cryptodev_sym_session_free(s);
> +	return rc;
> +}
> +
> +int
> +create_session(struct ipsec_sa *sa, struct rte_mempool *pool)
> +{
> +	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE)
> +		return create_sec_session(sa, pool);
> +	else
> +		return create_crypto_session(sa, pool);
>  }
> 
>  /*
> @@ -393,13 +461,6 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
> ipsec_ctx *ipsec_ctx,
>  			priv->cop.status =
> RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
> 
>  			rte_prefetch0(&priv->sym_cop);
> -
> -			if ((unlikely(sa->sec_session == NULL)) &&
> -					create_session(ipsec_ctx, sa)) {
> -				rte_pktmbuf_free(pkts[i]);
> -				continue;
> -			}
> -
>  			sym_cop = get_sym_cop(&priv->cop);
>  			sym_cop->m_src = pkts[i];
> 
> @@ -412,13 +473,6 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
> ipsec_ctx *ipsec_ctx,
>  			priv->cop.status =
> RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
> 
>  			rte_prefetch0(&priv->sym_cop);
> -
> -			if ((unlikely(sa->crypto_session == NULL)) &&
> -					create_session(ipsec_ctx, sa)) {
> -				rte_pktmbuf_free(pkts[i]);
> -				continue;
> -			}
> -
>  			rte_crypto_op_attach_sym_session(&priv->cop,
>  					sa->crypto_session);
> 
> @@ -429,12 +483,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
> ipsec_ctx *ipsec_ctx,
>  			}
>  			break;
>  		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
> -			if ((unlikely(sa->sec_session == NULL)) &&
> -					create_session(ipsec_ctx, sa)) {
> -				rte_pktmbuf_free(pkts[i]);
> -				continue;
> -			}
> -
> +			RTE_ASSERT(sa->sec_session != NULL);
>  			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
>  			if (sa->ol_flags &
> RTE_SECURITY_TX_OLOAD_NEED_MDATA)
>  				rte_security_set_pkt_metadata(
> @@ -442,17 +491,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
> ipsec_ctx *ipsec_ctx,
>  						sa->sec_session, pkts[i], NULL);
>  			continue;
>  		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
> +			RTE_ASSERT(sa->sec_session != NULL);
>  			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
>  			priv->cop.status =
> RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
> 
>  			rte_prefetch0(&priv->sym_cop);
> -
> -			if ((unlikely(sa->sec_session == NULL)) &&
> -					create_session(ipsec_ctx, sa)) {
> -				rte_pktmbuf_free(pkts[i]);
> -				continue;
> -			}
> -
>  			rte_security_attach_session(&priv->cop,
>  					sa->sec_session);
> 
> diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
> index 99f49d6..804330c 100644
> --- a/examples/ipsec-secgw/ipsec.h
> +++ b/examples/ipsec-secgw/ipsec.h
> @@ -83,6 +83,14 @@ struct app_sa_prm {
> 
>  extern struct app_sa_prm app_sa_prm;
> 
> +/*
> + * Number of enabled crypto devices
> + * This number is needed when checking crypto device capabilities
> + */
> +extern uint8_t crypto_dev_num;
> +/* array of crypto device ID's */
> +extern uint8_t crypto_devid[RTE_CRYPTO_MAX_DEVS];
> +
>  struct ipsec_sa {
>  	struct rte_ipsec_session ips; /* one session per sa for now */
>  	uint32_t spi;
> @@ -306,6 +314,6 @@ void
>  enqueue_cop_burst(struct cdev_qp *cqp);
> 
>  int
> -create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
> +create_session(struct ipsec_sa *sa, struct rte_mempool *pool);
> 
>  #endif /* __IPSEC_H__ */
> diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-
> secgw/ipsec_process.c
> index 3f9cacb..0df6969 100644
> --- a/examples/ipsec-secgw/ipsec_process.c
> +++ b/examples/ipsec-secgw/ipsec_process.c
> @@ -86,39 +86,6 @@ enqueue_cop_bulk(struct cdev_qp *cqp, struct
> rte_crypto_op *cop[], uint32_t num)
>  	cqp->len = len;
>  }
> 
> -static inline int
> -fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
> -	struct ipsec_sa *sa)
> -{
> -	int32_t rc;
> -
> -	/* setup crypto section */
> -	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
> -		if (sa->crypto_session == NULL) {
> -			rc = create_session(ctx, sa);
> -			if (rc != 0)
> -				return rc;
> -		}
> -		ss->crypto.ses = sa->crypto_session;
> -	/* setup session action type */
> -	} else {
> -		if (sa->sec_session == NULL) {
> -			rc = create_session(ctx, sa);
> -			if (rc != 0)
> -				return rc;
> -		}
> -		ss->security.ses = sa->sec_session;
> -		ss->security.ctx = sa->security_ctx;
> -		ss->security.ol_flags = sa->ol_flags;
> -	}
> -
> -	rc = rte_ipsec_session_prepare(ss);
> -	if (rc != 0)
> -		memset(ss, 0, sizeof(*ss));
> -
> -	return rc;
> -}
> -
>  /*
>   * group input packets byt the SA they belong to.
>   */
> @@ -219,9 +186,8 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic
> *trf)
> 
>  		ips = &sa->ips;
> 
> -		/* no valid HW session for that SA, try to create one */
> -		if (sa == NULL || (ips->crypto.ses == NULL &&
> -				fill_ipsec_session(ips, ctx, sa) != 0))
> +		/* no valid HW session for that SA */
> +		if (sa == NULL || ips->crypto.ses == NULL)
>  			k = 0;
> 
>  		/* process packets inline */
> diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
> index a7298a3..0f36f5b 100644
> --- a/examples/ipsec-secgw/sa.c
> +++ b/examples/ipsec-secgw/sa.c
> @@ -774,14 +774,14 @@ check_eth_dev_caps(uint16_t portid, uint32_t
> inbound)
>  	return 0;
>  }
> 
> -
>  static int
>  sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> -		uint32_t nb_entries, uint32_t inbound)
> +	uint32_t nb_entries, uint32_t inbound, struct socket_ctx *skt_ctx)
>  {
>  	struct ipsec_sa *sa;
>  	uint32_t i, idx;
>  	uint16_t iv_length, aad_length;
> +	int32_t rc;
> 
>  	/* for ESN upper 32 bits of SQN also need to be part of AAD */
>  	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
> @@ -902,6 +902,12 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct
> ipsec_sa entries[],
> 
>  			print_one_sa_rule(sa, inbound);
>  		}
> +		rc = create_session(sa, skt_ctx->session_pool);
> +		if (rc != 0) {
> +			RTE_LOG(ERR, IPSEC_ESP,
> +					"create_session() failed\n");
> +			return -EINVAL;
> +		}
>  	}
> 
>  	return 0;
> @@ -909,16 +915,16 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct
> ipsec_sa entries[],
> 
>  static inline int
>  sa_out_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> -		uint32_t nb_entries)
> +		uint32_t nb_entries, struct socket_ctx *skt_ctx)
>  {
> -	return sa_add_rules(sa_ctx, entries, nb_entries, 0);
> +	return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx);
>  }
> 
>  static inline int
>  sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> -		uint32_t nb_entries)
> +		uint32_t nb_entries, struct socket_ctx *skt_ctx)
>  {
> -	return sa_add_rules(sa_ctx, entries, nb_entries, 1);
> +	return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx);
>  }
> 
>  /*
> @@ -1012,10 +1018,12 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm,
> const struct ipsec_sa *ss,
>  	return 0;
>  }
> 
> -static void
> +static int
>  fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
>  	const struct ipsec_sa *lsa)
>  {
> +	int32_t rc = 0;
> +
>  	ss->sa = sa;
>  	ss->type = lsa->type;
> 
> @@ -1028,6 +1036,12 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct
> rte_ipsec_sa *sa,
>  		ss->security.ctx = lsa->security_ctx;
>  		ss->security.ol_flags = lsa->ol_flags;
>  	}
> +
> +	rc = rte_ipsec_session_prepare(ss);
> +	if (rc != 0)
> +		memset(ss, 0, sizeof(*ss));
> +
> +	return rc;
>  }
> 
>  /*
> @@ -1062,8 +1076,8 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa
> *sa, uint32_t sa_size)
>  	if (rc < 0)
>  		return rc;
> 
> -	fill_ipsec_session(&lsa->ips, sa, lsa);
> -	return 0;
> +	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
> +	return rc;
>  }
> 
>  /*
> @@ -1141,7 +1155,10 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
>  				"context %s in socket %d\n", rte_errno,
>  				name, socket_id);
> 
> -		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in);
> +		rc = sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx);
> +		if (rc != 0)
> +			rte_exit(EXIT_FAILURE,
> +				"failed to add inbound rules\n");
> 
>  		if (app_sa_prm.enable != 0) {
>  			rc = ipsec_satbl_init(ctx->sa_in, sa_in, nb_sa_in,
> @@ -1161,7 +1178,10 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
>  				"context %s in socket %d\n", rte_errno,
>  				name, socket_id);
> 
> -		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out);
> +		rc = sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx);
> +		if (rc != 0)
> +			rte_exit(EXIT_FAILURE,
> +				"failed to add outbound rules\n");
> 
>  		if (app_sa_prm.enable != 0) {
>  			rc = ipsec_satbl_init(ctx->sa_out, sa_out, nb_sa_out,
> --
> 2.7.4

Re: [dpdk-dev] [PATCH v3 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Iremonger, Bernard]

Hi Akhil,

> -----Original Message-----
> From: Akhil Goyal [mailto:akhil.goyal@nxp.com]
> Sent: Wednesday, April 17, 2019 12:52 PM
> To: Iremonger, Bernard <bernard.iremonger@intel.com>; dev@dpdk.org;
> Ananyev, Konstantin <konstantin.ananyev@intel.com>
> Cc: stable@dpdk.org
> Subject: RE: [PATCH v3 1/2] examples/ipsec-secgw: fix 1st packet dropped
> for inline crypto
> 
> >
> > Inline crypto installs a flow rule in the NIC. This flow rule must be
> > installed before the first inbound packet is received.
> >
> > The create_session() function installs the flow rule.
> >
> > Refactor ipsec-secgw.c, sa.c, ipsec.h and ipsec.c to create sessions
> > at startup to fix the issue of the first packet being dropped for
> > inline crypto.
> >
> > The create_session() function is now called at initialisation in
> > sa_add_rules() which is called from sa_init().
> > The return code for add_rules is checked.
> > Calls to create_session() in other functions are dropped.
> >
> > Add crypto_devid_fill() in ipsec-secgw.c Add max_session_size() in
> > ipsec-secgw.c Add check_cryptodev_capability() in ipsec.c Add
> > check_cryptodev_aead_capability() in ipsec.c Add create_sec_session()
> > and create_crypto_session() in ipsec.c
> >
> > The crypto_dev_fill() function has been added to find the enabled
> > crypto devices.
> >
> > The max_session_size() function has been added to calculate memory
> > requirements.
> >
> > The check_cryptodev_capability() and check_cryptodev_aead_capability()
> > functions have been added to check that the SA is supported by the
> > crypto device.
> >
> > The create_session() function is refactored to use the
> > create_sec_session() and create_crypto_session() functions.
> >
> > The cryprodev_init() function has been refactored to drop calls to
> > rte_mempool_create() and to drop calculation of memory requirements.
> >
> > The main() function has been refactored to call crypto_devid_fill()
> > and max_session_size() and to call session_pool_init() and
> > session_priv_pool_init().
> > The ports are started now before adding a flow rule in main().
> > The sa_init(), sp4_init(), sp6_init() and rt_init() functions are now
> > called after the ports have been started.
> >
> > Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
> > Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample
> > application")
> > Cc: stable@dpdk.org
> >
> > Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> > ---
> >  examples/ipsec-secgw/ipsec-secgw.c   | 271 +++++++++--------
> >  examples/ipsec-secgw/ipsec.c         | 569 +++++++++++++++++++-----------
> -----
> >  examples/ipsec-secgw/ipsec.h         |  10 +-
> >  examples/ipsec-secgw/ipsec_process.c |  38 +--
> >  examples/ipsec-secgw/sa.c            |  42 ++-
> >  5 files changed, 495 insertions(+), 435 deletions(-)
> >
> > diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-
> > secgw/ipsec-secgw.c index ffbd00b..cc8bb57 100644
> > --- a/examples/ipsec-secgw/ipsec-secgw.c
> > +++ b/examples/ipsec-secgw/ipsec-secgw.c
> > @@ -182,6 +182,14 @@ struct lcore_params {
> >  	uint8_t lcore_id;
> >  } __rte_cache_aligned;
> >
> > +/*
> > + * Number of enabled crypto devices
> > + * This number is needed when checking crypto device capabilities  */
> > +uint8_t crypto_dev_num;
> > +/* array of crypto device ID's */
> > +uint8_t crypto_devid[RTE_CRYPTO_MAX_DEVS];
> > +
> >  static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS];
> >
> >  static struct lcore_params *lcore_params; @@ -1623,13 +1631,27 @@
> > check_cryptodev_mask(uint8_t cdev_id)
> >  	return -1;
> >  }
> >
> > +static void
> > +crypto_devid_fill(void)
> > +{
> > +	uint32_t i, n;
> > +
> > +	n = rte_cryptodev_count();
> > +
> > +	for (i = 0; i != n; i++) {
> > +		if (check_cryptodev_mask(i) == 0)
> > +			crypto_devid[crypto_dev_num++] = i;
> > +	}
> > +}
> > +
> >  static int32_t
> >  cryptodevs_init(void)
> >  {
> >  	struct rte_cryptodev_config dev_conf;
> >  	struct rte_cryptodev_qp_conf qp_conf;
> >  	uint16_t idx, max_nb_qps, qp, i;
> > -	int16_t cdev_id, port_id;
> > +	int16_t cdev_id;
> > +	uint32_t dev_max_sess;
> >  	struct rte_hash_parameters params = { 0 };
> >
> >  	params.entries = CDEV_MAP_ENTRIES;
> > @@ -1652,45 +1674,6 @@ cryptodevs_init(void)
> >
> >  	printf("lcore/cryptodev/qp mappings:\n");
> >
> > -	uint32_t max_sess_sz = 0, sess_sz;
> > -	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
> > -		void *sec_ctx;
> > -
> > -		/* Get crypto priv session size */
> > -		sess_sz =
> rte_cryptodev_sym_get_private_session_size(cdev_id);
> > -		if (sess_sz > max_sess_sz)
> > -			max_sess_sz = sess_sz;
> > -
> > -		/*
> > -		 * If crypto device is security capable, need to check the
> > -		 * size of security session as well.
> > -		 */
> > -
> > -		/* Get security context of the crypto device */
> > -		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
> > -		if (sec_ctx == NULL)
> > -			continue;
> > -
> > -		/* Get size of security session */
> > -		sess_sz = rte_security_session_get_size(sec_ctx);
> > -		if (sess_sz > max_sess_sz)
> > -			max_sess_sz = sess_sz;
> > -	}
> > -	RTE_ETH_FOREACH_DEV(port_id) {
> > -		void *sec_ctx;
> > -
> > -		if ((enabled_port_mask & (1 << port_id)) == 0)
> > -			continue;
> > -
> > -		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
> > -		if (sec_ctx == NULL)
> > -			continue;
> > -
> > -		sess_sz = rte_security_session_get_size(sec_ctx);
> > -		if (sess_sz > max_sess_sz)
> > -			max_sess_sz = sess_sz;
> > -	}
> > -
> >  	idx = 0;
> >  	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
> >  		struct rte_cryptodev_info cdev_info; @@ -1722,51 +1705,12
> @@
> > cryptodevs_init(void)
> >  		dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id);
> >  		dev_conf.nb_queue_pairs = qp;
> >
> > -		uint32_t dev_max_sess = cdev_info.sym.max_nb_sessions;
> > +		dev_max_sess = cdev_info.sym.max_nb_sessions;
> >  		if (dev_max_sess != 0 && dev_max_sess <
> CDEV_MP_NB_OBJS)
> >  			rte_exit(EXIT_FAILURE,
> >  				"Device does not support at least %u "
> >  				"sessions", CDEV_MP_NB_OBJS);
> >
> > -		if (!socket_ctx[dev_conf.socket_id].session_pool) {
> > -			char mp_name[RTE_MEMPOOL_NAMESIZE];
> > -			struct rte_mempool *sess_mp;
> > -
> > -			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> > -					"sess_mp_%u", dev_conf.socket_id);
> > -			sess_mp =
> rte_cryptodev_sym_session_pool_create(
> > -					mp_name, CDEV_MP_NB_OBJS,
> > -					0, CDEV_MP_CACHE_SZ, 0,
> > -					dev_conf.socket_id);
> > -			socket_ctx[dev_conf.socket_id].session_pool =
> > sess_mp;
> > -		}
> > -
> > -		if (!socket_ctx[dev_conf.socket_id].session_priv_pool) {
> > -			char mp_name[RTE_MEMPOOL_NAMESIZE];
> > -			struct rte_mempool *sess_mp;
> > -
> > -			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> > -					"sess_mp_priv_%u",
> > dev_conf.socket_id);
> > -			sess_mp = rte_mempool_create(mp_name,
> > -					CDEV_MP_NB_OBJS,
> > -					max_sess_sz,
> > -					CDEV_MP_CACHE_SZ,
> > -					0, NULL, NULL, NULL,
> > -					NULL, dev_conf.socket_id,
> > -					0);
> > -			socket_ctx[dev_conf.socket_id].session_priv_pool =
> > -					sess_mp;
> > -		}
> > -
> > -		if (!socket_ctx[dev_conf.socket_id].session_priv_pool ||
> > -
> 	!socket_ctx[dev_conf.socket_id].session_pool)
> > -			rte_exit(EXIT_FAILURE,
> > -				"Cannot create session pool on socket %d\n",
> > -				dev_conf.socket_id);
> > -		else
> > -			printf("Allocated session pool on socket %d\n",
> > -					dev_conf.socket_id);
> > -
> >  		if (rte_cryptodev_configure(cdev_id, &dev_conf))
> >  			rte_panic("Failed to initialize cryptodev %u\n",
> >  					cdev_id);
> > @@ -1787,38 +1731,6 @@ cryptodevs_init(void)
> >  					cdev_id);
> >  	}
> >
> > -	/* create session pools for eth devices that implement security */
> > -	RTE_ETH_FOREACH_DEV(port_id) {
> > -		if ((enabled_port_mask & (1 << port_id)) &&
> > -				rte_eth_dev_get_sec_ctx(port_id)) {
> > -			int socket_id = rte_eth_dev_socket_id(port_id);
> > -
> > -			if (!socket_ctx[socket_id].session_pool) {
> > -				char mp_name[RTE_MEMPOOL_NAMESIZE];
> > -				struct rte_mempool *sess_mp;
> > -
> > -				snprintf(mp_name,
> RTE_MEMPOOL_NAMESIZE,
> > -						"sess_mp_%u", socket_id);
> > -				sess_mp = rte_mempool_create(mp_name,
> > -						(CDEV_MP_NB_OBJS * 2),
> > -						max_sess_sz,
> > -						CDEV_MP_CACHE_SZ,
> > -						0, NULL, NULL, NULL,
> > -						NULL, socket_id,
> > -						0);
> > -				if (sess_mp == NULL)
> > -					rte_exit(EXIT_FAILURE,
> > -						"Cannot create session pool "
> > -						"on socket %d\n", socket_id);
> > -				else
> > -					printf("Allocated session pool "
> > -						"on socket %d\n", socket_id);
> > -				socket_ctx[socket_id].session_pool =
> sess_mp;
> > -			}
> > -		}
> > -	}
> > -
> > -
> >  	printf("\n");
> >
> >  	return 0;
> > @@ -1984,6 +1896,98 @@ port_init(uint16_t portid, uint64_t
> > req_rx_offloads, uint64_t req_tx_offloads)
> >  	printf("\n");
> >  }
> >
> > +static size_t
> > +max_session_size(void)
> > +{
> > +	size_t max_sz, sz;
> > +	void *sec_ctx;
> > +	int16_t cdev_id, port_id, n;
> > +
> > +	max_sz = 0;
> > +	n =  rte_cryptodev_count();
> > +	for (cdev_id = 0; cdev_id != n; cdev_id++) {
> > +		sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
> > +		if (sz > max_sz)
> > +			max_sz = sz;
> > +		/*
> > +		 * If crypto device is security capable, need to check the
> > +		 * size of security session as well.
> > +		 */
> > +
> > +		/* Get security context of the crypto device */
> > +		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
> > +		if (sec_ctx == NULL)
> > +			continue;
> > +
> > +		/* Get size of security session */
> > +		sz = rte_security_session_get_size(sec_ctx);
> > +		if (sz > max_sz)
> > +			max_sz = sz;
> > +	}
> > +
> > +	RTE_ETH_FOREACH_DEV(port_id) {
> > +		if ((enabled_port_mask & (1 << port_id)) == 0)
> > +			continue;
> > +
> > +		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
> > +		if (sec_ctx == NULL)
> > +			continue;
> > +
> > +		sz = rte_security_session_get_size(sec_ctx);
> > +		if (sz > max_sz)
> > +			max_sz = sz;
> > +	}
> > +
> > +	return max_sz;
> > +}
> > +
> > +static void
> > +session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t
> > +sess_sz) {
> > +	char mp_name[RTE_MEMPOOL_NAMESIZE];
> > +	struct rte_mempool *sess_mp;
> > +
> > +	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> > +			"sess_mp_%u", socket_id);
> > +	sess_mp = rte_cryptodev_sym_session_pool_create(
> > +			mp_name, CDEV_MP_NB_OBJS,
> > +			sess_sz, CDEV_MP_CACHE_SZ, 0,
> > +			socket_id);
> > +	ctx->session_pool = sess_mp;
> > +
> > +	if (ctx->session_pool == NULL)
> > +		rte_exit(EXIT_FAILURE,
> > +			"Cannot init session pool on socket %d\n",
> socket_id);
> > +	else
> > +		printf("Allocated session pool on socket %d\n",
> 	socket_id);
> > +}
> > +
> > +static void
> > +session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
> > +	size_t sess_sz)
> > +{
> > +	char mp_name[RTE_MEMPOOL_NAMESIZE];
> > +	struct rte_mempool *sess_mp;
> > +
> > +	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> > +			"sess_mp_priv_%u", socket_id);
> > +	sess_mp = rte_mempool_create(mp_name,
> > +			CDEV_MP_NB_OBJS,
> > +			sess_sz,
> > +			CDEV_MP_CACHE_SZ,
> > +			0, NULL, NULL, NULL,
> > +			NULL, socket_id,
> > +			0);
> > +	ctx->session_priv_pool = sess_mp;
> > +	if (ctx->session_priv_pool == NULL)
> > +		rte_exit(EXIT_FAILURE,
> > +			"Cannot init session priv pool on socket %d\n",
> > +			socket_id);
> > +	else
> > +		printf("Allocated session priv pool on socket %d\n",
> > +			socket_id);
> > +}
> > +
> >  static void
> >  pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t
> > nb_mbuf)  { @@ -2064,9 +2068,11 @@ main(int32_t argc, char **argv)  {
> >  	int32_t ret;
> >  	uint32_t lcore_id;
> > +	uint32_t i;
> >  	uint8_t socket_id;
> >  	uint16_t portid;
> >  	uint64_t req_rx_offloads, req_tx_offloads;
> > +	size_t sess_sz;
> >
> >  	/* init EAL */
> >  	ret = rte_eal_init(argc, argv);
> > @@ -2094,7 +2100,10 @@ main(int32_t argc, char **argv)
> >
> >  	nb_lcores = rte_lcore_count();
> >
> > -	/* Replicate each context per socket */
> > +	crypto_devid_fill();
> > +
> > +	sess_sz = max_session_size();
> > +
> >  	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
> >  		if (rte_lcore_is_enabled(lcore_id) == 0)
> >  			continue;
> > @@ -2104,20 +2113,17 @@ main(int32_t argc, char **argv)
> >  		else
> >  			socket_id = 0;
> >
> > +		/* mbuf_pool is initialised by the pool_init() function*/
> >  		if (socket_ctx[socket_id].mbuf_pool)
> >  			continue;
> >
> > -		/* initilaze SPD */
> > -		sp4_init(&socket_ctx[socket_id], socket_id);
> > -
> > -		sp6_init(&socket_ctx[socket_id], socket_id);
> > -
> > -		/* initilaze SAD */
> > -		sa_init(&socket_ctx[socket_id], socket_id);
> > -
> > -		rt_init(&socket_ctx[socket_id], socket_id);
> > -
> >  		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
> > +		session_pool_init(&socket_ctx[socket_id], socket_id,
> sess_sz);
> > +		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
> > +			sess_sz);
> > +
> > +		if (!numa_on)
> > +			break;
> >  	}
> >
> >  	RTE_ETH_FOREACH_DEV(portid) {
> > @@ -2135,7 +2141,11 @@ main(int32_t argc, char **argv)
> >  		if ((enabled_port_mask & (1 << portid)) == 0)
> >  			continue;
> >
> > -		/* Start device */
> > +		/*
> > +		 * Start device
> > +		 * note: device must be started before a flow rule
> > +		 * can be installed.
> > +		 */
> >  		ret = rte_eth_dev_start(portid);
> >  		if (ret < 0)
> >  			rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
> > @@ -2153,6 +2163,19 @@ main(int32_t argc, char **argv)
> >  			RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback,
> NULL);
> >  	}
> >
> > +	/* Replicate each context per socket */
> > +	for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
> > +		socket_id = rte_socket_id_by_idx(i);
> > +		if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
> > +			(socket_ctx[socket_id].sa_in == NULL) &&
> > +			(socket_ctx[socket_id].sa_out == NULL)) {
> > +			sa_init(&socket_ctx[socket_id], socket_id);
> > +			sp4_init(&socket_ctx[socket_id], socket_id);
> > +			sp6_init(&socket_ctx[socket_id], socket_id);
> > +			rt_init(&socket_ctx[socket_id], socket_id);
> > +		}
> > +	}
> > +
> >  	check_all_ports_link_status(enabled_port_mask);
> >
> >  	/* launch per-lcore init on every lcore */ diff --git
> > a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c index
> > 4352cb8..e31d472 100644
> > --- a/examples/ipsec-secgw/ipsec.c
> > +++ b/examples/ipsec-secgw/ipsec.c
> > @@ -39,42 +39,17 @@ set_ipsec_conf(struct ipsec_sa *sa, struct
> > rte_security_ipsec_xform *ipsec)
> >  	ipsec->esn_soft_limit = IPSEC_OFFLOAD_ESN_SOFTLIMIT;  }
> >
> > -int
> > -create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
> > +static int
> > +create_sec_session(struct ipsec_sa *sa, struct rte_mempool *pool)
> >  {
> > -	struct rte_cryptodev_info cdev_info;
> > -	unsigned long cdev_id_qp = 0;
> > +	const struct rte_security_capability *sec_cap;
> > +	struct rte_security_ctx *ctx;
> >  	int32_t ret = 0;
> > -	struct cdev_key key = { 0 };
> > -
> > -	key.lcore_id = (uint8_t)rte_lcore_id();
> > -
> > -	key.cipher_algo = (uint8_t)sa->cipher_algo;
> > -	key.auth_algo = (uint8_t)sa->auth_algo;
> > -	key.aead_algo = (uint8_t)sa->aead_algo;
> > -
> > -	if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {
> > -		ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
> > -				(void **)&cdev_id_qp);
> > -		if (ret < 0) {
> > -			RTE_LOG(ERR, IPSEC,
> > -				"No cryptodev: core %u, cipher_algo %u, "
> > -				"auth_algo %u, aead_algo %u\n",
> > -				key.lcore_id,
> > -				key.cipher_algo,
> > -				key.auth_algo,
> > -				key.aead_algo);
> > -			return -1;
> > -		}
> > -	}
> >
> > -	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on
> cryptodev
> > "
> > -			"%u qp %u\n", sa->spi,
> > -			ipsec_ctx->tbl[cdev_id_qp].id,
> > -			ipsec_ctx->tbl[cdev_id_qp].qp);
> > +	if ((sa == NULL) || (pool == NULL))
> > +		return -EINVAL;
> >
> > -	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
> > -		struct rte_security_session_conf sess_conf = {
> > +	struct rte_security_session_conf sess_conf = {
> >  			.action_type = sa->type,
> >  			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
> >  			{.ipsec = {
> > @@ -90,247 +65,340 @@ create_session(struct ipsec_ctx *ipsec_ctx,
> > struct ipsec_sa *sa)
> >  			} },
> >  			.crypto_xform = sa->xforms,
> >  			.userdata = NULL,
> > -
> >  		};
> >
> > -		if (sa->type ==
> > RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > -			struct rte_security_ctx *ctx = (struct rte_security_ctx
> *)
> > -
> > 	rte_cryptodev_get_sec_ctx(
> > -							ipsec_ctx-
> > >tbl[cdev_id_qp].id);
> > -
> > -			/* Set IPsec parameters in conf */
> > -			set_ipsec_conf(sa, &(sess_conf.ipsec));
> > -
> > -			sa->sec_session = rte_security_session_create(ctx,
> > -					&sess_conf, ipsec_ctx-
> >session_pool);
> > -			if (sa->sec_session == NULL) {
> > -				RTE_LOG(ERR, IPSEC,
> > -				"SEC Session init failed: err: %d\n", ret);
> > -				return -1;
> > -			}
> > -		} else if (sa->type ==
> > RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
> > -			struct rte_flow_error err;
> > -			struct rte_security_ctx *ctx = (struct rte_security_ctx
> *)
> > -
> > 	rte_eth_dev_get_sec_ctx(
> > -							sa->portid);
> > -			const struct rte_security_capability *sec_cap;
> > -			int ret = 0;
> > -
> > -			sa->sec_session = rte_security_session_create(ctx,
> > -					&sess_conf, ipsec_ctx-
> >session_pool);
> > -			if (sa->sec_session == NULL) {
> > -				RTE_LOG(ERR, IPSEC,
> > -				"SEC Session init failed: err: %d\n", ret);
> > -				return -1;
> > -			}
> > +	if (sa->type ==
> RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > +		ctx = (struct rte_security_ctx *)
> > +				rte_eth_dev_get_sec_ctx(sa->portid);
> >
> > -			sec_cap = rte_security_capabilities_get(ctx);
> > +		/* Set IPsec parameters in conf */
> > +		set_ipsec_conf(sa, &(sess_conf.ipsec));
> >
> > -			/* iterate until ESP tunnel*/
> > -			while (sec_cap->action !=
> > -
> 	RTE_SECURITY_ACTION_TYPE_NONE)
> > {
> > +		sa->sec_session = rte_security_session_create(ctx,
> > +				&sess_conf, pool);
> > +		if (sa->sec_session == NULL) {
> > +			RTE_LOG(ERR, IPSEC,
> > +				"SEC Session init failed: err: %d\n",
> > +				ret);
> > +			return -1;
> > +		}
> > +	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
> {
> > +		struct rte_flow_error err;
> > +		ctx = (struct rte_security_ctx *)
> > +				rte_eth_dev_get_sec_ctx(sa->portid);
> > +		sa->sec_session = rte_security_session_create(ctx,
> > +				&sess_conf, pool);
> > +		if (sa->sec_session == NULL) {
> > +			RTE_LOG(ERR, IPSEC, "SEC Session init failed\n");
> > +			return -1;
> > +		}
> >
> > -				if (sec_cap->action == sa->type &&
> > -				    sec_cap->protocol ==
> > -					RTE_SECURITY_PROTOCOL_IPSEC &&
> > -				    sec_cap->ipsec.mode ==
> > -
> > 	RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
> > -				    sec_cap->ipsec.direction == sa->direction)
> > -					break;
> > -				sec_cap++;
> > -			}
> > +		sec_cap = rte_security_capabilities_get(ctx);
> > +
> > +		/* iterate until ESP tunnel*/
> > +		while (sec_cap->action !=
> RTE_SECURITY_ACTION_TYPE_NONE)
> > {
> > +			if (sec_cap->action == sa->type &&
> > +				sec_cap->protocol ==
> > +				RTE_SECURITY_PROTOCOL_IPSEC &&
> > +				sec_cap->ipsec.mode ==
> > +				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL
> &&
> > +				sec_cap->ipsec.direction == sa->direction)
> > +				break;
> > +			sec_cap++;
> > +		}
> >
> > -			if (sec_cap->action ==
> > RTE_SECURITY_ACTION_TYPE_NONE) {
> > -				RTE_LOG(ERR, IPSEC,
> > +		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE)
> {
> > +			RTE_LOG(ERR, IPSEC,
> >  				"No suitable security capability found\n");
> >  				return -1;
> > -			}
> > +		}
> >
> > -			sa->ol_flags = sec_cap->ol_flags;
> > -			sa->security_ctx = ctx;
> > -			sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
> > -
> > -			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
> > -			sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
> > -			if (sa->flags & IP6_TUNNEL) {
> > -				sa->pattern[1].spec = &sa->ipv6_spec;
> > -				memcpy(sa->ipv6_spec.hdr.dst_addr,
> > -					sa->dst.ip.ip6.ip6_b, 16);
> > -				memcpy(sa->ipv6_spec.hdr.src_addr,
> > -				       sa->src.ip.ip6.ip6_b, 16);
> > -			} else {
> > -				sa->pattern[1].spec = &sa->ipv4_spec;
> > -				sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
> > -				sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
> > -			}
> > +		sa->ol_flags = sec_cap->ol_flags;
> > +		sa->security_ctx = ctx;
> > +		sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
> > +
> > +		sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
> > +		sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
> > +		if (sa->flags & IP6_TUNNEL) {
> > +			sa->pattern[1].spec = &sa->ipv6_spec;
> > +			memcpy(sa->ipv6_spec.hdr.dst_addr,
> > +				sa->dst.ip.ip6.ip6_b, 16);
> > +			memcpy(sa->ipv6_spec.hdr.src_addr,
> > +			       sa->src.ip.ip6.ip6_b, 16);
> > +		} else {
> > +			sa->pattern[1].spec = &sa->ipv4_spec;
> > +			sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
> > +			sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
> > +		}
> >
> > -			sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
> > -			sa->pattern[2].spec = &sa->esp_spec;
> > -			sa->pattern[2].mask = &rte_flow_item_esp_mask;
> > -			sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
> > +		sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
> > +		sa->pattern[2].spec = &sa->esp_spec;
> > +		sa->pattern[2].mask = &rte_flow_item_esp_mask;
> > +		sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
> >
> > -			sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
> > +		sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
> >
> > -			sa->action[0].type =
> > RTE_FLOW_ACTION_TYPE_SECURITY;
> > -			sa->action[0].conf = sa->sec_session;
> > +		sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
> > +		sa->action[0].conf = sa->sec_session;
> >
> > -			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
> > +		sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
> >
> > -			sa->attr.egress = (sa->direction ==
> > +		sa->attr.egress = (sa->direction ==
> >
> > 	RTE_SECURITY_IPSEC_SA_DIR_EGRESS);
> > -			sa->attr.ingress = (sa->direction ==
> > +		sa->attr.ingress = (sa->direction ==
> >
> > 	RTE_SECURITY_IPSEC_SA_DIR_INGRESS);
> > -			if (sa->attr.ingress) {
> > -				uint8_t rss_key[40];
> > -				struct rte_eth_rss_conf rss_conf = {
> > -					.rss_key = rss_key,
> > -					.rss_key_len = 40,
> > -				};
> > -				struct rte_eth_dev *eth_dev;
> > -				uint16_t
> > queue[RTE_MAX_QUEUES_PER_PORT];
> > -				struct rte_flow_action_rss action_rss;
> > -				unsigned int i;
> > -				unsigned int j;
> > -
> > -				sa->action[2].type =
> > RTE_FLOW_ACTION_TYPE_END;
> > -				/* Try RSS. */
> > -				sa->action[1].type =
> > RTE_FLOW_ACTION_TYPE_RSS;
> > -				sa->action[1].conf = &action_rss;
> > -				eth_dev = ctx->device;
> > -				rte_eth_dev_rss_hash_conf_get(sa->portid,
> > -							      &rss_conf);
> > -				for (i = 0, j = 0;
> > -				     i < eth_dev->data->nb_rx_queues; ++i)
> > -					if (eth_dev->data->rx_queues[i])
> > -						queue[j++] = i;
> > +		if (sa->attr.ingress) {
> > +			uint8_t rss_key[40];
> > +			struct rte_eth_rss_conf rss_conf = {
> > +				.rss_key = rss_key,
> > +				.rss_key_len = 40,
> > +			};
> > +			struct rte_eth_dev *eth_dev;
> > +			uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
> > +			struct rte_flow_action_rss action_rss;
> > +			unsigned int i;
> > +			unsigned int j;
> > +
> > +			sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
> > +			/* Try RSS. */
> > +			sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
> > +			sa->action[1].conf = &action_rss;
> > +			eth_dev = ctx->device;
> > +			rte_eth_dev_rss_hash_conf_get(sa->portid,
> > +						&rss_conf);
> > +			for (i = 0, j = 0; i < eth_dev->data->nb_rx_queues;
> > +				++i)
> > +				if (eth_dev->data->rx_queues[i])
> > +					queue[j++] = i;
> Compilation error
> 
> /home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c: In
> function 'create_sec_session':
> /home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c:169:4:
> error: this 'for' clause does not guard... [-Werror=misleading-indentation]
>     for (i = 0, j = 0; i < eth_dev->data->nb_rx_queues;
>     ^~~
> /home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c:173:5:
> note: ...this statement, but the latter is misleadingly indented as if it is
> guarded by the 'for'
>      action_rss = (struct rte_flow_action_rss){
>      ^~~~~~~~~~
> 
<snip>

I will send a v4.
What compiler are you using?

Regards,

Bernard.

Re: [dpdk-dev] [PATCH v3 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Iremonger, Bernard]

Hi Akhil,

> -----Original Message-----
> From: Akhil Goyal [mailto:akhil.goyal@nxp.com]
> Sent: Wednesday, April 17, 2019 12:52 PM
> To: Iremonger, Bernard <bernard.iremonger@intel.com>; dev@dpdk.org;
> Ananyev, Konstantin <konstantin.ananyev@intel.com>
> Cc: stable@dpdk.org
> Subject: RE: [PATCH v3 1/2] examples/ipsec-secgw: fix 1st packet dropped
> for inline crypto
> 
> >
> > Inline crypto installs a flow rule in the NIC. This flow rule must be
> > installed before the first inbound packet is received.
> >
> > The create_session() function installs the flow rule.
> >
> > Refactor ipsec-secgw.c, sa.c, ipsec.h and ipsec.c to create sessions
> > at startup to fix the issue of the first packet being dropped for
> > inline crypto.
> >
> > The create_session() function is now called at initialisation in
> > sa_add_rules() which is called from sa_init().
> > The return code for add_rules is checked.
> > Calls to create_session() in other functions are dropped.
> >
> > Add crypto_devid_fill() in ipsec-secgw.c Add max_session_size() in
> > ipsec-secgw.c Add check_cryptodev_capability() in ipsec.c Add
> > check_cryptodev_aead_capability() in ipsec.c Add create_sec_session()
> > and create_crypto_session() in ipsec.c
> >
> > The crypto_dev_fill() function has been added to find the enabled
> > crypto devices.
> >
> > The max_session_size() function has been added to calculate memory
> > requirements.
> >
> > The check_cryptodev_capability() and check_cryptodev_aead_capability()
> > functions have been added to check that the SA is supported by the
> > crypto device.
> >
> > The create_session() function is refactored to use the
> > create_sec_session() and create_crypto_session() functions.
> >
> > The cryprodev_init() function has been refactored to drop calls to
> > rte_mempool_create() and to drop calculation of memory requirements.
> >
> > The main() function has been refactored to call crypto_devid_fill()
> > and max_session_size() and to call session_pool_init() and
> > session_priv_pool_init().
> > The ports are started now before adding a flow rule in main().
> > The sa_init(), sp4_init(), sp6_init() and rt_init() functions are now
> > called after the ports have been started.
> >
> > Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
> > Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample
> > application")
> > Cc: stable@dpdk.org
> >
> > Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> > ---
> >  examples/ipsec-secgw/ipsec-secgw.c   | 271 +++++++++--------
> >  examples/ipsec-secgw/ipsec.c         | 569 +++++++++++++++++++-----------
> -----
> >  examples/ipsec-secgw/ipsec.h         |  10 +-
> >  examples/ipsec-secgw/ipsec_process.c |  38 +--
> >  examples/ipsec-secgw/sa.c            |  42 ++-
> >  5 files changed, 495 insertions(+), 435 deletions(-)
> >
> > diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-
> > secgw/ipsec-secgw.c index ffbd00b..cc8bb57 100644
> > --- a/examples/ipsec-secgw/ipsec-secgw.c
> > +++ b/examples/ipsec-secgw/ipsec-secgw.c
> > @@ -182,6 +182,14 @@ struct lcore_params {
> >  	uint8_t lcore_id;
> >  } __rte_cache_aligned;
> >
> > +/*
> > + * Number of enabled crypto devices
> > + * This number is needed when checking crypto device capabilities  */
> > +uint8_t crypto_dev_num;
> > +/* array of crypto device ID's */
> > +uint8_t crypto_devid[RTE_CRYPTO_MAX_DEVS];
> > +
> >  static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS];
> >
> >  static struct lcore_params *lcore_params; @@ -1623,13 +1631,27 @@
> > check_cryptodev_mask(uint8_t cdev_id)
> >  	return -1;
> >  }
> >
> > +static void
> > +crypto_devid_fill(void)
> > +{
> > +	uint32_t i, n;
> > +
> > +	n = rte_cryptodev_count();
> > +
> > +	for (i = 0; i != n; i++) {
> > +		if (check_cryptodev_mask(i) == 0)
> > +			crypto_devid[crypto_dev_num++] = i;
> > +	}
> > +}
> > +
> >  static int32_t
> >  cryptodevs_init(void)
> >  {
> >  	struct rte_cryptodev_config dev_conf;
> >  	struct rte_cryptodev_qp_conf qp_conf;
> >  	uint16_t idx, max_nb_qps, qp, i;
> > -	int16_t cdev_id, port_id;
> > +	int16_t cdev_id;
> > +	uint32_t dev_max_sess;
> >  	struct rte_hash_parameters params = { 0 };
> >
> >  	params.entries = CDEV_MAP_ENTRIES;
> > @@ -1652,45 +1674,6 @@ cryptodevs_init(void)
> >
> >  	printf("lcore/cryptodev/qp mappings:\n");
> >
> > -	uint32_t max_sess_sz = 0, sess_sz;
> > -	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
> > -		void *sec_ctx;
> > -
> > -		/* Get crypto priv session size */
> > -		sess_sz =
> rte_cryptodev_sym_get_private_session_size(cdev_id);
> > -		if (sess_sz > max_sess_sz)
> > -			max_sess_sz = sess_sz;
> > -
> > -		/*
> > -		 * If crypto device is security capable, need to check the
> > -		 * size of security session as well.
> > -		 */
> > -
> > -		/* Get security context of the crypto device */
> > -		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
> > -		if (sec_ctx == NULL)
> > -			continue;
> > -
> > -		/* Get size of security session */
> > -		sess_sz = rte_security_session_get_size(sec_ctx);
> > -		if (sess_sz > max_sess_sz)
> > -			max_sess_sz = sess_sz;
> > -	}
> > -	RTE_ETH_FOREACH_DEV(port_id) {
> > -		void *sec_ctx;
> > -
> > -		if ((enabled_port_mask & (1 << port_id)) == 0)
> > -			continue;
> > -
> > -		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
> > -		if (sec_ctx == NULL)
> > -			continue;
> > -
> > -		sess_sz = rte_security_session_get_size(sec_ctx);
> > -		if (sess_sz > max_sess_sz)
> > -			max_sess_sz = sess_sz;
> > -	}
> > -
> >  	idx = 0;
> >  	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
> >  		struct rte_cryptodev_info cdev_info; @@ -1722,51 +1705,12
> @@
> > cryptodevs_init(void)
> >  		dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id);
> >  		dev_conf.nb_queue_pairs = qp;
> >
> > -		uint32_t dev_max_sess = cdev_info.sym.max_nb_sessions;
> > +		dev_max_sess = cdev_info.sym.max_nb_sessions;
> >  		if (dev_max_sess != 0 && dev_max_sess <
> CDEV_MP_NB_OBJS)
> >  			rte_exit(EXIT_FAILURE,
> >  				"Device does not support at least %u "
> >  				"sessions", CDEV_MP_NB_OBJS);
> >
> > -		if (!socket_ctx[dev_conf.socket_id].session_pool) {
> > -			char mp_name[RTE_MEMPOOL_NAMESIZE];
> > -			struct rte_mempool *sess_mp;
> > -
> > -			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> > -					"sess_mp_%u", dev_conf.socket_id);
> > -			sess_mp =
> rte_cryptodev_sym_session_pool_create(
> > -					mp_name, CDEV_MP_NB_OBJS,
> > -					0, CDEV_MP_CACHE_SZ, 0,
> > -					dev_conf.socket_id);
> > -			socket_ctx[dev_conf.socket_id].session_pool =
> > sess_mp;
> > -		}
> > -
> > -		if (!socket_ctx[dev_conf.socket_id].session_priv_pool) {
> > -			char mp_name[RTE_MEMPOOL_NAMESIZE];
> > -			struct rte_mempool *sess_mp;
> > -
> > -			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> > -					"sess_mp_priv_%u",
> > dev_conf.socket_id);
> > -			sess_mp = rte_mempool_create(mp_name,
> > -					CDEV_MP_NB_OBJS,
> > -					max_sess_sz,
> > -					CDEV_MP_CACHE_SZ,
> > -					0, NULL, NULL, NULL,
> > -					NULL, dev_conf.socket_id,
> > -					0);
> > -			socket_ctx[dev_conf.socket_id].session_priv_pool =
> > -					sess_mp;
> > -		}
> > -
> > -		if (!socket_ctx[dev_conf.socket_id].session_priv_pool ||
> > -
> 	!socket_ctx[dev_conf.socket_id].session_pool)
> > -			rte_exit(EXIT_FAILURE,
> > -				"Cannot create session pool on socket %d\n",
> > -				dev_conf.socket_id);
> > -		else
> > -			printf("Allocated session pool on socket %d\n",
> > -					dev_conf.socket_id);
> > -
> >  		if (rte_cryptodev_configure(cdev_id, &dev_conf))
> >  			rte_panic("Failed to initialize cryptodev %u\n",
> >  					cdev_id);
> > @@ -1787,38 +1731,6 @@ cryptodevs_init(void)
> >  					cdev_id);
> >  	}
> >
> > -	/* create session pools for eth devices that implement security */
> > -	RTE_ETH_FOREACH_DEV(port_id) {
> > -		if ((enabled_port_mask & (1 << port_id)) &&
> > -				rte_eth_dev_get_sec_ctx(port_id)) {
> > -			int socket_id = rte_eth_dev_socket_id(port_id);
> > -
> > -			if (!socket_ctx[socket_id].session_pool) {
> > -				char mp_name[RTE_MEMPOOL_NAMESIZE];
> > -				struct rte_mempool *sess_mp;
> > -
> > -				snprintf(mp_name,
> RTE_MEMPOOL_NAMESIZE,
> > -						"sess_mp_%u", socket_id);
> > -				sess_mp = rte_mempool_create(mp_name,
> > -						(CDEV_MP_NB_OBJS * 2),
> > -						max_sess_sz,
> > -						CDEV_MP_CACHE_SZ,
> > -						0, NULL, NULL, NULL,
> > -						NULL, socket_id,
> > -						0);
> > -				if (sess_mp == NULL)
> > -					rte_exit(EXIT_FAILURE,
> > -						"Cannot create session pool "
> > -						"on socket %d\n", socket_id);
> > -				else
> > -					printf("Allocated session pool "
> > -						"on socket %d\n", socket_id);
> > -				socket_ctx[socket_id].session_pool =
> sess_mp;
> > -			}
> > -		}
> > -	}
> > -
> > -
> >  	printf("\n");
> >
> >  	return 0;
> > @@ -1984,6 +1896,98 @@ port_init(uint16_t portid, uint64_t
> > req_rx_offloads, uint64_t req_tx_offloads)
> >  	printf("\n");
> >  }
> >
> > +static size_t
> > +max_session_size(void)
> > +{
> > +	size_t max_sz, sz;
> > +	void *sec_ctx;
> > +	int16_t cdev_id, port_id, n;
> > +
> > +	max_sz = 0;
> > +	n =  rte_cryptodev_count();
> > +	for (cdev_id = 0; cdev_id != n; cdev_id++) {
> > +		sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
> > +		if (sz > max_sz)
> > +			max_sz = sz;
> > +		/*
> > +		 * If crypto device is security capable, need to check the
> > +		 * size of security session as well.
> > +		 */
> > +
> > +		/* Get security context of the crypto device */
> > +		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
> > +		if (sec_ctx == NULL)
> > +			continue;
> > +
> > +		/* Get size of security session */
> > +		sz = rte_security_session_get_size(sec_ctx);
> > +		if (sz > max_sz)
> > +			max_sz = sz;
> > +	}
> > +
> > +	RTE_ETH_FOREACH_DEV(port_id) {
> > +		if ((enabled_port_mask & (1 << port_id)) == 0)
> > +			continue;
> > +
> > +		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
> > +		if (sec_ctx == NULL)
> > +			continue;
> > +
> > +		sz = rte_security_session_get_size(sec_ctx);
> > +		if (sz > max_sz)
> > +			max_sz = sz;
> > +	}
> > +
> > +	return max_sz;
> > +}
> > +
> > +static void
> > +session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t
> > +sess_sz) {
> > +	char mp_name[RTE_MEMPOOL_NAMESIZE];
> > +	struct rte_mempool *sess_mp;
> > +
> > +	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> > +			"sess_mp_%u", socket_id);
> > +	sess_mp = rte_cryptodev_sym_session_pool_create(
> > +			mp_name, CDEV_MP_NB_OBJS,
> > +			sess_sz, CDEV_MP_CACHE_SZ, 0,
> > +			socket_id);
> > +	ctx->session_pool = sess_mp;
> > +
> > +	if (ctx->session_pool == NULL)
> > +		rte_exit(EXIT_FAILURE,
> > +			"Cannot init session pool on socket %d\n",
> socket_id);
> > +	else
> > +		printf("Allocated session pool on socket %d\n",
> 	socket_id);
> > +}
> > +
> > +static void
> > +session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
> > +	size_t sess_sz)
> > +{
> > +	char mp_name[RTE_MEMPOOL_NAMESIZE];
> > +	struct rte_mempool *sess_mp;
> > +
> > +	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> > +			"sess_mp_priv_%u", socket_id);
> > +	sess_mp = rte_mempool_create(mp_name,
> > +			CDEV_MP_NB_OBJS,
> > +			sess_sz,
> > +			CDEV_MP_CACHE_SZ,
> > +			0, NULL, NULL, NULL,
> > +			NULL, socket_id,
> > +			0);
> > +	ctx->session_priv_pool = sess_mp;
> > +	if (ctx->session_priv_pool == NULL)
> > +		rte_exit(EXIT_FAILURE,
> > +			"Cannot init session priv pool on socket %d\n",
> > +			socket_id);
> > +	else
> > +		printf("Allocated session priv pool on socket %d\n",
> > +			socket_id);
> > +}
> > +
> >  static void
> >  pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t
> > nb_mbuf)  { @@ -2064,9 +2068,11 @@ main(int32_t argc, char **argv)  {
> >  	int32_t ret;
> >  	uint32_t lcore_id;
> > +	uint32_t i;
> >  	uint8_t socket_id;
> >  	uint16_t portid;
> >  	uint64_t req_rx_offloads, req_tx_offloads;
> > +	size_t sess_sz;
> >
> >  	/* init EAL */
> >  	ret = rte_eal_init(argc, argv);
> > @@ -2094,7 +2100,10 @@ main(int32_t argc, char **argv)
> >
> >  	nb_lcores = rte_lcore_count();
> >
> > -	/* Replicate each context per socket */
> > +	crypto_devid_fill();
> > +
> > +	sess_sz = max_session_size();
> > +
> >  	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
> >  		if (rte_lcore_is_enabled(lcore_id) == 0)
> >  			continue;
> > @@ -2104,20 +2113,17 @@ main(int32_t argc, char **argv)
> >  		else
> >  			socket_id = 0;
> >
> > +		/* mbuf_pool is initialised by the pool_init() function*/
> >  		if (socket_ctx[socket_id].mbuf_pool)
> >  			continue;
> >
> > -		/* initilaze SPD */
> > -		sp4_init(&socket_ctx[socket_id], socket_id);
> > -
> > -		sp6_init(&socket_ctx[socket_id], socket_id);
> > -
> > -		/* initilaze SAD */
> > -		sa_init(&socket_ctx[socket_id], socket_id);
> > -
> > -		rt_init(&socket_ctx[socket_id], socket_id);
> > -
> >  		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
> > +		session_pool_init(&socket_ctx[socket_id], socket_id,
> sess_sz);
> > +		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
> > +			sess_sz);
> > +
> > +		if (!numa_on)
> > +			break;
> >  	}
> >
> >  	RTE_ETH_FOREACH_DEV(portid) {
> > @@ -2135,7 +2141,11 @@ main(int32_t argc, char **argv)
> >  		if ((enabled_port_mask & (1 << portid)) == 0)
> >  			continue;
> >
> > -		/* Start device */
> > +		/*
> > +		 * Start device
> > +		 * note: device must be started before a flow rule
> > +		 * can be installed.
> > +		 */
> >  		ret = rte_eth_dev_start(portid);
> >  		if (ret < 0)
> >  			rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
> > @@ -2153,6 +2163,19 @@ main(int32_t argc, char **argv)
> >  			RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback,
> NULL);
> >  	}
> >
> > +	/* Replicate each context per socket */
> > +	for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
> > +		socket_id = rte_socket_id_by_idx(i);
> > +		if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
> > +			(socket_ctx[socket_id].sa_in == NULL) &&
> > +			(socket_ctx[socket_id].sa_out == NULL)) {
> > +			sa_init(&socket_ctx[socket_id], socket_id);
> > +			sp4_init(&socket_ctx[socket_id], socket_id);
> > +			sp6_init(&socket_ctx[socket_id], socket_id);
> > +			rt_init(&socket_ctx[socket_id], socket_id);
> > +		}
> > +	}
> > +
> >  	check_all_ports_link_status(enabled_port_mask);
> >
> >  	/* launch per-lcore init on every lcore */ diff --git
> > a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c index
> > 4352cb8..e31d472 100644
> > --- a/examples/ipsec-secgw/ipsec.c
> > +++ b/examples/ipsec-secgw/ipsec.c
> > @@ -39,42 +39,17 @@ set_ipsec_conf(struct ipsec_sa *sa, struct
> > rte_security_ipsec_xform *ipsec)
> >  	ipsec->esn_soft_limit = IPSEC_OFFLOAD_ESN_SOFTLIMIT;  }
> >
> > -int
> > -create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
> > +static int
> > +create_sec_session(struct ipsec_sa *sa, struct rte_mempool *pool)
> >  {
> > -	struct rte_cryptodev_info cdev_info;
> > -	unsigned long cdev_id_qp = 0;
> > +	const struct rte_security_capability *sec_cap;
> > +	struct rte_security_ctx *ctx;
> >  	int32_t ret = 0;
> > -	struct cdev_key key = { 0 };
> > -
> > -	key.lcore_id = (uint8_t)rte_lcore_id();
> > -
> > -	key.cipher_algo = (uint8_t)sa->cipher_algo;
> > -	key.auth_algo = (uint8_t)sa->auth_algo;
> > -	key.aead_algo = (uint8_t)sa->aead_algo;
> > -
> > -	if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {
> > -		ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
> > -				(void **)&cdev_id_qp);
> > -		if (ret < 0) {
> > -			RTE_LOG(ERR, IPSEC,
> > -				"No cryptodev: core %u, cipher_algo %u, "
> > -				"auth_algo %u, aead_algo %u\n",
> > -				key.lcore_id,
> > -				key.cipher_algo,
> > -				key.auth_algo,
> > -				key.aead_algo);
> > -			return -1;
> > -		}
> > -	}
> >
> > -	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on
> cryptodev
> > "
> > -			"%u qp %u\n", sa->spi,
> > -			ipsec_ctx->tbl[cdev_id_qp].id,
> > -			ipsec_ctx->tbl[cdev_id_qp].qp);
> > +	if ((sa == NULL) || (pool == NULL))
> > +		return -EINVAL;
> >
> > -	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
> > -		struct rte_security_session_conf sess_conf = {
> > +	struct rte_security_session_conf sess_conf = {
> >  			.action_type = sa->type,
> >  			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
> >  			{.ipsec = {
> > @@ -90,247 +65,340 @@ create_session(struct ipsec_ctx *ipsec_ctx,
> > struct ipsec_sa *sa)
> >  			} },
> >  			.crypto_xform = sa->xforms,
> >  			.userdata = NULL,
> > -
> >  		};
> >
> > -		if (sa->type ==
> > RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > -			struct rte_security_ctx *ctx = (struct rte_security_ctx
> *)
> > -
> > 	rte_cryptodev_get_sec_ctx(
> > -							ipsec_ctx-
> > >tbl[cdev_id_qp].id);
> > -
> > -			/* Set IPsec parameters in conf */
> > -			set_ipsec_conf(sa, &(sess_conf.ipsec));
> > -
> > -			sa->sec_session = rte_security_session_create(ctx,
> > -					&sess_conf, ipsec_ctx-
> >session_pool);
> > -			if (sa->sec_session == NULL) {
> > -				RTE_LOG(ERR, IPSEC,
> > -				"SEC Session init failed: err: %d\n", ret);
> > -				return -1;
> > -			}
> > -		} else if (sa->type ==
> > RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
> > -			struct rte_flow_error err;
> > -			struct rte_security_ctx *ctx = (struct rte_security_ctx
> *)
> > -
> > 	rte_eth_dev_get_sec_ctx(
> > -							sa->portid);
> > -			const struct rte_security_capability *sec_cap;
> > -			int ret = 0;
> > -
> > -			sa->sec_session = rte_security_session_create(ctx,
> > -					&sess_conf, ipsec_ctx-
> >session_pool);
> > -			if (sa->sec_session == NULL) {
> > -				RTE_LOG(ERR, IPSEC,
> > -				"SEC Session init failed: err: %d\n", ret);
> > -				return -1;
> > -			}
> > +	if (sa->type ==
> RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > +		ctx = (struct rte_security_ctx *)
> > +				rte_eth_dev_get_sec_ctx(sa->portid);
> >
> > -			sec_cap = rte_security_capabilities_get(ctx);
> > +		/* Set IPsec parameters in conf */
> > +		set_ipsec_conf(sa, &(sess_conf.ipsec));
> >
> > -			/* iterate until ESP tunnel*/
> > -			while (sec_cap->action !=
> > -
> 	RTE_SECURITY_ACTION_TYPE_NONE)
> > {
> > +		sa->sec_session = rte_security_session_create(ctx,
> > +				&sess_conf, pool);
> > +		if (sa->sec_session == NULL) {
> > +			RTE_LOG(ERR, IPSEC,
> > +				"SEC Session init failed: err: %d\n",
> > +				ret);
> > +			return -1;
> > +		}
> > +	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
> {
> > +		struct rte_flow_error err;
> > +		ctx = (struct rte_security_ctx *)
> > +				rte_eth_dev_get_sec_ctx(sa->portid);
> > +		sa->sec_session = rte_security_session_create(ctx,
> > +				&sess_conf, pool);
> > +		if (sa->sec_session == NULL) {
> > +			RTE_LOG(ERR, IPSEC, "SEC Session init failed\n");
> > +			return -1;
> > +		}
> >
> > -				if (sec_cap->action == sa->type &&
> > -				    sec_cap->protocol ==
> > -					RTE_SECURITY_PROTOCOL_IPSEC &&
> > -				    sec_cap->ipsec.mode ==
> > -
> > 	RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
> > -				    sec_cap->ipsec.direction == sa->direction)
> > -					break;
> > -				sec_cap++;
> > -			}
> > +		sec_cap = rte_security_capabilities_get(ctx);
> > +
> > +		/* iterate until ESP tunnel*/
> > +		while (sec_cap->action !=
> RTE_SECURITY_ACTION_TYPE_NONE)
> > {
> > +			if (sec_cap->action == sa->type &&
> > +				sec_cap->protocol ==
> > +				RTE_SECURITY_PROTOCOL_IPSEC &&
> > +				sec_cap->ipsec.mode ==
> > +				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL
> &&
> > +				sec_cap->ipsec.direction == sa->direction)
> > +				break;
> > +			sec_cap++;
> > +		}
> >
> > -			if (sec_cap->action ==
> > RTE_SECURITY_ACTION_TYPE_NONE) {
> > -				RTE_LOG(ERR, IPSEC,
> > +		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE)
> {
> > +			RTE_LOG(ERR, IPSEC,
> >  				"No suitable security capability found\n");
> >  				return -1;
> > -			}
> > +		}
> >
> > -			sa->ol_flags = sec_cap->ol_flags;
> > -			sa->security_ctx = ctx;
> > -			sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
> > -
> > -			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
> > -			sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
> > -			if (sa->flags & IP6_TUNNEL) {
> > -				sa->pattern[1].spec = &sa->ipv6_spec;
> > -				memcpy(sa->ipv6_spec.hdr.dst_addr,
> > -					sa->dst.ip.ip6.ip6_b, 16);
> > -				memcpy(sa->ipv6_spec.hdr.src_addr,
> > -				       sa->src.ip.ip6.ip6_b, 16);
> > -			} else {
> > -				sa->pattern[1].spec = &sa->ipv4_spec;
> > -				sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
> > -				sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
> > -			}
> > +		sa->ol_flags = sec_cap->ol_flags;
> > +		sa->security_ctx = ctx;
> > +		sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
> > +
> > +		sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
> > +		sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
> > +		if (sa->flags & IP6_TUNNEL) {
> > +			sa->pattern[1].spec = &sa->ipv6_spec;
> > +			memcpy(sa->ipv6_spec.hdr.dst_addr,
> > +				sa->dst.ip.ip6.ip6_b, 16);
> > +			memcpy(sa->ipv6_spec.hdr.src_addr,
> > +			       sa->src.ip.ip6.ip6_b, 16);
> > +		} else {
> > +			sa->pattern[1].spec = &sa->ipv4_spec;
> > +			sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
> > +			sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
> > +		}
> >
> > -			sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
> > -			sa->pattern[2].spec = &sa->esp_spec;
> > -			sa->pattern[2].mask = &rte_flow_item_esp_mask;
> > -			sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
> > +		sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
> > +		sa->pattern[2].spec = &sa->esp_spec;
> > +		sa->pattern[2].mask = &rte_flow_item_esp_mask;
> > +		sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
> >
> > -			sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
> > +		sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
> >
> > -			sa->action[0].type =
> > RTE_FLOW_ACTION_TYPE_SECURITY;
> > -			sa->action[0].conf = sa->sec_session;
> > +		sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
> > +		sa->action[0].conf = sa->sec_session;
> >
> > -			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
> > +		sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
> >
> > -			sa->attr.egress = (sa->direction ==
> > +		sa->attr.egress = (sa->direction ==
> >
> > 	RTE_SECURITY_IPSEC_SA_DIR_EGRESS);
> > -			sa->attr.ingress = (sa->direction ==
> > +		sa->attr.ingress = (sa->direction ==
> >
> > 	RTE_SECURITY_IPSEC_SA_DIR_INGRESS);
> > -			if (sa->attr.ingress) {
> > -				uint8_t rss_key[40];
> > -				struct rte_eth_rss_conf rss_conf = {
> > -					.rss_key = rss_key,
> > -					.rss_key_len = 40,
> > -				};
> > -				struct rte_eth_dev *eth_dev;
> > -				uint16_t
> > queue[RTE_MAX_QUEUES_PER_PORT];
> > -				struct rte_flow_action_rss action_rss;
> > -				unsigned int i;
> > -				unsigned int j;
> > -
> > -				sa->action[2].type =
> > RTE_FLOW_ACTION_TYPE_END;
> > -				/* Try RSS. */
> > -				sa->action[1].type =
> > RTE_FLOW_ACTION_TYPE_RSS;
> > -				sa->action[1].conf = &action_rss;
> > -				eth_dev = ctx->device;
> > -				rte_eth_dev_rss_hash_conf_get(sa->portid,
> > -							      &rss_conf);
> > -				for (i = 0, j = 0;
> > -				     i < eth_dev->data->nb_rx_queues; ++i)
> > -					if (eth_dev->data->rx_queues[i])
> > -						queue[j++] = i;
> > +		if (sa->attr.ingress) {
> > +			uint8_t rss_key[40];
> > +			struct rte_eth_rss_conf rss_conf = {
> > +				.rss_key = rss_key,
> > +				.rss_key_len = 40,
> > +			};
> > +			struct rte_eth_dev *eth_dev;
> > +			uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
> > +			struct rte_flow_action_rss action_rss;
> > +			unsigned int i;
> > +			unsigned int j;
> > +
> > +			sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
> > +			/* Try RSS. */
> > +			sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
> > +			sa->action[1].conf = &action_rss;
> > +			eth_dev = ctx->device;
> > +			rte_eth_dev_rss_hash_conf_get(sa->portid,
> > +						&rss_conf);
> > +			for (i = 0, j = 0; i < eth_dev->data->nb_rx_queues;
> > +				++i)
> > +				if (eth_dev->data->rx_queues[i])
> > +					queue[j++] = i;
> Compilation error
> 
> /home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c: In
> function 'create_sec_session':
> /home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c:169:4:
> error: this 'for' clause does not guard... [-Werror=misleading-indentation]
>     for (i = 0, j = 0; i < eth_dev->data->nb_rx_queues;
>     ^~~
> /home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c:173:5:
> note: ...this statement, but the latter is misleadingly indented as if it is
> guarded by the 'for'
>      action_rss = (struct rte_flow_action_rss){
>      ^~~~~~~~~~
> 
<snip>

I will send a v4.
What compiler are you using?

Regards,

Bernard.

Re: [dpdk-dev] [EXT] RE: [PATCH v3 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Akhil Goyal]

> > > +                   rte_eth_dev_rss_hash_conf_get(sa->portid,
> > > +                                           &rss_conf);
> > > +                   for (i = 0, j = 0; i < eth_dev->data->nb_rx_queues;
> > > +                           ++i)
> > > +                           if (eth_dev->data->rx_queues[i])
> > > +                                   queue[j++] = i;
> > Compilation error
> >
> > /home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c: In
> > function 'create_sec_session':
> > /home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c:169:4:
> > error: this 'for' clause does not guard... [-Werror=misleading-indentation]
> >     for (i = 0, j = 0; i < eth_dev->data->nb_rx_queues;
> >     ^~~
> > /home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c:173:5:
> > note: ...this statement, but the latter is misleadingly indented as if it is
> > guarded by the 'for'
> >      action_rss = (struct rte_flow_action_rss){
> >      ^~~~~~~~~~
> >
> <snip>
> 
> I will send a v4.
> What compiler are you using?

Cross compile for arm
gcc-linaro-6.1.1-2016.08-x86_64_aarch64-linux-gnu
> 
> Regards,
> 
> Bernard.
> 

Re: [dpdk-dev] [EXT] RE: [PATCH v3 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Akhil Goyal]

> > > +                   rte_eth_dev_rss_hash_conf_get(sa->portid,
> > > +                                           &rss_conf);
> > > +                   for (i = 0, j = 0; i < eth_dev->data->nb_rx_queues;
> > > +                           ++i)
> > > +                           if (eth_dev->data->rx_queues[i])
> > > +                                   queue[j++] = i;
> > Compilation error
> >
> > /home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c: In
> > function 'create_sec_session':
> > /home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c:169:4:
> > error: this 'for' clause does not guard... [-Werror=misleading-indentation]
> >     for (i = 0, j = 0; i < eth_dev->data->nb_rx_queues;
> >     ^~~
> > /home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c:173:5:
> > note: ...this statement, but the latter is misleadingly indented as if it is
> > guarded by the 'for'
> >      action_rss = (struct rte_flow_action_rss){
> >      ^~~~~~~~~~
> >
> <snip>
> 
> I will send a v4.
> What compiler are you using?

Cross compile for arm
gcc-linaro-6.1.1-2016.08-x86_64_aarch64-linux-gnu
> 
> Regards,
> 
> Bernard.
> 

[dpdk-dev] [PATCH v4 0/2] examples/ipsec-secgw: fix 1st pkt dropped

[Bernard Iremonger]

This patchset fixes the issue of the first inbound packet
being dropped for inline crypto.

Changes in v4:
--------------
Fix indentation at line 173 in ipsec.c to solve compile error on ARM.

Changes in v3:
--------------
The previous refactoring of the create_session() function has been dropped.
The create_session() function is now called from sa_init() at startup.

The following functions have been added:
crypto_devid_fill() in ipsec-secgw.c
check_cryptodev_capability() in ipsec.c
check_cryptodev_aead_capability() in ipsec.c
create_sec_session() and create_crypto_session() in ipsec.c

The create_session() function has been refactored to call
the create_sec_session() and create_crypto_session() functions.


Changes in v2:
--------------
The first three patches of the v1 have been squashed.
The commit message for the squashed patch has been updated.
Patches 4,5 and 6 of the v1 have been dropped from this patchset.
A patch to fix the test scripts has been added.

Bernard Iremonger (2):
  examples/ipsec-secgw: fix 1st packet dropped for inline crypto
  examples/ipsec-secgw/test: fix inline test scripts

 examples/ipsec-secgw/ipsec-secgw.c           | 271 +++++++------
 examples/ipsec-secgw/ipsec.c                 | 581 ++++++++++++++-------------
 examples/ipsec-secgw/ipsec.h                 |  10 +-
 examples/ipsec-secgw/ipsec_process.c         |  38 +-
 examples/ipsec-secgw/sa.c                    |  42 +-
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 -
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 -
 7 files changed, 501 insertions(+), 461 deletions(-)

-- 
2.7.4

[dpdk-dev] [PATCH v4 0/2] examples/ipsec-secgw: fix 1st pkt dropped

[Bernard Iremonger]

This patchset fixes the issue of the first inbound packet
being dropped for inline crypto.

Changes in v4:
--------------
Fix indentation at line 173 in ipsec.c to solve compile error on ARM.

Changes in v3:
--------------
The previous refactoring of the create_session() function has been dropped.
The create_session() function is now called from sa_init() at startup.

The following functions have been added:
crypto_devid_fill() in ipsec-secgw.c
check_cryptodev_capability() in ipsec.c
check_cryptodev_aead_capability() in ipsec.c
create_sec_session() and create_crypto_session() in ipsec.c

The create_session() function has been refactored to call
the create_sec_session() and create_crypto_session() functions.


Changes in v2:
--------------
The first three patches of the v1 have been squashed.
The commit message for the squashed patch has been updated.
Patches 4,5 and 6 of the v1 have been dropped from this patchset.
A patch to fix the test scripts has been added.

Bernard Iremonger (2):
  examples/ipsec-secgw: fix 1st packet dropped for inline crypto
  examples/ipsec-secgw/test: fix inline test scripts

 examples/ipsec-secgw/ipsec-secgw.c           | 271 +++++++------
 examples/ipsec-secgw/ipsec.c                 | 581 ++++++++++++++-------------
 examples/ipsec-secgw/ipsec.h                 |  10 +-
 examples/ipsec-secgw/ipsec_process.c         |  38 +-
 examples/ipsec-secgw/sa.c                    |  42 +-
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 -
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 -
 7 files changed, 501 insertions(+), 461 deletions(-)

-- 
2.7.4

[dpdk-dev] [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Bernard Iremonger]

Inline crypto installs a flow rule in the NIC. This flow
rule must be installed before the first inbound packet is
received.

The create_session() function installs the flow rule.

Refactor ipsec-secgw.c, sa.c, ipsec.h and ipsec.c to create
sessions at startup to fix the issue of the first packet
being dropped for inline crypto.

The create_session() function is now called at initialisation in
sa_add_rules() which is called from sa_init().
The return code for add_rules is checked.
Calls to create_session() in other functions are dropped.

Add crypto_devid_fill() in ipsec-secgw.c
Add max_session_size() in ipsec-secgw.c
Add check_cryptodev_capability() in ipsec.c
Add check_cryptodev_aead_capability() in ipsec.c
Add create_sec_session() and create_crypto_session() in ipsec.c

The crypto_dev_fill() function has been added to find the
enabled crypto devices.

The max_session_size() function has been added to calculate memory
requirements.

The check_cryptodev_capability() and check_cryptodev_aead_capability()
functions have been added to check that the SA is supported by the
crypto device.

The create_session() function is refactored to use the
create_sec_session() and create_crypto_session() functions.

The cryprodev_init() function has been refactored to drop calls to
rte_mempool_create() and to drop calculation of memory requirements.

The main() function has been refactored to call crypto_devid_fill()
and max_session_size() and to call session_pool_init() and
session_priv_pool_init().
The ports are started now before adding a flow rule in main().
The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
now called after the ports have been started.

Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
---
 examples/ipsec-secgw/ipsec-secgw.c   | 271 ++++++++--------
 examples/ipsec-secgw/ipsec.c         | 581 +++++++++++++++++++----------------
 examples/ipsec-secgw/ipsec.h         |  10 +-
 examples/ipsec-secgw/ipsec_process.c |  38 +--
 examples/ipsec-secgw/sa.c            |  42 ++-
 5 files changed, 501 insertions(+), 441 deletions(-)

diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index ffbd00b..cc8bb57 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -182,6 +182,14 @@ struct lcore_params {
 	uint8_t lcore_id;
 } __rte_cache_aligned;
 
+/*
+ * Number of enabled crypto devices
+ * This number is needed when checking crypto device capabilities
+ */
+uint8_t crypto_dev_num;
+/* array of crypto device ID's */
+uint8_t crypto_devid[RTE_CRYPTO_MAX_DEVS];
+
 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS];
 
 static struct lcore_params *lcore_params;
@@ -1623,13 +1631,27 @@ check_cryptodev_mask(uint8_t cdev_id)
 	return -1;
 }
 
+static void
+crypto_devid_fill(void)
+{
+	uint32_t i, n;
+
+	n = rte_cryptodev_count();
+
+	for (i = 0; i != n; i++) {
+		if (check_cryptodev_mask(i) == 0)
+			crypto_devid[crypto_dev_num++] = i;
+	}
+}
+
 static int32_t
 cryptodevs_init(void)
 {
 	struct rte_cryptodev_config dev_conf;
 	struct rte_cryptodev_qp_conf qp_conf;
 	uint16_t idx, max_nb_qps, qp, i;
-	int16_t cdev_id, port_id;
+	int16_t cdev_id;
+	uint32_t dev_max_sess;
 	struct rte_hash_parameters params = { 0 };
 
 	params.entries = CDEV_MAP_ENTRIES;
@@ -1652,45 +1674,6 @@ cryptodevs_init(void)
 
 	printf("lcore/cryptodev/qp mappings:\n");
 
-	uint32_t max_sess_sz = 0, sess_sz;
-	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
-		void *sec_ctx;
-
-		/* Get crypto priv session size */
-		sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-
-		/*
-		 * If crypto device is security capable, need to check the
-		 * size of security session as well.
-		 */
-
-		/* Get security context of the crypto device */
-		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		/* Get size of security session */
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-	RTE_ETH_FOREACH_DEV(port_id) {
-		void *sec_ctx;
-
-		if ((enabled_port_mask & (1 << port_id)) == 0)
-			continue;
-
-		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-
 	idx = 0;
 	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
 		struct rte_cryptodev_info cdev_info;
@@ -1722,51 +1705,12 @@ cryptodevs_init(void)
 		dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id);
 		dev_conf.nb_queue_pairs = qp;
 
-		uint32_t dev_max_sess = cdev_info.sym.max_nb_sessions;
+		dev_max_sess = cdev_info.sym.max_nb_sessions;
 		if (dev_max_sess != 0 && dev_max_sess < CDEV_MP_NB_OBJS)
 			rte_exit(EXIT_FAILURE,
 				"Device does not support at least %u "
 				"sessions", CDEV_MP_NB_OBJS);
 
-		if (!socket_ctx[dev_conf.socket_id].session_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_%u", dev_conf.socket_id);
-			sess_mp = rte_cryptodev_sym_session_pool_create(
-					mp_name, CDEV_MP_NB_OBJS,
-					0, CDEV_MP_CACHE_SZ, 0,
-					dev_conf.socket_id);
-			socket_ctx[dev_conf.socket_id].session_pool = sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_priv_%u", dev_conf.socket_id);
-			sess_mp = rte_mempool_create(mp_name,
-					CDEV_MP_NB_OBJS,
-					max_sess_sz,
-					CDEV_MP_CACHE_SZ,
-					0, NULL, NULL, NULL,
-					NULL, dev_conf.socket_id,
-					0);
-			socket_ctx[dev_conf.socket_id].session_priv_pool =
-					sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool ||
-				!socket_ctx[dev_conf.socket_id].session_pool)
-			rte_exit(EXIT_FAILURE,
-				"Cannot create session pool on socket %d\n",
-				dev_conf.socket_id);
-		else
-			printf("Allocated session pool on socket %d\n",
-					dev_conf.socket_id);
-
 		if (rte_cryptodev_configure(cdev_id, &dev_conf))
 			rte_panic("Failed to initialize cryptodev %u\n",
 					cdev_id);
@@ -1787,38 +1731,6 @@ cryptodevs_init(void)
 					cdev_id);
 	}
 
-	/* create session pools for eth devices that implement security */
-	RTE_ETH_FOREACH_DEV(port_id) {
-		if ((enabled_port_mask & (1 << port_id)) &&
-				rte_eth_dev_get_sec_ctx(port_id)) {
-			int socket_id = rte_eth_dev_socket_id(port_id);
-
-			if (!socket_ctx[socket_id].session_pool) {
-				char mp_name[RTE_MEMPOOL_NAMESIZE];
-				struct rte_mempool *sess_mp;
-
-				snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-						"sess_mp_%u", socket_id);
-				sess_mp = rte_mempool_create(mp_name,
-						(CDEV_MP_NB_OBJS * 2),
-						max_sess_sz,
-						CDEV_MP_CACHE_SZ,
-						0, NULL, NULL, NULL,
-						NULL, socket_id,
-						0);
-				if (sess_mp == NULL)
-					rte_exit(EXIT_FAILURE,
-						"Cannot create session pool "
-						"on socket %d\n", socket_id);
-				else
-					printf("Allocated session pool "
-						"on socket %d\n", socket_id);
-				socket_ctx[socket_id].session_pool = sess_mp;
-			}
-		}
-	}
-
-
 	printf("\n");
 
 	return 0;
@@ -1984,6 +1896,98 @@ port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads)
 	printf("\n");
 }
 
+static size_t
+max_session_size(void)
+{
+	size_t max_sz, sz;
+	void *sec_ctx;
+	int16_t cdev_id, port_id, n;
+
+	max_sz = 0;
+	n =  rte_cryptodev_count();
+	for (cdev_id = 0; cdev_id != n; cdev_id++) {
+		sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
+		if (sz > max_sz)
+			max_sz = sz;
+		/*
+		 * If crypto device is security capable, need to check the
+		 * size of security session as well.
+		 */
+
+		/* Get security context of the crypto device */
+		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		/* Get size of security session */
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	RTE_ETH_FOREACH_DEV(port_id) {
+		if ((enabled_port_mask & (1 << port_id)) == 0)
+			continue;
+
+		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	return max_sz;
+}
+
+static void
+session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_%u", socket_id);
+	sess_mp = rte_cryptodev_sym_session_pool_create(
+			mp_name, CDEV_MP_NB_OBJS,
+			sess_sz, CDEV_MP_CACHE_SZ, 0,
+			socket_id);
+	ctx->session_pool = sess_mp;
+
+	if (ctx->session_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session pool on socket %d\n", socket_id);
+	else
+		printf("Allocated session pool on socket %d\n",	socket_id);
+}
+
+static void
+session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
+	size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_priv_%u", socket_id);
+	sess_mp = rte_mempool_create(mp_name,
+			CDEV_MP_NB_OBJS,
+			sess_sz,
+			CDEV_MP_CACHE_SZ,
+			0, NULL, NULL, NULL,
+			NULL, socket_id,
+			0);
+	ctx->session_priv_pool = sess_mp;
+	if (ctx->session_priv_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session priv pool on socket %d\n",
+			socket_id);
+	else
+		printf("Allocated session priv pool on socket %d\n",
+			socket_id);
+}
+
 static void
 pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf)
 {
@@ -2064,9 +2068,11 @@ main(int32_t argc, char **argv)
 {
 	int32_t ret;
 	uint32_t lcore_id;
+	uint32_t i;
 	uint8_t socket_id;
 	uint16_t portid;
 	uint64_t req_rx_offloads, req_tx_offloads;
+	size_t sess_sz;
 
 	/* init EAL */
 	ret = rte_eal_init(argc, argv);
@@ -2094,7 +2100,10 @@ main(int32_t argc, char **argv)
 
 	nb_lcores = rte_lcore_count();
 
-	/* Replicate each context per socket */
+	crypto_devid_fill();
+
+	sess_sz = max_session_size();
+
 	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
 		if (rte_lcore_is_enabled(lcore_id) == 0)
 			continue;
@@ -2104,20 +2113,17 @@ main(int32_t argc, char **argv)
 		else
 			socket_id = 0;
 
+		/* mbuf_pool is initialised by the pool_init() function*/
 		if (socket_ctx[socket_id].mbuf_pool)
 			continue;
 
-		/* initilaze SPD */
-		sp4_init(&socket_ctx[socket_id], socket_id);
-
-		sp6_init(&socket_ctx[socket_id], socket_id);
-
-		/* initilaze SAD */
-		sa_init(&socket_ctx[socket_id], socket_id);
-
-		rt_init(&socket_ctx[socket_id], socket_id);
-
 		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
+		session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
+		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
+			sess_sz);
+
+		if (!numa_on)
+			break;
 	}
 
 	RTE_ETH_FOREACH_DEV(portid) {
@@ -2135,7 +2141,11 @@ main(int32_t argc, char **argv)
 		if ((enabled_port_mask & (1 << portid)) == 0)
 			continue;
 
-		/* Start device */
+		/*
+		 * Start device
+		 * note: device must be started before a flow rule
+		 * can be installed.
+		 */
 		ret = rte_eth_dev_start(portid);
 		if (ret < 0)
 			rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
@@ -2153,6 +2163,19 @@ main(int32_t argc, char **argv)
 			RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback, NULL);
 	}
 
+	/* Replicate each context per socket */
+	for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
+		socket_id = rte_socket_id_by_idx(i);
+		if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
+			(socket_ctx[socket_id].sa_in == NULL) &&
+			(socket_ctx[socket_id].sa_out == NULL)) {
+			sa_init(&socket_ctx[socket_id], socket_id);
+			sp4_init(&socket_ctx[socket_id], socket_id);
+			sp6_init(&socket_ctx[socket_id], socket_id);
+			rt_init(&socket_ctx[socket_id], socket_id);
+		}
+	}
+
 	check_all_ports_link_status(enabled_port_mask);
 
 	/* launch per-lcore init on every lcore */
diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index 4352cb8..83080f7 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -39,42 +39,17 @@ set_ipsec_conf(struct ipsec_sa *sa, struct rte_security_ipsec_xform *ipsec)
 	ipsec->esn_soft_limit = IPSEC_OFFLOAD_ESN_SOFTLIMIT;
 }
 
-int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
+static int
+create_sec_session(struct ipsec_sa *sa, struct rte_mempool *pool)
 {
-	struct rte_cryptodev_info cdev_info;
-	unsigned long cdev_id_qp = 0;
+	const struct rte_security_capability *sec_cap;
+	struct rte_security_ctx *ctx;
 	int32_t ret = 0;
-	struct cdev_key key = { 0 };
-
-	key.lcore_id = (uint8_t)rte_lcore_id();
-
-	key.cipher_algo = (uint8_t)sa->cipher_algo;
-	key.auth_algo = (uint8_t)sa->auth_algo;
-	key.aead_algo = (uint8_t)sa->aead_algo;
-
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
-				(void **)&cdev_id_qp);
-		if (ret < 0) {
-			RTE_LOG(ERR, IPSEC,
-				"No cryptodev: core %u, cipher_algo %u, "
-				"auth_algo %u, aead_algo %u\n",
-				key.lcore_id,
-				key.cipher_algo,
-				key.auth_algo,
-				key.aead_algo);
-			return -1;
-		}
-	}
 
-	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on cryptodev "
-			"%u qp %u\n", sa->spi,
-			ipsec_ctx->tbl[cdev_id_qp].id,
-			ipsec_ctx->tbl[cdev_id_qp].qp);
+	if ((sa == NULL) || (pool == NULL))
+		return -EINVAL;
 
-	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
-		struct rte_security_session_conf sess_conf = {
+	struct rte_security_session_conf sess_conf = {
 			.action_type = sa->type,
 			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 			{.ipsec = {
@@ -90,247 +65,340 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			} },
 			.crypto_xform = sa->xforms,
 			.userdata = NULL,
-
 		};
 
-		if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
-			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
-							rte_cryptodev_get_sec_ctx(
-							ipsec_ctx->tbl[cdev_id_qp].id);
-
-			/* Set IPsec parameters in conf */
-			set_ipsec_conf(sa, &(sess_conf.ipsec));
-
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"SEC Session init failed: err: %d\n", ret);
-				return -1;
-			}
-		} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
-			struct rte_flow_error err;
-			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
-							rte_eth_dev_get_sec_ctx(
-							sa->portid);
-			const struct rte_security_capability *sec_cap;
-			int ret = 0;
-
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"SEC Session init failed: err: %d\n", ret);
-				return -1;
-			}
+	if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
+		ctx = (struct rte_security_ctx *)
+				rte_eth_dev_get_sec_ctx(sa->portid);
 
-			sec_cap = rte_security_capabilities_get(ctx);
+		/* Set IPsec parameters in conf */
+		set_ipsec_conf(sa, &(sess_conf.ipsec));
 
-			/* iterate until ESP tunnel*/
-			while (sec_cap->action !=
-					RTE_SECURITY_ACTION_TYPE_NONE) {
+		sa->sec_session = rte_security_session_create(ctx,
+				&sess_conf, pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC,
+				"SEC Session init failed: err: %d\n",
+				ret);
+			return -1;
+		}
+	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		struct rte_flow_error err;
+		ctx = (struct rte_security_ctx *)
+				rte_eth_dev_get_sec_ctx(sa->portid);
+		sa->sec_session = rte_security_session_create(ctx,
+				&sess_conf, pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC, "SEC Session init failed\n");
+			return -1;
+		}
 
-				if (sec_cap->action == sa->type &&
-				    sec_cap->protocol ==
-					RTE_SECURITY_PROTOCOL_IPSEC &&
-				    sec_cap->ipsec.mode ==
-					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
-				    sec_cap->ipsec.direction == sa->direction)
-					break;
-				sec_cap++;
-			}
+		sec_cap = rte_security_capabilities_get(ctx);
+
+		/* iterate until ESP tunnel*/
+		while (sec_cap->action != RTE_SECURITY_ACTION_TYPE_NONE) {
+			if (sec_cap->action == sa->type &&
+				sec_cap->protocol ==
+				RTE_SECURITY_PROTOCOL_IPSEC &&
+				sec_cap->ipsec.mode ==
+				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
+				sec_cap->ipsec.direction == sa->direction)
+				break;
+			sec_cap++;
+		}
 
-			if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
-				RTE_LOG(ERR, IPSEC,
+		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
+			RTE_LOG(ERR, IPSEC,
 				"No suitable security capability found\n");
 				return -1;
-			}
+		}
 
-			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
-			sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
-
-			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
-			sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
-			if (sa->flags & IP6_TUNNEL) {
-				sa->pattern[1].spec = &sa->ipv6_spec;
-				memcpy(sa->ipv6_spec.hdr.dst_addr,
-					sa->dst.ip.ip6.ip6_b, 16);
-				memcpy(sa->ipv6_spec.hdr.src_addr,
-				       sa->src.ip.ip6.ip6_b, 16);
-			} else {
-				sa->pattern[1].spec = &sa->ipv4_spec;
-				sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
-				sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
-			}
+		sa->ol_flags = sec_cap->ol_flags;
+		sa->security_ctx = ctx;
+		sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
+
+		sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
+		sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
+		if (sa->flags & IP6_TUNNEL) {
+			sa->pattern[1].spec = &sa->ipv6_spec;
+			memcpy(sa->ipv6_spec.hdr.dst_addr,
+				sa->dst.ip.ip6.ip6_b, 16);
+			memcpy(sa->ipv6_spec.hdr.src_addr,
+			       sa->src.ip.ip6.ip6_b, 16);
+		} else {
+			sa->pattern[1].spec = &sa->ipv4_spec;
+			sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
+			sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
+		}
 
-			sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
-			sa->pattern[2].spec = &sa->esp_spec;
-			sa->pattern[2].mask = &rte_flow_item_esp_mask;
-			sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
+		sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
+		sa->pattern[2].spec = &sa->esp_spec;
+		sa->pattern[2].mask = &rte_flow_item_esp_mask;
+		sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
 
-			sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
+		sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
 
-			sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
-			sa->action[0].conf = sa->sec_session;
+		sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
+		sa->action[0].conf = sa->sec_session;
 
-			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
+		sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
 
-			sa->attr.egress = (sa->direction ==
+		sa->attr.egress = (sa->direction ==
 					RTE_SECURITY_IPSEC_SA_DIR_EGRESS);
-			sa->attr.ingress = (sa->direction ==
+		sa->attr.ingress = (sa->direction ==
 					RTE_SECURITY_IPSEC_SA_DIR_INGRESS);
-			if (sa->attr.ingress) {
-				uint8_t rss_key[40];
-				struct rte_eth_rss_conf rss_conf = {
-					.rss_key = rss_key,
-					.rss_key_len = 40,
-				};
-				struct rte_eth_dev *eth_dev;
-				uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
-				struct rte_flow_action_rss action_rss;
-				unsigned int i;
-				unsigned int j;
-
-				sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
-				/* Try RSS. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
-				sa->action[1].conf = &action_rss;
-				eth_dev = ctx->device;
-				rte_eth_dev_rss_hash_conf_get(sa->portid,
-							      &rss_conf);
-				for (i = 0, j = 0;
-				     i < eth_dev->data->nb_rx_queues; ++i)
-					if (eth_dev->data->rx_queues[i])
-						queue[j++] = i;
-				action_rss = (struct rte_flow_action_rss){
-					.types = rss_conf.rss_hf,
-					.key_len = rss_conf.rss_key_len,
-					.queue_num = j,
-					.key = rss_key,
-					.queue = queue,
-				};
-				ret = rte_flow_validate(sa->portid, &sa->attr,
-							sa->pattern, sa->action,
-							&err);
-				if (!ret)
-					goto flow_create;
-				/* Try Queue. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_QUEUE;
-				sa->action[1].conf =
-					&(struct rte_flow_action_queue){
-					.index = 0,
-				};
-				ret = rte_flow_validate(sa->portid, &sa->attr,
-							sa->pattern, sa->action,
-							&err);
-				/* Try End. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
-				sa->action[1].conf = NULL;
-				ret = rte_flow_validate(sa->portid, &sa->attr,
+		if (sa->attr.ingress) {
+			uint8_t rss_key[40];
+			struct rte_eth_rss_conf rss_conf = {
+				.rss_key = rss_key,
+				.rss_key_len = 40,
+			};
+			struct rte_eth_dev *eth_dev;
+			uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
+			struct rte_flow_action_rss action_rss;
+			unsigned int i;
+			unsigned int j;
+
+			sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
+			/* Try RSS. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
+			sa->action[1].conf = &action_rss;
+			eth_dev = ctx->device;
+			rte_eth_dev_rss_hash_conf_get(sa->portid,
+						&rss_conf);
+			for (i = 0, j = 0;
+			     i < eth_dev->data->nb_rx_queues; ++i)
+				if (eth_dev->data->rx_queues[i])
+					queue[j++] = i;
+			action_rss = (struct rte_flow_action_rss){
+				.types = rss_conf.rss_hf,
+				.key_len = rss_conf.rss_key_len,
+				.queue_num = j,
+				.key = rss_key,
+				.queue = queue,
+			};
+			ret = rte_flow_validate(sa->portid, &sa->attr,
+						sa->pattern, sa->action,
+						&err);
+			if (!ret)
+				goto flow_create;
+			/* Try Queue. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_QUEUE;
+			sa->action[1].conf =
+				&(struct rte_flow_action_queue){
+				.index = 0,
+			};
+			ret = rte_flow_validate(sa->portid, &sa->attr,
+						sa->pattern, sa->action,
+						&err);
+			/* Try End. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
+			sa->action[1].conf = NULL;
+			ret = rte_flow_validate(sa->portid, &sa->attr,
 							sa->pattern, sa->action,
 							&err);
-				if (ret)
-					goto flow_create_failure;
-			} else if (sa->attr.egress &&
-				   (sa->ol_flags &
-				    RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
-				sa->action[1].type =
-					RTE_FLOW_ACTION_TYPE_PASSTHRU;
-				sa->action[2].type =
-					RTE_FLOW_ACTION_TYPE_END;
-			}
+			if (ret)
+				goto flow_create_failure;
+		} else if (sa->attr.egress &&
+				(sa->ol_flags &
+				RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
+			sa->action[1].type =
+				RTE_FLOW_ACTION_TYPE_PASSTHRU;
+			sa->action[2].type =
+				RTE_FLOW_ACTION_TYPE_END;
+		}
 flow_create:
-			sa->flow = rte_flow_create(sa->portid,
-				&sa->attr, sa->pattern, sa->action, &err);
-			if (sa->flow == NULL) {
+		sa->flow = rte_flow_create(sa->portid,
+			&sa->attr, sa->pattern, sa->action, &err);
+		if (sa->flow == NULL) {
 flow_create_failure:
-				RTE_LOG(ERR, IPSEC,
-					"Failed to create ipsec flow msg: %s\n",
-					err.message);
-				return -1;
-			}
-		} else if (sa->type ==
-				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
-			struct rte_security_ctx *ctx =
-					(struct rte_security_ctx *)
-					rte_eth_dev_get_sec_ctx(sa->portid);
-			const struct rte_security_capability *sec_cap;
-
-			if (ctx == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"Ethernet device doesn't have security features registered\n");
-				return -1;
-			}
+			RTE_LOG(ERR, IPSEC,
+				"Failed to create ipsec flow msg: %s\n",
+				err.message);
+			return -1;
+		}
+	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
+		struct rte_security_ctx *ctx =
+				(struct rte_security_ctx *)
+				rte_eth_dev_get_sec_ctx(sa->portid);
+		const struct rte_security_capability *sec_cap;
 
-			/* Set IPsec parameters in conf */
-			set_ipsec_conf(sa, &(sess_conf.ipsec));
-
-			/* Save SA as userdata for the security session. When
-			 * the packet is received, this userdata will be
-			 * retrieved using the metadata from the packet.
-			 *
-			 * The PMD is expected to set similar metadata for other
-			 * operations, like rte_eth_event, which are tied to
-			 * security session. In such cases, the userdata could
-			 * be obtained to uniquely identify the security
-			 * parameters denoted.
-			 */
-
-			sess_conf.userdata = (void *) sa;
-
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"SEC Session init failed: err: %d\n", ret);
-				return -1;
-			}
+		if (ctx == NULL) {
+			RTE_LOG(ERR, IPSEC,
+			"Ethernet device doesn't have security features registered\n");
+			return -1;
+		}
+
+		/* Set IPsec parameters in conf */
+		set_ipsec_conf(sa, &(sess_conf.ipsec));
+
+		/* Save SA as userdata for the security session. When
+		 * the packet is received, this userdata will be
+		 * retrieved using the metadata from the packet.
+		 *
+		 * The PMD is expected to set similar metadata for other
+		 * operations, like rte_eth_event, which are tied to
+		 * security session. In such cases, the userdata could
+		 * be obtained to uniquely identify the security
+		 * parameters denoted.
+		 */
+
+		sess_conf.userdata = (void *) sa;
+
+		sa->sec_session = rte_security_session_create(ctx,
+			&sess_conf, pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC,
+			"SEC Session init failed: err: %d\n", ret);
+			return -1;
+		}
 
-			sec_cap = rte_security_capabilities_get(ctx);
+		sec_cap = rte_security_capabilities_get(ctx);
 
-			if (sec_cap == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"No capabilities registered\n");
-				return -1;
-			}
+		if (sec_cap == NULL) {
+			RTE_LOG(ERR, IPSEC,
+			"No capabilities registered\n");
+			return -1;
+		}
 
-			/* iterate until ESP tunnel*/
-			while (sec_cap->action !=
+		/* iterate until ESP tunnel*/
+		while (sec_cap->action !=
 					RTE_SECURITY_ACTION_TYPE_NONE) {
 
-				if (sec_cap->action == sa->type &&
-				    sec_cap->protocol ==
-					RTE_SECURITY_PROTOCOL_IPSEC &&
-				    sec_cap->ipsec.mode ==
-					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
-				    sec_cap->ipsec.direction == sa->direction)
-					break;
-				sec_cap++;
-			}
+			if (sec_cap->action == sa->type &&
+				sec_cap->protocol ==
+				RTE_SECURITY_PROTOCOL_IPSEC &&
+				sec_cap->ipsec.mode ==
+				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
+				sec_cap->ipsec.direction == sa->direction)
+				break;
+			sec_cap++;
+		}
 
-			if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
-				RTE_LOG(ERR, IPSEC,
-				"No suitable security capability found\n");
-				return -1;
-			}
+		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
+			RTE_LOG(ERR, IPSEC,
+			"No suitable security capability found\n");
+			return -1;
+		}
+
+		sa->ol_flags = sec_cap->ol_flags;
+		sa->security_ctx = ctx;
+	} else
+		return -EINVAL;
+	return 0;
+}
+
+#define CDEV_IV_SIZE 12
+
+static int
+check_cryptodev_aead_capablity(const struct ipsec_sa *ss, uint8_t dev_id)
+{
+	struct rte_cryptodev_sym_capability_idx cap_idx;
+	const struct rte_cryptodev_symmetric_capability *cap;
 
-			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
+	cap_idx.type = RTE_CRYPTO_SYM_XFORM_AEAD;
+	cap_idx.algo.aead = ss->aead_algo;
+
+	cap = rte_cryptodev_sym_capability_get(dev_id, &cap_idx);
+	if (cap == NULL)
+		return -ENOENT;
+
+	return rte_cryptodev_sym_capability_check_aead(cap,
+					ss->cipher_key_len,
+					ss->digest_len,
+					ss->aad_len,
+					CDEV_IV_SIZE);
+}
+
+static int
+check_cryptodev_capablity(const struct ipsec_sa *ss, uint8_t dev_id)
+{
+	struct rte_cryptodev_sym_capability_idx cap_idx;
+	const struct rte_cryptodev_symmetric_capability *cap;
+	uint16_t auth_iv_len;
+	int rc = -1;
+
+	if (ss == NULL)
+		return rc;
+
+	if (ss->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
+		return check_cryptodev_aead_capablity(ss, dev_id);
+
+	auth_iv_len = 0;
+
+	cap_idx.type = RTE_CRYPTO_SYM_XFORM_AUTH;
+	cap_idx.algo.auth = ss->auth_algo;
+	cap = rte_cryptodev_sym_capability_get(dev_id, &cap_idx);
+	if (cap != NULL) {
+		rc = rte_cryptodev_sym_capability_check_auth(
+				cap, ss->auth_key_len, ss->digest_len,
+				auth_iv_len);
+		if (rc == 0) {
+			cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
+			cap_idx.algo.cipher = ss->cipher_algo;
+			cap = rte_cryptodev_sym_capability_get(dev_id,
+					&cap_idx);
+			if (cap != NULL)
+				rc = rte_cryptodev_sym_capability_check_cipher(
+						cap, ss->cipher_key_len,
+						ss->iv_len);
 		}
-	} else {
-		sa->crypto_session = rte_cryptodev_sym_session_create(
-				ipsec_ctx->session_pool);
-		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
-				sa->crypto_session, sa->xforms,
-				ipsec_ctx->session_priv_pool);
-
-		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
-				&cdev_info);
 	}
-	sa->cdev_id_qp = cdev_id_qp;
 
-	return 0;
+	return rc;
+}
+
+static int
+create_crypto_session(struct ipsec_sa *sa, struct rte_mempool *pool)
+{
+	int32_t rc;
+	uint32_t devnum, i;
+	struct rte_cryptodev_sym_session *s;
+	uint8_t devid[RTE_CRYPTO_MAX_DEVS];
+
+	/* check which cryptodevs support SA */
+	devnum = 0;
+	for (i = 0; i < crypto_dev_num; i++) {
+		rc = check_cryptodev_capablity(sa, crypto_devid[i]);
+		if (rc == 0)
+			devid[devnum++] = crypto_devid[i];
+	}
+
+	if (devnum == 0)
+		return -ENODEV;
+
+	s = rte_cryptodev_sym_session_create(pool);
+	if (s == NULL)
+		return -ENOMEM;
+
+	/* initialize SA crypto session for all supported devices */
+	for (i = 0; i != devnum; i++) {
+		rc = rte_cryptodev_sym_session_init(devid[i], s, sa->xforms,
+			pool);
+		if (rc != 0)
+			break;
+	}
+
+	if (i == devnum) {
+		sa->crypto_session = s;
+		return 0;
+	}
+
+	/* failure, do cleanup */
+	while (i-- != 0)
+		rte_cryptodev_sym_session_clear(devid[i], s);
+
+	rte_cryptodev_sym_session_free(s);
+	return rc;
+}
+
+int
+create_session(struct ipsec_sa *sa, struct rte_mempool *pool)
+{
+	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE)
+		return create_sec_session(sa, pool);
+	else
+		return create_crypto_session(sa, pool);
 }
 
 /*
@@ -393,13 +461,6 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			sym_cop = get_sym_cop(&priv->cop);
 			sym_cop->m_src = pkts[i];
 
@@ -412,13 +473,6 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->crypto_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			rte_crypto_op_attach_sym_session(&priv->cop,
 					sa->crypto_session);
 
@@ -429,12 +483,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 			break;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
+			RTE_ASSERT(sa->sec_session != NULL);
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
 			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
@@ -442,17 +491,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 						sa->sec_session, pkts[i], NULL);
 			continue;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
+			RTE_ASSERT(sa->sec_session != NULL);
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			rte_security_attach_session(&priv->cop,
 					sa->sec_session);
 
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index 99f49d6..804330c 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -83,6 +83,14 @@ struct app_sa_prm {
 
 extern struct app_sa_prm app_sa_prm;
 
+/*
+ * Number of enabled crypto devices
+ * This number is needed when checking crypto device capabilities
+ */
+extern uint8_t crypto_dev_num;
+/* array of crypto device ID's */
+extern uint8_t crypto_devid[RTE_CRYPTO_MAX_DEVS];
+
 struct ipsec_sa {
 	struct rte_ipsec_session ips; /* one session per sa for now */
 	uint32_t spi;
@@ -306,6 +314,6 @@ void
 enqueue_cop_burst(struct cdev_qp *cqp);
 
 int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+create_session(struct ipsec_sa *sa, struct rte_mempool *pool);
 
 #endif /* __IPSEC_H__ */
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index 3f9cacb..0df6969 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -86,39 +86,6 @@ enqueue_cop_bulk(struct cdev_qp *cqp, struct rte_crypto_op *cop[], uint32_t num)
 	cqp->len = len;
 }
 
-static inline int
-fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
-	struct ipsec_sa *sa)
-{
-	int32_t rc;
-
-	/* setup crypto section */
-	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		if (sa->crypto_session == NULL) {
-			rc = create_session(ctx, sa);
-			if (rc != 0)
-				return rc;
-		}
-		ss->crypto.ses = sa->crypto_session;
-	/* setup session action type */
-	} else {
-		if (sa->sec_session == NULL) {
-			rc = create_session(ctx, sa);
-			if (rc != 0)
-				return rc;
-		}
-		ss->security.ses = sa->sec_session;
-		ss->security.ctx = sa->security_ctx;
-		ss->security.ol_flags = sa->ol_flags;
-	}
-
-	rc = rte_ipsec_session_prepare(ss);
-	if (rc != 0)
-		memset(ss, 0, sizeof(*ss));
-
-	return rc;
-}
-
 /*
  * group input packets byt the SA they belong to.
  */
@@ -219,9 +186,8 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 
 		ips = &sa->ips;
 
-		/* no valid HW session for that SA, try to create one */
-		if (sa == NULL || (ips->crypto.ses == NULL &&
-				fill_ipsec_session(ips, ctx, sa) != 0))
+		/* no valid HW session for that SA */
+		if (sa == NULL || ips->crypto.ses == NULL)
 			k = 0;
 
 		/* process packets inline */
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index a7298a3..0f36f5b 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -774,14 +774,14 @@ check_eth_dev_caps(uint16_t portid, uint32_t inbound)
 	return 0;
 }
 
-
 static int
 sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries, uint32_t inbound)
+	uint32_t nb_entries, uint32_t inbound, struct socket_ctx *skt_ctx)
 {
 	struct ipsec_sa *sa;
 	uint32_t i, idx;
 	uint16_t iv_length, aad_length;
+	int32_t rc;
 
 	/* for ESN upper 32 bits of SQN also need to be part of AAD */
 	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
@@ -902,6 +902,12 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			print_one_sa_rule(sa, inbound);
 		}
+		rc = create_session(sa, skt_ctx->session_pool);
+		if (rc != 0) {
+			RTE_LOG(ERR, IPSEC_ESP,
+					"create_session() failed\n");
+			return -EINVAL;
+		}
 	}
 
 	return 0;
@@ -909,16 +915,16 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 static inline int
 sa_out_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 0);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx);
 }
 
 static inline int
 sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 1);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx);
 }
 
 /*
@@ -1012,10 +1018,12 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm, const struct ipsec_sa *ss,
 	return 0;
 }
 
-static void
+static int
 fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 	const struct ipsec_sa *lsa)
 {
+	int32_t rc = 0;
+
 	ss->sa = sa;
 	ss->type = lsa->type;
 
@@ -1028,6 +1036,12 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 		ss->security.ctx = lsa->security_ctx;
 		ss->security.ol_flags = lsa->ol_flags;
 	}
+
+	rc = rte_ipsec_session_prepare(ss);
+	if (rc != 0)
+		memset(ss, 0, sizeof(*ss));
+
+	return rc;
 }
 
 /*
@@ -1062,8 +1076,8 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	fill_ipsec_session(&lsa->ips, sa, lsa);
-	return 0;
+	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
+	return rc;
 }
 
 /*
@@ -1141,7 +1155,10 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in);
+		rc = sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx);
+		if (rc != 0)
+			rte_exit(EXIT_FAILURE,
+				"failed to add inbound rules\n");
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_in, sa_in, nb_sa_in,
@@ -1161,7 +1178,10 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out);
+		rc = sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx);
+		if (rc != 0)
+			rte_exit(EXIT_FAILURE,
+				"failed to add outbound rules\n");
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_out, sa_out, nb_sa_out,
-- 
2.7.4

[dpdk-dev] [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Bernard Iremonger]

Inline crypto installs a flow rule in the NIC. This flow
rule must be installed before the first inbound packet is
received.

The create_session() function installs the flow rule.

Refactor ipsec-secgw.c, sa.c, ipsec.h and ipsec.c to create
sessions at startup to fix the issue of the first packet
being dropped for inline crypto.

The create_session() function is now called at initialisation in
sa_add_rules() which is called from sa_init().
The return code for add_rules is checked.
Calls to create_session() in other functions are dropped.

Add crypto_devid_fill() in ipsec-secgw.c
Add max_session_size() in ipsec-secgw.c
Add check_cryptodev_capability() in ipsec.c
Add check_cryptodev_aead_capability() in ipsec.c
Add create_sec_session() and create_crypto_session() in ipsec.c

The crypto_dev_fill() function has been added to find the
enabled crypto devices.

The max_session_size() function has been added to calculate memory
requirements.

The check_cryptodev_capability() and check_cryptodev_aead_capability()
functions have been added to check that the SA is supported by the
crypto device.

The create_session() function is refactored to use the
create_sec_session() and create_crypto_session() functions.

The cryprodev_init() function has been refactored to drop calls to
rte_mempool_create() and to drop calculation of memory requirements.

The main() function has been refactored to call crypto_devid_fill()
and max_session_size() and to call session_pool_init() and
session_priv_pool_init().
The ports are started now before adding a flow rule in main().
The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
now called after the ports have been started.

Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
---
 examples/ipsec-secgw/ipsec-secgw.c   | 271 ++++++++--------
 examples/ipsec-secgw/ipsec.c         | 581 +++++++++++++++++++----------------
 examples/ipsec-secgw/ipsec.h         |  10 +-
 examples/ipsec-secgw/ipsec_process.c |  38 +--
 examples/ipsec-secgw/sa.c            |  42 ++-
 5 files changed, 501 insertions(+), 441 deletions(-)

diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index ffbd00b..cc8bb57 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -182,6 +182,14 @@ struct lcore_params {
 	uint8_t lcore_id;
 } __rte_cache_aligned;
 
+/*
+ * Number of enabled crypto devices
+ * This number is needed when checking crypto device capabilities
+ */
+uint8_t crypto_dev_num;
+/* array of crypto device ID's */
+uint8_t crypto_devid[RTE_CRYPTO_MAX_DEVS];
+
 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS];
 
 static struct lcore_params *lcore_params;
@@ -1623,13 +1631,27 @@ check_cryptodev_mask(uint8_t cdev_id)
 	return -1;
 }
 
+static void
+crypto_devid_fill(void)
+{
+	uint32_t i, n;
+
+	n = rte_cryptodev_count();
+
+	for (i = 0; i != n; i++) {
+		if (check_cryptodev_mask(i) == 0)
+			crypto_devid[crypto_dev_num++] = i;
+	}
+}
+
 static int32_t
 cryptodevs_init(void)
 {
 	struct rte_cryptodev_config dev_conf;
 	struct rte_cryptodev_qp_conf qp_conf;
 	uint16_t idx, max_nb_qps, qp, i;
-	int16_t cdev_id, port_id;
+	int16_t cdev_id;
+	uint32_t dev_max_sess;
 	struct rte_hash_parameters params = { 0 };
 
 	params.entries = CDEV_MAP_ENTRIES;
@@ -1652,45 +1674,6 @@ cryptodevs_init(void)
 
 	printf("lcore/cryptodev/qp mappings:\n");
 
-	uint32_t max_sess_sz = 0, sess_sz;
-	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
-		void *sec_ctx;
-
-		/* Get crypto priv session size */
-		sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-
-		/*
-		 * If crypto device is security capable, need to check the
-		 * size of security session as well.
-		 */
-
-		/* Get security context of the crypto device */
-		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		/* Get size of security session */
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-	RTE_ETH_FOREACH_DEV(port_id) {
-		void *sec_ctx;
-
-		if ((enabled_port_mask & (1 << port_id)) == 0)
-			continue;
-
-		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-
 	idx = 0;
 	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
 		struct rte_cryptodev_info cdev_info;
@@ -1722,51 +1705,12 @@ cryptodevs_init(void)
 		dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id);
 		dev_conf.nb_queue_pairs = qp;
 
-		uint32_t dev_max_sess = cdev_info.sym.max_nb_sessions;
+		dev_max_sess = cdev_info.sym.max_nb_sessions;
 		if (dev_max_sess != 0 && dev_max_sess < CDEV_MP_NB_OBJS)
 			rte_exit(EXIT_FAILURE,
 				"Device does not support at least %u "
 				"sessions", CDEV_MP_NB_OBJS);
 
-		if (!socket_ctx[dev_conf.socket_id].session_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_%u", dev_conf.socket_id);
-			sess_mp = rte_cryptodev_sym_session_pool_create(
-					mp_name, CDEV_MP_NB_OBJS,
-					0, CDEV_MP_CACHE_SZ, 0,
-					dev_conf.socket_id);
-			socket_ctx[dev_conf.socket_id].session_pool = sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_priv_%u", dev_conf.socket_id);
-			sess_mp = rte_mempool_create(mp_name,
-					CDEV_MP_NB_OBJS,
-					max_sess_sz,
-					CDEV_MP_CACHE_SZ,
-					0, NULL, NULL, NULL,
-					NULL, dev_conf.socket_id,
-					0);
-			socket_ctx[dev_conf.socket_id].session_priv_pool =
-					sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool ||
-				!socket_ctx[dev_conf.socket_id].session_pool)
-			rte_exit(EXIT_FAILURE,
-				"Cannot create session pool on socket %d\n",
-				dev_conf.socket_id);
-		else
-			printf("Allocated session pool on socket %d\n",
-					dev_conf.socket_id);
-
 		if (rte_cryptodev_configure(cdev_id, &dev_conf))
 			rte_panic("Failed to initialize cryptodev %u\n",
 					cdev_id);
@@ -1787,38 +1731,6 @@ cryptodevs_init(void)
 					cdev_id);
 	}
 
-	/* create session pools for eth devices that implement security */
-	RTE_ETH_FOREACH_DEV(port_id) {
-		if ((enabled_port_mask & (1 << port_id)) &&
-				rte_eth_dev_get_sec_ctx(port_id)) {
-			int socket_id = rte_eth_dev_socket_id(port_id);
-
-			if (!socket_ctx[socket_id].session_pool) {
-				char mp_name[RTE_MEMPOOL_NAMESIZE];
-				struct rte_mempool *sess_mp;
-
-				snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-						"sess_mp_%u", socket_id);
-				sess_mp = rte_mempool_create(mp_name,
-						(CDEV_MP_NB_OBJS * 2),
-						max_sess_sz,
-						CDEV_MP_CACHE_SZ,
-						0, NULL, NULL, NULL,
-						NULL, socket_id,
-						0);
-				if (sess_mp == NULL)
-					rte_exit(EXIT_FAILURE,
-						"Cannot create session pool "
-						"on socket %d\n", socket_id);
-				else
-					printf("Allocated session pool "
-						"on socket %d\n", socket_id);
-				socket_ctx[socket_id].session_pool = sess_mp;
-			}
-		}
-	}
-
-
 	printf("\n");
 
 	return 0;
@@ -1984,6 +1896,98 @@ port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads)
 	printf("\n");
 }
 
+static size_t
+max_session_size(void)
+{
+	size_t max_sz, sz;
+	void *sec_ctx;
+	int16_t cdev_id, port_id, n;
+
+	max_sz = 0;
+	n =  rte_cryptodev_count();
+	for (cdev_id = 0; cdev_id != n; cdev_id++) {
+		sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
+		if (sz > max_sz)
+			max_sz = sz;
+		/*
+		 * If crypto device is security capable, need to check the
+		 * size of security session as well.
+		 */
+
+		/* Get security context of the crypto device */
+		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		/* Get size of security session */
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	RTE_ETH_FOREACH_DEV(port_id) {
+		if ((enabled_port_mask & (1 << port_id)) == 0)
+			continue;
+
+		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	return max_sz;
+}
+
+static void
+session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_%u", socket_id);
+	sess_mp = rte_cryptodev_sym_session_pool_create(
+			mp_name, CDEV_MP_NB_OBJS,
+			sess_sz, CDEV_MP_CACHE_SZ, 0,
+			socket_id);
+	ctx->session_pool = sess_mp;
+
+	if (ctx->session_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session pool on socket %d\n", socket_id);
+	else
+		printf("Allocated session pool on socket %d\n",	socket_id);
+}
+
+static void
+session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
+	size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_priv_%u", socket_id);
+	sess_mp = rte_mempool_create(mp_name,
+			CDEV_MP_NB_OBJS,
+			sess_sz,
+			CDEV_MP_CACHE_SZ,
+			0, NULL, NULL, NULL,
+			NULL, socket_id,
+			0);
+	ctx->session_priv_pool = sess_mp;
+	if (ctx->session_priv_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session priv pool on socket %d\n",
+			socket_id);
+	else
+		printf("Allocated session priv pool on socket %d\n",
+			socket_id);
+}
+
 static void
 pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf)
 {
@@ -2064,9 +2068,11 @@ main(int32_t argc, char **argv)
 {
 	int32_t ret;
 	uint32_t lcore_id;
+	uint32_t i;
 	uint8_t socket_id;
 	uint16_t portid;
 	uint64_t req_rx_offloads, req_tx_offloads;
+	size_t sess_sz;
 
 	/* init EAL */
 	ret = rte_eal_init(argc, argv);
@@ -2094,7 +2100,10 @@ main(int32_t argc, char **argv)
 
 	nb_lcores = rte_lcore_count();
 
-	/* Replicate each context per socket */
+	crypto_devid_fill();
+
+	sess_sz = max_session_size();
+
 	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
 		if (rte_lcore_is_enabled(lcore_id) == 0)
 			continue;
@@ -2104,20 +2113,17 @@ main(int32_t argc, char **argv)
 		else
 			socket_id = 0;
 
+		/* mbuf_pool is initialised by the pool_init() function*/
 		if (socket_ctx[socket_id].mbuf_pool)
 			continue;
 
-		/* initilaze SPD */
-		sp4_init(&socket_ctx[socket_id], socket_id);
-
-		sp6_init(&socket_ctx[socket_id], socket_id);
-
-		/* initilaze SAD */
-		sa_init(&socket_ctx[socket_id], socket_id);
-
-		rt_init(&socket_ctx[socket_id], socket_id);
-
 		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
+		session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
+		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
+			sess_sz);
+
+		if (!numa_on)
+			break;
 	}
 
 	RTE_ETH_FOREACH_DEV(portid) {
@@ -2135,7 +2141,11 @@ main(int32_t argc, char **argv)
 		if ((enabled_port_mask & (1 << portid)) == 0)
 			continue;
 
-		/* Start device */
+		/*
+		 * Start device
+		 * note: device must be started before a flow rule
+		 * can be installed.
+		 */
 		ret = rte_eth_dev_start(portid);
 		if (ret < 0)
 			rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
@@ -2153,6 +2163,19 @@ main(int32_t argc, char **argv)
 			RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback, NULL);
 	}
 
+	/* Replicate each context per socket */
+	for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
+		socket_id = rte_socket_id_by_idx(i);
+		if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
+			(socket_ctx[socket_id].sa_in == NULL) &&
+			(socket_ctx[socket_id].sa_out == NULL)) {
+			sa_init(&socket_ctx[socket_id], socket_id);
+			sp4_init(&socket_ctx[socket_id], socket_id);
+			sp6_init(&socket_ctx[socket_id], socket_id);
+			rt_init(&socket_ctx[socket_id], socket_id);
+		}
+	}
+
 	check_all_ports_link_status(enabled_port_mask);
 
 	/* launch per-lcore init on every lcore */
diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index 4352cb8..83080f7 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -39,42 +39,17 @@ set_ipsec_conf(struct ipsec_sa *sa, struct rte_security_ipsec_xform *ipsec)
 	ipsec->esn_soft_limit = IPSEC_OFFLOAD_ESN_SOFTLIMIT;
 }
 
-int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
+static int
+create_sec_session(struct ipsec_sa *sa, struct rte_mempool *pool)
 {
-	struct rte_cryptodev_info cdev_info;
-	unsigned long cdev_id_qp = 0;
+	const struct rte_security_capability *sec_cap;
+	struct rte_security_ctx *ctx;
 	int32_t ret = 0;
-	struct cdev_key key = { 0 };
-
-	key.lcore_id = (uint8_t)rte_lcore_id();
-
-	key.cipher_algo = (uint8_t)sa->cipher_algo;
-	key.auth_algo = (uint8_t)sa->auth_algo;
-	key.aead_algo = (uint8_t)sa->aead_algo;
-
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
-				(void **)&cdev_id_qp);
-		if (ret < 0) {
-			RTE_LOG(ERR, IPSEC,
-				"No cryptodev: core %u, cipher_algo %u, "
-				"auth_algo %u, aead_algo %u\n",
-				key.lcore_id,
-				key.cipher_algo,
-				key.auth_algo,
-				key.aead_algo);
-			return -1;
-		}
-	}
 
-	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on cryptodev "
-			"%u qp %u\n", sa->spi,
-			ipsec_ctx->tbl[cdev_id_qp].id,
-			ipsec_ctx->tbl[cdev_id_qp].qp);
+	if ((sa == NULL) || (pool == NULL))
+		return -EINVAL;
 
-	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
-		struct rte_security_session_conf sess_conf = {
+	struct rte_security_session_conf sess_conf = {
 			.action_type = sa->type,
 			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 			{.ipsec = {
@@ -90,247 +65,340 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			} },
 			.crypto_xform = sa->xforms,
 			.userdata = NULL,
-
 		};
 
-		if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
-			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
-							rte_cryptodev_get_sec_ctx(
-							ipsec_ctx->tbl[cdev_id_qp].id);
-
-			/* Set IPsec parameters in conf */
-			set_ipsec_conf(sa, &(sess_conf.ipsec));
-
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"SEC Session init failed: err: %d\n", ret);
-				return -1;
-			}
-		} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
-			struct rte_flow_error err;
-			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
-							rte_eth_dev_get_sec_ctx(
-							sa->portid);
-			const struct rte_security_capability *sec_cap;
-			int ret = 0;
-
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"SEC Session init failed: err: %d\n", ret);
-				return -1;
-			}
+	if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
+		ctx = (struct rte_security_ctx *)
+				rte_eth_dev_get_sec_ctx(sa->portid);
 
-			sec_cap = rte_security_capabilities_get(ctx);
+		/* Set IPsec parameters in conf */
+		set_ipsec_conf(sa, &(sess_conf.ipsec));
 
-			/* iterate until ESP tunnel*/
-			while (sec_cap->action !=
-					RTE_SECURITY_ACTION_TYPE_NONE) {
+		sa->sec_session = rte_security_session_create(ctx,
+				&sess_conf, pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC,
+				"SEC Session init failed: err: %d\n",
+				ret);
+			return -1;
+		}
+	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		struct rte_flow_error err;
+		ctx = (struct rte_security_ctx *)
+				rte_eth_dev_get_sec_ctx(sa->portid);
+		sa->sec_session = rte_security_session_create(ctx,
+				&sess_conf, pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC, "SEC Session init failed\n");
+			return -1;
+		}
 
-				if (sec_cap->action == sa->type &&
-				    sec_cap->protocol ==
-					RTE_SECURITY_PROTOCOL_IPSEC &&
-				    sec_cap->ipsec.mode ==
-					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
-				    sec_cap->ipsec.direction == sa->direction)
-					break;
-				sec_cap++;
-			}
+		sec_cap = rte_security_capabilities_get(ctx);
+
+		/* iterate until ESP tunnel*/
+		while (sec_cap->action != RTE_SECURITY_ACTION_TYPE_NONE) {
+			if (sec_cap->action == sa->type &&
+				sec_cap->protocol ==
+				RTE_SECURITY_PROTOCOL_IPSEC &&
+				sec_cap->ipsec.mode ==
+				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
+				sec_cap->ipsec.direction == sa->direction)
+				break;
+			sec_cap++;
+		}
 
-			if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
-				RTE_LOG(ERR, IPSEC,
+		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
+			RTE_LOG(ERR, IPSEC,
 				"No suitable security capability found\n");
 				return -1;
-			}
+		}
 
-			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
-			sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
-
-			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
-			sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
-			if (sa->flags & IP6_TUNNEL) {
-				sa->pattern[1].spec = &sa->ipv6_spec;
-				memcpy(sa->ipv6_spec.hdr.dst_addr,
-					sa->dst.ip.ip6.ip6_b, 16);
-				memcpy(sa->ipv6_spec.hdr.src_addr,
-				       sa->src.ip.ip6.ip6_b, 16);
-			} else {
-				sa->pattern[1].spec = &sa->ipv4_spec;
-				sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
-				sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
-			}
+		sa->ol_flags = sec_cap->ol_flags;
+		sa->security_ctx = ctx;
+		sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
+
+		sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
+		sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
+		if (sa->flags & IP6_TUNNEL) {
+			sa->pattern[1].spec = &sa->ipv6_spec;
+			memcpy(sa->ipv6_spec.hdr.dst_addr,
+				sa->dst.ip.ip6.ip6_b, 16);
+			memcpy(sa->ipv6_spec.hdr.src_addr,
+			       sa->src.ip.ip6.ip6_b, 16);
+		} else {
+			sa->pattern[1].spec = &sa->ipv4_spec;
+			sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
+			sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
+		}
 
-			sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
-			sa->pattern[2].spec = &sa->esp_spec;
-			sa->pattern[2].mask = &rte_flow_item_esp_mask;
-			sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
+		sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
+		sa->pattern[2].spec = &sa->esp_spec;
+		sa->pattern[2].mask = &rte_flow_item_esp_mask;
+		sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
 
-			sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
+		sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
 
-			sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
-			sa->action[0].conf = sa->sec_session;
+		sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
+		sa->action[0].conf = sa->sec_session;
 
-			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
+		sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
 
-			sa->attr.egress = (sa->direction ==
+		sa->attr.egress = (sa->direction ==
 					RTE_SECURITY_IPSEC_SA_DIR_EGRESS);
-			sa->attr.ingress = (sa->direction ==
+		sa->attr.ingress = (sa->direction ==
 					RTE_SECURITY_IPSEC_SA_DIR_INGRESS);
-			if (sa->attr.ingress) {
-				uint8_t rss_key[40];
-				struct rte_eth_rss_conf rss_conf = {
-					.rss_key = rss_key,
-					.rss_key_len = 40,
-				};
-				struct rte_eth_dev *eth_dev;
-				uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
-				struct rte_flow_action_rss action_rss;
-				unsigned int i;
-				unsigned int j;
-
-				sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
-				/* Try RSS. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
-				sa->action[1].conf = &action_rss;
-				eth_dev = ctx->device;
-				rte_eth_dev_rss_hash_conf_get(sa->portid,
-							      &rss_conf);
-				for (i = 0, j = 0;
-				     i < eth_dev->data->nb_rx_queues; ++i)
-					if (eth_dev->data->rx_queues[i])
-						queue[j++] = i;
-				action_rss = (struct rte_flow_action_rss){
-					.types = rss_conf.rss_hf,
-					.key_len = rss_conf.rss_key_len,
-					.queue_num = j,
-					.key = rss_key,
-					.queue = queue,
-				};
-				ret = rte_flow_validate(sa->portid, &sa->attr,
-							sa->pattern, sa->action,
-							&err);
-				if (!ret)
-					goto flow_create;
-				/* Try Queue. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_QUEUE;
-				sa->action[1].conf =
-					&(struct rte_flow_action_queue){
-					.index = 0,
-				};
-				ret = rte_flow_validate(sa->portid, &sa->attr,
-							sa->pattern, sa->action,
-							&err);
-				/* Try End. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
-				sa->action[1].conf = NULL;
-				ret = rte_flow_validate(sa->portid, &sa->attr,
+		if (sa->attr.ingress) {
+			uint8_t rss_key[40];
+			struct rte_eth_rss_conf rss_conf = {
+				.rss_key = rss_key,
+				.rss_key_len = 40,
+			};
+			struct rte_eth_dev *eth_dev;
+			uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
+			struct rte_flow_action_rss action_rss;
+			unsigned int i;
+			unsigned int j;
+
+			sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
+			/* Try RSS. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
+			sa->action[1].conf = &action_rss;
+			eth_dev = ctx->device;
+			rte_eth_dev_rss_hash_conf_get(sa->portid,
+						&rss_conf);
+			for (i = 0, j = 0;
+			     i < eth_dev->data->nb_rx_queues; ++i)
+				if (eth_dev->data->rx_queues[i])
+					queue[j++] = i;
+			action_rss = (struct rte_flow_action_rss){
+				.types = rss_conf.rss_hf,
+				.key_len = rss_conf.rss_key_len,
+				.queue_num = j,
+				.key = rss_key,
+				.queue = queue,
+			};
+			ret = rte_flow_validate(sa->portid, &sa->attr,
+						sa->pattern, sa->action,
+						&err);
+			if (!ret)
+				goto flow_create;
+			/* Try Queue. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_QUEUE;
+			sa->action[1].conf =
+				&(struct rte_flow_action_queue){
+				.index = 0,
+			};
+			ret = rte_flow_validate(sa->portid, &sa->attr,
+						sa->pattern, sa->action,
+						&err);
+			/* Try End. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
+			sa->action[1].conf = NULL;
+			ret = rte_flow_validate(sa->portid, &sa->attr,
 							sa->pattern, sa->action,
 							&err);
-				if (ret)
-					goto flow_create_failure;
-			} else if (sa->attr.egress &&
-				   (sa->ol_flags &
-				    RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
-				sa->action[1].type =
-					RTE_FLOW_ACTION_TYPE_PASSTHRU;
-				sa->action[2].type =
-					RTE_FLOW_ACTION_TYPE_END;
-			}
+			if (ret)
+				goto flow_create_failure;
+		} else if (sa->attr.egress &&
+				(sa->ol_flags &
+				RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
+			sa->action[1].type =
+				RTE_FLOW_ACTION_TYPE_PASSTHRU;
+			sa->action[2].type =
+				RTE_FLOW_ACTION_TYPE_END;
+		}
 flow_create:
-			sa->flow = rte_flow_create(sa->portid,
-				&sa->attr, sa->pattern, sa->action, &err);
-			if (sa->flow == NULL) {
+		sa->flow = rte_flow_create(sa->portid,
+			&sa->attr, sa->pattern, sa->action, &err);
+		if (sa->flow == NULL) {
 flow_create_failure:
-				RTE_LOG(ERR, IPSEC,
-					"Failed to create ipsec flow msg: %s\n",
-					err.message);
-				return -1;
-			}
-		} else if (sa->type ==
-				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
-			struct rte_security_ctx *ctx =
-					(struct rte_security_ctx *)
-					rte_eth_dev_get_sec_ctx(sa->portid);
-			const struct rte_security_capability *sec_cap;
-
-			if (ctx == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"Ethernet device doesn't have security features registered\n");
-				return -1;
-			}
+			RTE_LOG(ERR, IPSEC,
+				"Failed to create ipsec flow msg: %s\n",
+				err.message);
+			return -1;
+		}
+	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
+		struct rte_security_ctx *ctx =
+				(struct rte_security_ctx *)
+				rte_eth_dev_get_sec_ctx(sa->portid);
+		const struct rte_security_capability *sec_cap;
 
-			/* Set IPsec parameters in conf */
-			set_ipsec_conf(sa, &(sess_conf.ipsec));
-
-			/* Save SA as userdata for the security session. When
-			 * the packet is received, this userdata will be
-			 * retrieved using the metadata from the packet.
-			 *
-			 * The PMD is expected to set similar metadata for other
-			 * operations, like rte_eth_event, which are tied to
-			 * security session. In such cases, the userdata could
-			 * be obtained to uniquely identify the security
-			 * parameters denoted.
-			 */
-
-			sess_conf.userdata = (void *) sa;
-
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"SEC Session init failed: err: %d\n", ret);
-				return -1;
-			}
+		if (ctx == NULL) {
+			RTE_LOG(ERR, IPSEC,
+			"Ethernet device doesn't have security features registered\n");
+			return -1;
+		}
+
+		/* Set IPsec parameters in conf */
+		set_ipsec_conf(sa, &(sess_conf.ipsec));
+
+		/* Save SA as userdata for the security session. When
+		 * the packet is received, this userdata will be
+		 * retrieved using the metadata from the packet.
+		 *
+		 * The PMD is expected to set similar metadata for other
+		 * operations, like rte_eth_event, which are tied to
+		 * security session. In such cases, the userdata could
+		 * be obtained to uniquely identify the security
+		 * parameters denoted.
+		 */
+
+		sess_conf.userdata = (void *) sa;
+
+		sa->sec_session = rte_security_session_create(ctx,
+			&sess_conf, pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC,
+			"SEC Session init failed: err: %d\n", ret);
+			return -1;
+		}
 
-			sec_cap = rte_security_capabilities_get(ctx);
+		sec_cap = rte_security_capabilities_get(ctx);
 
-			if (sec_cap == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"No capabilities registered\n");
-				return -1;
-			}
+		if (sec_cap == NULL) {
+			RTE_LOG(ERR, IPSEC,
+			"No capabilities registered\n");
+			return -1;
+		}
 
-			/* iterate until ESP tunnel*/
-			while (sec_cap->action !=
+		/* iterate until ESP tunnel*/
+		while (sec_cap->action !=
 					RTE_SECURITY_ACTION_TYPE_NONE) {
 
-				if (sec_cap->action == sa->type &&
-				    sec_cap->protocol ==
-					RTE_SECURITY_PROTOCOL_IPSEC &&
-				    sec_cap->ipsec.mode ==
-					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
-				    sec_cap->ipsec.direction == sa->direction)
-					break;
-				sec_cap++;
-			}
+			if (sec_cap->action == sa->type &&
+				sec_cap->protocol ==
+				RTE_SECURITY_PROTOCOL_IPSEC &&
+				sec_cap->ipsec.mode ==
+				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
+				sec_cap->ipsec.direction == sa->direction)
+				break;
+			sec_cap++;
+		}
 
-			if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
-				RTE_LOG(ERR, IPSEC,
-				"No suitable security capability found\n");
-				return -1;
-			}
+		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
+			RTE_LOG(ERR, IPSEC,
+			"No suitable security capability found\n");
+			return -1;
+		}
+
+		sa->ol_flags = sec_cap->ol_flags;
+		sa->security_ctx = ctx;
+	} else
+		return -EINVAL;
+	return 0;
+}
+
+#define CDEV_IV_SIZE 12
+
+static int
+check_cryptodev_aead_capablity(const struct ipsec_sa *ss, uint8_t dev_id)
+{
+	struct rte_cryptodev_sym_capability_idx cap_idx;
+	const struct rte_cryptodev_symmetric_capability *cap;
 
-			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
+	cap_idx.type = RTE_CRYPTO_SYM_XFORM_AEAD;
+	cap_idx.algo.aead = ss->aead_algo;
+
+	cap = rte_cryptodev_sym_capability_get(dev_id, &cap_idx);
+	if (cap == NULL)
+		return -ENOENT;
+
+	return rte_cryptodev_sym_capability_check_aead(cap,
+					ss->cipher_key_len,
+					ss->digest_len,
+					ss->aad_len,
+					CDEV_IV_SIZE);
+}
+
+static int
+check_cryptodev_capablity(const struct ipsec_sa *ss, uint8_t dev_id)
+{
+	struct rte_cryptodev_sym_capability_idx cap_idx;
+	const struct rte_cryptodev_symmetric_capability *cap;
+	uint16_t auth_iv_len;
+	int rc = -1;
+
+	if (ss == NULL)
+		return rc;
+
+	if (ss->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
+		return check_cryptodev_aead_capablity(ss, dev_id);
+
+	auth_iv_len = 0;
+
+	cap_idx.type = RTE_CRYPTO_SYM_XFORM_AUTH;
+	cap_idx.algo.auth = ss->auth_algo;
+	cap = rte_cryptodev_sym_capability_get(dev_id, &cap_idx);
+	if (cap != NULL) {
+		rc = rte_cryptodev_sym_capability_check_auth(
+				cap, ss->auth_key_len, ss->digest_len,
+				auth_iv_len);
+		if (rc == 0) {
+			cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
+			cap_idx.algo.cipher = ss->cipher_algo;
+			cap = rte_cryptodev_sym_capability_get(dev_id,
+					&cap_idx);
+			if (cap != NULL)
+				rc = rte_cryptodev_sym_capability_check_cipher(
+						cap, ss->cipher_key_len,
+						ss->iv_len);
 		}
-	} else {
-		sa->crypto_session = rte_cryptodev_sym_session_create(
-				ipsec_ctx->session_pool);
-		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
-				sa->crypto_session, sa->xforms,
-				ipsec_ctx->session_priv_pool);
-
-		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
-				&cdev_info);
 	}
-	sa->cdev_id_qp = cdev_id_qp;
 
-	return 0;
+	return rc;
+}
+
+static int
+create_crypto_session(struct ipsec_sa *sa, struct rte_mempool *pool)
+{
+	int32_t rc;
+	uint32_t devnum, i;
+	struct rte_cryptodev_sym_session *s;
+	uint8_t devid[RTE_CRYPTO_MAX_DEVS];
+
+	/* check which cryptodevs support SA */
+	devnum = 0;
+	for (i = 0; i < crypto_dev_num; i++) {
+		rc = check_cryptodev_capablity(sa, crypto_devid[i]);
+		if (rc == 0)
+			devid[devnum++] = crypto_devid[i];
+	}
+
+	if (devnum == 0)
+		return -ENODEV;
+
+	s = rte_cryptodev_sym_session_create(pool);
+	if (s == NULL)
+		return -ENOMEM;
+
+	/* initialize SA crypto session for all supported devices */
+	for (i = 0; i != devnum; i++) {
+		rc = rte_cryptodev_sym_session_init(devid[i], s, sa->xforms,
+			pool);
+		if (rc != 0)
+			break;
+	}
+
+	if (i == devnum) {
+		sa->crypto_session = s;
+		return 0;
+	}
+
+	/* failure, do cleanup */
+	while (i-- != 0)
+		rte_cryptodev_sym_session_clear(devid[i], s);
+
+	rte_cryptodev_sym_session_free(s);
+	return rc;
+}
+
+int
+create_session(struct ipsec_sa *sa, struct rte_mempool *pool)
+{
+	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE)
+		return create_sec_session(sa, pool);
+	else
+		return create_crypto_session(sa, pool);
 }
 
 /*
@@ -393,13 +461,6 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			sym_cop = get_sym_cop(&priv->cop);
 			sym_cop->m_src = pkts[i];
 
@@ -412,13 +473,6 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->crypto_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			rte_crypto_op_attach_sym_session(&priv->cop,
 					sa->crypto_session);
 
@@ -429,12 +483,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 			break;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
+			RTE_ASSERT(sa->sec_session != NULL);
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
 			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
@@ -442,17 +491,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 						sa->sec_session, pkts[i], NULL);
 			continue;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
+			RTE_ASSERT(sa->sec_session != NULL);
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			rte_security_attach_session(&priv->cop,
 					sa->sec_session);
 
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index 99f49d6..804330c 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -83,6 +83,14 @@ struct app_sa_prm {
 
 extern struct app_sa_prm app_sa_prm;
 
+/*
+ * Number of enabled crypto devices
+ * This number is needed when checking crypto device capabilities
+ */
+extern uint8_t crypto_dev_num;
+/* array of crypto device ID's */
+extern uint8_t crypto_devid[RTE_CRYPTO_MAX_DEVS];
+
 struct ipsec_sa {
 	struct rte_ipsec_session ips; /* one session per sa for now */
 	uint32_t spi;
@@ -306,6 +314,6 @@ void
 enqueue_cop_burst(struct cdev_qp *cqp);
 
 int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+create_session(struct ipsec_sa *sa, struct rte_mempool *pool);
 
 #endif /* __IPSEC_H__ */
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index 3f9cacb..0df6969 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -86,39 +86,6 @@ enqueue_cop_bulk(struct cdev_qp *cqp, struct rte_crypto_op *cop[], uint32_t num)
 	cqp->len = len;
 }
 
-static inline int
-fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
-	struct ipsec_sa *sa)
-{
-	int32_t rc;
-
-	/* setup crypto section */
-	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		if (sa->crypto_session == NULL) {
-			rc = create_session(ctx, sa);
-			if (rc != 0)
-				return rc;
-		}
-		ss->crypto.ses = sa->crypto_session;
-	/* setup session action type */
-	} else {
-		if (sa->sec_session == NULL) {
-			rc = create_session(ctx, sa);
-			if (rc != 0)
-				return rc;
-		}
-		ss->security.ses = sa->sec_session;
-		ss->security.ctx = sa->security_ctx;
-		ss->security.ol_flags = sa->ol_flags;
-	}
-
-	rc = rte_ipsec_session_prepare(ss);
-	if (rc != 0)
-		memset(ss, 0, sizeof(*ss));
-
-	return rc;
-}
-
 /*
  * group input packets byt the SA they belong to.
  */
@@ -219,9 +186,8 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 
 		ips = &sa->ips;
 
-		/* no valid HW session for that SA, try to create one */
-		if (sa == NULL || (ips->crypto.ses == NULL &&
-				fill_ipsec_session(ips, ctx, sa) != 0))
+		/* no valid HW session for that SA */
+		if (sa == NULL || ips->crypto.ses == NULL)
 			k = 0;
 
 		/* process packets inline */
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index a7298a3..0f36f5b 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -774,14 +774,14 @@ check_eth_dev_caps(uint16_t portid, uint32_t inbound)
 	return 0;
 }
 
-
 static int
 sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries, uint32_t inbound)
+	uint32_t nb_entries, uint32_t inbound, struct socket_ctx *skt_ctx)
 {
 	struct ipsec_sa *sa;
 	uint32_t i, idx;
 	uint16_t iv_length, aad_length;
+	int32_t rc;
 
 	/* for ESN upper 32 bits of SQN also need to be part of AAD */
 	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
@@ -902,6 +902,12 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			print_one_sa_rule(sa, inbound);
 		}
+		rc = create_session(sa, skt_ctx->session_pool);
+		if (rc != 0) {
+			RTE_LOG(ERR, IPSEC_ESP,
+					"create_session() failed\n");
+			return -EINVAL;
+		}
 	}
 
 	return 0;
@@ -909,16 +915,16 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 static inline int
 sa_out_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 0);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx);
 }
 
 static inline int
 sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 1);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx);
 }
 
 /*
@@ -1012,10 +1018,12 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm, const struct ipsec_sa *ss,
 	return 0;
 }
 
-static void
+static int
 fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 	const struct ipsec_sa *lsa)
 {
+	int32_t rc = 0;
+
 	ss->sa = sa;
 	ss->type = lsa->type;
 
@@ -1028,6 +1036,12 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 		ss->security.ctx = lsa->security_ctx;
 		ss->security.ol_flags = lsa->ol_flags;
 	}
+
+	rc = rte_ipsec_session_prepare(ss);
+	if (rc != 0)
+		memset(ss, 0, sizeof(*ss));
+
+	return rc;
 }
 
 /*
@@ -1062,8 +1076,8 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	fill_ipsec_session(&lsa->ips, sa, lsa);
-	return 0;
+	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
+	return rc;
 }
 
 /*
@@ -1141,7 +1155,10 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in);
+		rc = sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx);
+		if (rc != 0)
+			rte_exit(EXIT_FAILURE,
+				"failed to add inbound rules\n");
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_in, sa_in, nb_sa_in,
@@ -1161,7 +1178,10 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out);
+		rc = sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx);
+		if (rc != 0)
+			rte_exit(EXIT_FAILURE,
+				"failed to add outbound rules\n");
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_out, sa_out, nb_sa_out,
-- 
2.7.4

[dpdk-dev] [PATCH v4 2/2] examples/ipsec-secgw/test: fix inline test scripts

[Bernard Iremonger]

Remove workaround in tun_aesgcm_defs.sh and trs_aesgcm_defs.sh
to get around the bug where the first inbound packet is dropped
for inline crypto.

Fixes: 929784452094 ("examples/ipsec-secgw: add scripts for functional test")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
---
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh | 10 ----------
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh | 10 ----------
 2 files changed, 20 deletions(-)

diff --git a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
index a4d902b..8382d3d 100644
--- a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
@@ -33,11 +33,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -68,9 +63,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV6}
 }
diff --git a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
index 1764ef6..8ae6532 100644
--- a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
@@ -35,11 +35,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -72,9 +67,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping6 -c 1 ${LOCAL_IPV6}
 }
-- 
2.7.4

[dpdk-dev] [PATCH v4 2/2] examples/ipsec-secgw/test: fix inline test scripts

[Bernard Iremonger]

Remove workaround in tun_aesgcm_defs.sh and trs_aesgcm_defs.sh
to get around the bug where the first inbound packet is dropped
for inline crypto.

Fixes: 929784452094 ("examples/ipsec-secgw: add scripts for functional test")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
---
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh | 10 ----------
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh | 10 ----------
 2 files changed, 20 deletions(-)

diff --git a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
index a4d902b..8382d3d 100644
--- a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
@@ -33,11 +33,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -68,9 +63,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV6}
 }
diff --git a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
index 1764ef6..8ae6532 100644
--- a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
@@ -35,11 +35,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -72,9 +67,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping6 -c 1 ${LOCAL_IPV6}
 }
-- 
2.7.4

[dpdk-dev] [PATCH v5 0/2] examples/ipsec-secgw: fix 1st pkt dropped

[Bernard Iremonger]

This patchset fixes the issue of the first inbound packet
being dropped for inline crypto. 
 
Changes in v5:
-------------
The v2 patchset has been rebased to the latest master.
The v4 patchset has been dropped as it caused issues with the lookaside code
which we are unable to test.
 
Bernard Iremonger (2):
  examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
  examples/ipsec-secgw/test: fix inline test scripts

 examples/ipsec-secgw/ipsec-secgw.c           | 244 +++++++++++++--------------
 examples/ipsec-secgw/ipsec.c                 | 122 +++++++++-----
 examples/ipsec-secgw/ipsec.h                 |   5 +-
 examples/ipsec-secgw/ipsec_process.c         |   9 +-
 examples/ipsec-secgw/sa.c                    |  46 +++--
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 --
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 --
 7 files changed, 245 insertions(+), 201 deletions(-)

Changes in v2:
-------------
The first three patches of the v1 have been squashed.
The commit message for the squashed patch has been updated.
Patches 4,5 and 6 of the v1 have been dropped from this patchset.
A patch to fix the test scripts has been added.

Bernard Iremonger (2):
  examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
  examples/ipsec-secgw/test: fix inline test scripts

 examples/ipsec-secgw/ipsec-secgw.c           | 243 +++++++++++++--------------
 examples/ipsec-secgw/ipsec.c                 | 123 +++++++++-----
 examples/ipsec-secgw/ipsec.h                 |   5 +-
 examples/ipsec-secgw/ipsec_process.c         |   9 +-
 examples/ipsec-secgw/sa.c                    |  46 +++--
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 --
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 --
 7 files changed, 246 insertions(+), 200 deletions(-)

-- 
2.7.4

[dpdk-dev] [PATCH v5 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Bernard Iremonger]

Inline crypto installs a flow rule in the NIC. This flow
rule must be installed before the first inbound packet is
received.

The create_session() function installs the flow rule,
create_session() has been refactored into create_inline_session()
and create_lookaside_session(). The create_inline_session() function
uses the socket_ctx data and is now called at initialisation in
sa_add_rules().

The max_session_size() function has been added to calculate memory
requirements.

The cryprodev_init() function has been refactored to drop calls to
rte_mempool_create() and to drop calculation of memory requirements.

The main() function has been refactored to call max_session_size() and
to call session_pool_init() and session_priv_pool_init() earlier.
The ports are started now before adding a flow rule in main().
The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
now called after the ports have been started.

The rte_ipsec_session_prepare() function is called in fill_ipsec_session()
for inline which is called from the ipsec_sa_init() function.

Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/ipsec-secgw.c   | 244 +++++++++++++++++------------------
 examples/ipsec-secgw/ipsec.c         | 122 ++++++++++++------
 examples/ipsec-secgw/ipsec.h         |   5 +-
 examples/ipsec-secgw/ipsec_process.c |   9 +-
 examples/ipsec-secgw/sa.c            |  46 +++++--
 5 files changed, 245 insertions(+), 181 deletions(-)

diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index 6c626fa..24876ba 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -1628,7 +1628,7 @@ cryptodevs_init(void)
 	struct rte_cryptodev_config dev_conf;
 	struct rte_cryptodev_qp_conf qp_conf;
 	uint16_t idx, max_nb_qps, qp, i;
-	int16_t cdev_id, port_id;
+	int16_t cdev_id;
 	struct rte_hash_parameters params = { 0 };
 
 	params.entries = CDEV_MAP_ENTRIES;
@@ -1651,45 +1651,6 @@ cryptodevs_init(void)
 
 	printf("lcore/cryptodev/qp mappings:\n");
 
-	uint32_t max_sess_sz = 0, sess_sz;
-	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
-		void *sec_ctx;
-
-		/* Get crypto priv session size */
-		sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-
-		/*
-		 * If crypto device is security capable, need to check the
-		 * size of security session as well.
-		 */
-
-		/* Get security context of the crypto device */
-		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		/* Get size of security session */
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-	RTE_ETH_FOREACH_DEV(port_id) {
-		void *sec_ctx;
-
-		if ((enabled_port_mask & (1 << port_id)) == 0)
-			continue;
-
-		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-
 	idx = 0;
 	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
 		struct rte_cryptodev_info cdev_info;
@@ -1727,45 +1688,6 @@ cryptodevs_init(void)
 				"Device does not support at least %u "
 				"sessions", CDEV_MP_NB_OBJS);
 
-		if (!socket_ctx[dev_conf.socket_id].session_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_%u", dev_conf.socket_id);
-			sess_mp = rte_cryptodev_sym_session_pool_create(
-					mp_name, CDEV_MP_NB_OBJS,
-					0, CDEV_MP_CACHE_SZ, 0,
-					dev_conf.socket_id);
-			socket_ctx[dev_conf.socket_id].session_pool = sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_priv_%u", dev_conf.socket_id);
-			sess_mp = rte_mempool_create(mp_name,
-					CDEV_MP_NB_OBJS,
-					max_sess_sz,
-					CDEV_MP_CACHE_SZ,
-					0, NULL, NULL, NULL,
-					NULL, dev_conf.socket_id,
-					0);
-			socket_ctx[dev_conf.socket_id].session_priv_pool =
-					sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool ||
-				!socket_ctx[dev_conf.socket_id].session_pool)
-			rte_exit(EXIT_FAILURE,
-				"Cannot create session pool on socket %d\n",
-				dev_conf.socket_id);
-		else
-			printf("Allocated session pool on socket %d\n",
-					dev_conf.socket_id);
-
 		if (rte_cryptodev_configure(cdev_id, &dev_conf))
 			rte_panic("Failed to initialize cryptodev %u\n",
 					cdev_id);
@@ -1786,39 +1708,6 @@ cryptodevs_init(void)
 					cdev_id);
 	}
 
-	/* create session pools for eth devices that implement security */
-	RTE_ETH_FOREACH_DEV(port_id) {
-		if ((enabled_port_mask & (1 << port_id)) &&
-				rte_eth_dev_get_sec_ctx(port_id)) {
-			int socket_id = rte_eth_dev_socket_id(port_id);
-
-			if (!socket_ctx[socket_id].session_priv_pool) {
-				char mp_name[RTE_MEMPOOL_NAMESIZE];
-				struct rte_mempool *sess_mp;
-
-				snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-						"sess_mp_%u", socket_id);
-				sess_mp = rte_mempool_create(mp_name,
-						(CDEV_MP_NB_OBJS * 2),
-						max_sess_sz,
-						CDEV_MP_CACHE_SZ,
-						0, NULL, NULL, NULL,
-						NULL, socket_id,
-						0);
-				if (sess_mp == NULL)
-					rte_exit(EXIT_FAILURE,
-						"Cannot create session pool "
-						"on socket %d\n", socket_id);
-				else
-					printf("Allocated session pool "
-						"on socket %d\n", socket_id);
-				socket_ctx[socket_id].session_priv_pool =
-						sess_mp;
-			}
-		}
-	}
-
-
 	printf("\n");
 
 	return 0;
@@ -1984,6 +1873,99 @@ port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads)
 	printf("\n");
 }
 
+static size_t
+max_session_size(void)
+{
+	size_t max_sz, sz;
+	void *sec_ctx;
+	int16_t cdev_id, port_id, n;
+
+	max_sz = 0;
+	n =  rte_cryptodev_count();
+	for (cdev_id = 0; cdev_id != n; cdev_id++) {
+		sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
+		if (sz > max_sz)
+			max_sz = sz;
+		/*
+		 * If crypto device is security capable, need to check the
+		 * size of security session as well.
+		 */
+
+		/* Get security context of the crypto device */
+		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		/* Get size of security session */
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	RTE_ETH_FOREACH_DEV(port_id) {
+		if ((enabled_port_mask & (1 << port_id)) == 0)
+			continue;
+
+		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	return max_sz;
+}
+
+static void
+session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_%u", socket_id);
+	sess_mp = rte_cryptodev_sym_session_pool_create(
+			mp_name, CDEV_MP_NB_OBJS,
+			sess_sz, CDEV_MP_CACHE_SZ, 0,
+			socket_id);
+	ctx->session_pool = sess_mp;
+
+	if (ctx->session_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session pool on socket %d\n", socket_id);
+	else
+		printf("Allocated session pool on socket %d\n",	socket_id);
+}
+
+static void
+session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
+	size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_priv_%u", socket_id);
+	sess_mp = rte_mempool_create(mp_name,
+			CDEV_MP_NB_OBJS,
+			sess_sz,
+			CDEV_MP_CACHE_SZ,
+			0, NULL, NULL, NULL,
+			NULL, socket_id,
+			0);
+	ctx->session_priv_pool = sess_mp;
+
+	if (ctx->session_priv_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session priv pool on socket %d\n",
+			socket_id);
+	else
+		printf("Allocated session priv pool on socket %d\n",
+			socket_id);
+}
+
 static void
 pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf)
 {
@@ -2064,9 +2046,11 @@ main(int32_t argc, char **argv)
 {
 	int32_t ret;
 	uint32_t lcore_id;
+	uint32_t i;
 	uint8_t socket_id;
 	uint16_t portid;
 	uint64_t req_rx_offloads, req_tx_offloads;
+	size_t sess_sz;
 
 	/* init EAL */
 	ret = rte_eal_init(argc, argv);
@@ -2094,7 +2078,8 @@ main(int32_t argc, char **argv)
 
 	nb_lcores = rte_lcore_count();
 
-	/* Replicate each context per socket */
+	sess_sz = max_session_size();
+
 	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
 		if (rte_lcore_is_enabled(lcore_id) == 0)
 			continue;
@@ -2104,20 +2089,14 @@ main(int32_t argc, char **argv)
 		else
 			socket_id = 0;
 
+		/* mbuf_pool is initialised by the pool_init() function*/
 		if (socket_ctx[socket_id].mbuf_pool)
 			continue;
 
-		/* initilaze SPD */
-		sp4_init(&socket_ctx[socket_id], socket_id);
-
-		sp6_init(&socket_ctx[socket_id], socket_id);
-
-		/* initilaze SAD */
-		sa_init(&socket_ctx[socket_id], socket_id);
-
-		rt_init(&socket_ctx[socket_id], socket_id);
-
 		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
+		session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
+		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
+			sess_sz);
 	}
 
 	RTE_ETH_FOREACH_DEV(portid) {
@@ -2135,7 +2114,11 @@ main(int32_t argc, char **argv)
 		if ((enabled_port_mask & (1 << portid)) == 0)
 			continue;
 
-		/* Start device */
+		/*
+		 * Start device
+		 * note: device must be started before a flow rule
+		 * can be installed.
+		 */
 		ret = rte_eth_dev_start(portid);
 		if (ret < 0)
 			rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
@@ -2153,6 +2136,19 @@ main(int32_t argc, char **argv)
 			RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback, NULL);
 	}
 
+	/* Replicate each context per socket */
+	for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
+		socket_id = rte_socket_id_by_idx(i);
+		if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
+			(socket_ctx[socket_id].sa_in == NULL) &&
+			(socket_ctx[socket_id].sa_out == NULL)) {
+			sa_init(&socket_ctx[socket_id], socket_id);
+			sp4_init(&socket_ctx[socket_id], socket_id);
+			sp6_init(&socket_ctx[socket_id], socket_id);
+			rt_init(&socket_ctx[socket_id], socket_id);
+		}
+	}
+
 	check_all_ports_link_status(enabled_port_mask);
 
 	/* launch per-lcore init on every lcore */
diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index 7b85330..40a6123 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -40,7 +40,7 @@ set_ipsec_conf(struct ipsec_sa *sa, struct rte_security_ipsec_xform *ipsec)
 }
 
 int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 {
 	struct rte_cryptodev_info cdev_info;
 	unsigned long cdev_id_qp = 0;
@@ -108,23 +108,81 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 				"SEC Session init failed: err: %d\n", ret);
 				return -1;
 			}
-		} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		} else if (
+			(sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) ||
+			(sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
+			) {
+				RTE_LOG(ERR, IPSEC, "Inline not supported\n");
+				return -1;
+		}
+	} else {
+		sa->crypto_session = rte_cryptodev_sym_session_create(
+				ipsec_ctx->session_pool);
+		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
+				sa->crypto_session, sa->xforms,
+				ipsec_ctx->session_priv_pool);
+
+		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
+				&cdev_info);
+	}
+
+	sa->cdev_id_qp = cdev_id_qp;
+
+	return 0;
+}
+
+int
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
+{
+	unsigned long cdev_id_qp = 0;
+	int32_t ret = 0;
+	struct rte_security_ctx *sec_ctx;
+
+	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on port %u\n",
+		sa->spi, sa->portid);
+
+	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
+		struct rte_security_session_conf sess_conf = {
+			.action_type = sa->type,
+			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
+			{.ipsec = {
+				.spi = sa->spi,
+				.salt = sa->salt,
+				.options = { 0 },
+				.direction = sa->direction,
+				.proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
+				.mode = (sa->flags == IP4_TUNNEL ||
+						sa->flags == IP6_TUNNEL) ?
+					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL :
+					RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
+			} },
+			.crypto_xform = sa->xforms,
+			.userdata = NULL,
+		};
+
+		if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
 			struct rte_flow_error err;
-			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
-							rte_eth_dev_get_sec_ctx(
-							sa->portid);
 			const struct rte_security_capability *sec_cap;
 			int ret = 0;
 
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_priv_pool);
+			sec_ctx = (struct rte_security_ctx *)
+							rte_eth_dev_get_sec_ctx(
+							sa->portid);
+			if (sec_ctx == NULL) {
+				RTE_LOG(ERR, IPSEC,
+				" rte_eth_dev_get_sec_ctx failed\n");
+				return -1;
+			}
+
+			sa->sec_session = rte_security_session_create(sec_ctx,
+					&sess_conf, skt_ctx->session_pool);
 			if (sa->sec_session == NULL) {
 				RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 				return -1;
 			}
 
-			sec_cap = rte_security_capabilities_get(ctx);
+			sec_cap = rte_security_capabilities_get(sec_ctx);
 
 			/* iterate until ESP tunnel*/
 			while (sec_cap->action !=
@@ -147,7 +205,7 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			}
 
 			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
+			sa->security_ctx = sec_ctx;
 			sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
 
 			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
@@ -196,7 +254,7 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 				/* Try RSS. */
 				sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
 				sa->action[1].conf = &action_rss;
-				eth_dev = ctx->device;
+				eth_dev = sec_ctx->device;
 				rte_eth_dev_rss_hash_conf_get(sa->portid,
 							      &rss_conf);
 				for (i = 0, j = 0;
@@ -252,12 +310,12 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			}
 		} else if (sa->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
-			struct rte_security_ctx *ctx =
-					(struct rte_security_ctx *)
-					rte_eth_dev_get_sec_ctx(sa->portid);
 			const struct rte_security_capability *sec_cap;
 
-			if (ctx == NULL) {
+			sec_ctx = (struct rte_security_ctx *)
+					rte_eth_dev_get_sec_ctx(sa->portid);
+
+			if (sec_ctx == NULL) {
 				RTE_LOG(ERR, IPSEC,
 				"Ethernet device doesn't have security features registered\n");
 				return -1;
@@ -279,15 +337,15 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 
 			sess_conf.userdata = (void *) sa;
 
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
+			sa->sec_session = rte_security_session_create(sec_ctx,
+					&sess_conf, skt_ctx->session_pool);
 			if (sa->sec_session == NULL) {
 				RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 				return -1;
 			}
 
-			sec_cap = rte_security_capabilities_get(ctx);
+			sec_cap = rte_security_capabilities_get(sec_ctx);
 
 			if (sec_cap == NULL) {
 				RTE_LOG(ERR, IPSEC,
@@ -316,17 +374,8 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			}
 
 			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
+			sa->security_ctx = sec_ctx;
 		}
-	} else {
-		sa->crypto_session = rte_cryptodev_sym_session_create(
-				ipsec_ctx->session_pool);
-		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
-				sa->crypto_session, sa->xforms,
-				ipsec_ctx->session_priv_pool);
-
-		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
-				&cdev_info);
 	}
 	sa->cdev_id_qp = cdev_id_qp;
 
@@ -395,7 +444,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			rte_prefetch0(&priv->sym_cop);
 
 			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
+				create_lookaside_session(ipsec_ctx, sa)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -414,7 +463,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			rte_prefetch0(&priv->sym_cop);
 
 			if ((unlikely(sa->crypto_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
+				create_lookaside_session(ipsec_ctx, sa)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -429,12 +478,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 			break;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
+			RTE_ASSERT(sa->sec_session != NULL);
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
 			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
@@ -442,17 +486,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 						sa->sec_session, pkts[i], NULL);
 			continue;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
+			RTE_ASSERT(sa->sec_session != NULL);
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			rte_security_attach_session(&priv->cop,
 					sa->sec_session);
 
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index e9272d7..41bac0b 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -312,6 +312,9 @@ void
 enqueue_cop_burst(struct cdev_qp *cqp);
 
 int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+
+int
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa);
 
 #endif /* __IPSEC_H__ */
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index 3f9cacb..868f1a2 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -95,22 +95,23 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
 	/* setup crypto section */
 	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
 		if (sa->crypto_session == NULL) {
-			rc = create_session(ctx, sa);
+			rc = create_lookaside_session(ctx, sa);
 			if (rc != 0)
 				return rc;
 		}
 		ss->crypto.ses = sa->crypto_session;
 	/* setup session action type */
-	} else {
+	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
 		if (sa->sec_session == NULL) {
-			rc = create_session(ctx, sa);
+			rc = create_lookaside_session(ctx, sa);
 			if (rc != 0)
 				return rc;
 		}
 		ss->security.ses = sa->sec_session;
 		ss->security.ctx = sa->security_ctx;
 		ss->security.ol_flags = sa->ol_flags;
-	}
+	} else
+		RTE_ASSERT(0);
 
 	rc = rte_ipsec_session_prepare(ss);
 	if (rc != 0)
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index 8d47d1d..e8e55bf 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -777,11 +777,13 @@ check_eth_dev_caps(uint16_t portid, uint32_t inbound)
 
 static int
 sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries, uint32_t inbound)
+		uint32_t nb_entries, uint32_t inbound,
+		struct socket_ctx *skt_ctx)
 {
 	struct ipsec_sa *sa;
 	uint32_t i, idx;
 	uint16_t iv_length, aad_length;
+	int32_t rc;
 
 	/* for ESN upper 32 bits of SQN also need to be part of AAD */
 	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
@@ -834,6 +836,17 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			sa->xforms = &sa_ctx->xf[idx].a;
 
+			if (sa->type ==
+				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
+				sa->type ==
+				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+				rc = create_inline_session(skt_ctx, sa);
+				if (rc != 0) {
+					RTE_LOG(ERR, IPSEC_ESP,
+						"create_inline_session() failed\n");
+					return -EINVAL;
+				}
+			}
 			print_one_sa_rule(sa, inbound);
 		} else {
 			switch (sa->cipher_algo) {
@@ -909,16 +922,16 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 static inline int
 sa_out_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 0);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx);
 }
 
 static inline int
 sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 1);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx);
 }
 
 /*
@@ -1012,10 +1025,12 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm, const struct ipsec_sa *ss,
 	return 0;
 }
 
-static void
+static int
 fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 	const struct ipsec_sa *lsa)
 {
+	int32_t rc = 0;
+
 	ss->sa = sa;
 	ss->type = lsa->type;
 
@@ -1028,6 +1043,17 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 		ss->security.ctx = lsa->security_ctx;
 		ss->security.ol_flags = lsa->ol_flags;
 	}
+
+	if (ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+		ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
+		if (ss->security.ses != NULL) {
+			rc = rte_ipsec_session_prepare(ss);
+			if (rc != 0)
+				memset(ss, 0, sizeof(*ss));
+		}
+	}
+
+	return rc;
 }
 
 /*
@@ -1062,8 +1088,8 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	fill_ipsec_session(&lsa->ips, sa, lsa);
-	return 0;
+	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
+	return rc;
 }
 
 /*
@@ -1166,7 +1192,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in);
+		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx);
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_in, sa_in, nb_sa_in,
@@ -1186,7 +1212,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out);
+		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx);
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_out, sa_out, nb_sa_out,
-- 
2.7.4

[dpdk-dev] [PATCH v5 2/2] examples/ipsec-secgw/test: fix inline test scripts

[Bernard Iremonger]

Remove workaround in tun_aesgcm_defs.sh and trs_aesgcm_defs.sh
to get around the bug where the first inbound packet is dropped
for inline crypto.

Fixes: 929784452094 ("examples/ipsec-secgw: add scripts for functional test")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh | 10 ----------
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh | 10 ----------
 2 files changed, 20 deletions(-)

diff --git a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
index a4d902b..8382d3d 100755
--- a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
@@ -33,11 +33,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -68,9 +63,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV6}
 }
diff --git a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
index 1764ef6..8ae6532 100755
--- a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
@@ -35,11 +35,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -72,9 +67,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping6 -c 1 ${LOCAL_IPV6}
 }
-- 
2.7.4

Re: [dpdk-dev] [PATCH v5 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Ananyev, Konstantin]

Hi Bernard,
Few small comments below.
Konstantin

> Inline crypto installs a flow rule in the NIC. This flow
> rule must be installed before the first inbound packet is
> received.
> 
> The create_session() function installs the flow rule,
> create_session() has been refactored into create_inline_session()
> and create_lookaside_session(). The create_inline_session() function
> uses the socket_ctx data and is now called at initialisation in
> sa_add_rules().
> 
> The max_session_size() function has been added to calculate memory
> requirements.
> 
> The cryprodev_init() function has been refactored to drop calls to
> rte_mempool_create() and to drop calculation of memory requirements.
> 
> The main() function has been refactored to call max_session_size() and
> to call session_pool_init() and session_priv_pool_init() earlier.
> The ports are started now before adding a flow rule in main().
> The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
> now called after the ports have been started.
> 
> The rte_ipsec_session_prepare() function is called in fill_ipsec_session()
> for inline which is called from the ipsec_sa_init() function.
> 
> Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
> Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
> Cc: stable@dpdk.org
> 
> Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> ---
>  examples/ipsec-secgw/ipsec-secgw.c   | 244 +++++++++++++++++------------------
>  examples/ipsec-secgw/ipsec.c         | 122 ++++++++++++------
>  examples/ipsec-secgw/ipsec.h         |   5 +-
>  examples/ipsec-secgw/ipsec_process.c |   9 +-
>  examples/ipsec-secgw/sa.c            |  46 +++++--
>  5 files changed, 245 insertions(+), 181 deletions(-)
> 
> diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
> index 6c626fa..24876ba 100644
> --- a/examples/ipsec-secgw/ipsec-secgw.c
> +++ b/examples/ipsec-secgw/ipsec-secgw.c
> @@ -1628,7 +1628,7 @@ cryptodevs_init(void)
>  	struct rte_cryptodev_config dev_conf;
>  	struct rte_cryptodev_qp_conf qp_conf;
>  	uint16_t idx, max_nb_qps, qp, i;
> -	int16_t cdev_id, port_id;
> +	int16_t cdev_id;
>  	struct rte_hash_parameters params = { 0 };
> 
>  	params.entries = CDEV_MAP_ENTRIES;
> @@ -1651,45 +1651,6 @@ cryptodevs_init(void)
> 
>  	printf("lcore/cryptodev/qp mappings:\n");
> 
> -	uint32_t max_sess_sz = 0, sess_sz;
> -	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
> -		void *sec_ctx;
> -
> -		/* Get crypto priv session size */
> -		sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
> -		if (sess_sz > max_sess_sz)
> -			max_sess_sz = sess_sz;
> -
> -		/*
> -		 * If crypto device is security capable, need to check the
> -		 * size of security session as well.
> -		 */
> -
> -		/* Get security context of the crypto device */
> -		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
> -		if (sec_ctx == NULL)
> -			continue;
> -
> -		/* Get size of security session */
> -		sess_sz = rte_security_session_get_size(sec_ctx);
> -		if (sess_sz > max_sess_sz)
> -			max_sess_sz = sess_sz;
> -	}
> -	RTE_ETH_FOREACH_DEV(port_id) {
> -		void *sec_ctx;
> -
> -		if ((enabled_port_mask & (1 << port_id)) == 0)
> -			continue;
> -
> -		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
> -		if (sec_ctx == NULL)
> -			continue;
> -
> -		sess_sz = rte_security_session_get_size(sec_ctx);
> -		if (sess_sz > max_sess_sz)
> -			max_sess_sz = sess_sz;
> -	}
> -
>  	idx = 0;
>  	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
>  		struct rte_cryptodev_info cdev_info;
> @@ -1727,45 +1688,6 @@ cryptodevs_init(void)
>  				"Device does not support at least %u "
>  				"sessions", CDEV_MP_NB_OBJS);
> 
> -		if (!socket_ctx[dev_conf.socket_id].session_pool) {
> -			char mp_name[RTE_MEMPOOL_NAMESIZE];
> -			struct rte_mempool *sess_mp;
> -
> -			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> -					"sess_mp_%u", dev_conf.socket_id);
> -			sess_mp = rte_cryptodev_sym_session_pool_create(
> -					mp_name, CDEV_MP_NB_OBJS,
> -					0, CDEV_MP_CACHE_SZ, 0,
> -					dev_conf.socket_id);
> -			socket_ctx[dev_conf.socket_id].session_pool = sess_mp;
> -		}
> -
> -		if (!socket_ctx[dev_conf.socket_id].session_priv_pool) {
> -			char mp_name[RTE_MEMPOOL_NAMESIZE];
> -			struct rte_mempool *sess_mp;
> -
> -			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> -					"sess_mp_priv_%u", dev_conf.socket_id);
> -			sess_mp = rte_mempool_create(mp_name,
> -					CDEV_MP_NB_OBJS,
> -					max_sess_sz,
> -					CDEV_MP_CACHE_SZ,
> -					0, NULL, NULL, NULL,
> -					NULL, dev_conf.socket_id,
> -					0);
> -			socket_ctx[dev_conf.socket_id].session_priv_pool =
> -					sess_mp;
> -		}
> -
> -		if (!socket_ctx[dev_conf.socket_id].session_priv_pool ||
> -				!socket_ctx[dev_conf.socket_id].session_pool)
> -			rte_exit(EXIT_FAILURE,
> -				"Cannot create session pool on socket %d\n",
> -				dev_conf.socket_id);
> -		else
> -			printf("Allocated session pool on socket %d\n",
> -					dev_conf.socket_id);
> -
>  		if (rte_cryptodev_configure(cdev_id, &dev_conf))
>  			rte_panic("Failed to initialize cryptodev %u\n",
>  					cdev_id);
> @@ -1786,39 +1708,6 @@ cryptodevs_init(void)
>  					cdev_id);
>  	}
> 
> -	/* create session pools for eth devices that implement security */
> -	RTE_ETH_FOREACH_DEV(port_id) {
> -		if ((enabled_port_mask & (1 << port_id)) &&
> -				rte_eth_dev_get_sec_ctx(port_id)) {
> -			int socket_id = rte_eth_dev_socket_id(port_id);
> -
> -			if (!socket_ctx[socket_id].session_priv_pool) {
> -				char mp_name[RTE_MEMPOOL_NAMESIZE];
> -				struct rte_mempool *sess_mp;
> -
> -				snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> -						"sess_mp_%u", socket_id);
> -				sess_mp = rte_mempool_create(mp_name,
> -						(CDEV_MP_NB_OBJS * 2),
> -						max_sess_sz,
> -						CDEV_MP_CACHE_SZ,
> -						0, NULL, NULL, NULL,
> -						NULL, socket_id,
> -						0);
> -				if (sess_mp == NULL)
> -					rte_exit(EXIT_FAILURE,
> -						"Cannot create session pool "
> -						"on socket %d\n", socket_id);
> -				else
> -					printf("Allocated session pool "
> -						"on socket %d\n", socket_id);
> -				socket_ctx[socket_id].session_priv_pool =
> -						sess_mp;
> -			}
> -		}
> -	}
> -
> -
>  	printf("\n");
> 
>  	return 0;
> @@ -1984,6 +1873,99 @@ port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads)
>  	printf("\n");
>  }
> 
> +static size_t
> +max_session_size(void)
> +{
> +	size_t max_sz, sz;
> +	void *sec_ctx;
> +	int16_t cdev_id, port_id, n;
> +
> +	max_sz = 0;
> +	n =  rte_cryptodev_count();
> +	for (cdev_id = 0; cdev_id != n; cdev_id++) {
> +		sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
> +		if (sz > max_sz)
> +			max_sz = sz;
> +		/*
> +		 * If crypto device is security capable, need to check the
> +		 * size of security session as well.
> +		 */
> +
> +		/* Get security context of the crypto device */
> +		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
> +		if (sec_ctx == NULL)
> +			continue;
> +
> +		/* Get size of security session */
> +		sz = rte_security_session_get_size(sec_ctx);
> +		if (sz > max_sz)
> +			max_sz = sz;
> +	}
> +
> +	RTE_ETH_FOREACH_DEV(port_id) {
> +		if ((enabled_port_mask & (1 << port_id)) == 0)
> +			continue;
> +
> +		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
> +		if (sec_ctx == NULL)
> +			continue;
> +
> +		sz = rte_security_session_get_size(sec_ctx);
> +		if (sz > max_sz)
> +			max_sz = sz;
> +	}
> +
> +	return max_sz;
> +}
> +
> +static void
> +session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz)
> +{
> +	char mp_name[RTE_MEMPOOL_NAMESIZE];
> +	struct rte_mempool *sess_mp;
> +
> +	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> +			"sess_mp_%u", socket_id);
> +	sess_mp = rte_cryptodev_sym_session_pool_create(
> +			mp_name, CDEV_MP_NB_OBJS,
> +			sess_sz, CDEV_MP_CACHE_SZ, 0,
> +			socket_id);
> +	ctx->session_pool = sess_mp;
> +
> +	if (ctx->session_pool == NULL)
> +		rte_exit(EXIT_FAILURE,
> +			"Cannot init session pool on socket %d\n", socket_id);
> +	else
> +		printf("Allocated session pool on socket %d\n",	socket_id);
> +}
> +
> +static void
> +session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
> +	size_t sess_sz)
> +{
> +	char mp_name[RTE_MEMPOOL_NAMESIZE];
> +	struct rte_mempool *sess_mp;
> +
> +	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> +			"sess_mp_priv_%u", socket_id);
> +	sess_mp = rte_mempool_create(mp_name,
> +			CDEV_MP_NB_OBJS,
> +			sess_sz,
> +			CDEV_MP_CACHE_SZ,
> +			0, NULL, NULL, NULL,
> +			NULL, socket_id,
> +			0);
> +	ctx->session_priv_pool = sess_mp;
> +
> +	if (ctx->session_priv_pool == NULL)
> +		rte_exit(EXIT_FAILURE,
> +			"Cannot init session priv pool on socket %d\n",
> +			socket_id);
> +	else
> +		printf("Allocated session priv pool on socket %d\n",
> +			socket_id);
> +}
> +
>  static void
>  pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf)
>  {
> @@ -2064,9 +2046,11 @@ main(int32_t argc, char **argv)
>  {
>  	int32_t ret;
>  	uint32_t lcore_id;
> +	uint32_t i;
>  	uint8_t socket_id;
>  	uint16_t portid;
>  	uint64_t req_rx_offloads, req_tx_offloads;
> +	size_t sess_sz;
> 
>  	/* init EAL */
>  	ret = rte_eal_init(argc, argv);
> @@ -2094,7 +2078,8 @@ main(int32_t argc, char **argv)
> 
>  	nb_lcores = rte_lcore_count();
> 
> -	/* Replicate each context per socket */
> +	sess_sz = max_session_size();
> +
>  	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
>  		if (rte_lcore_is_enabled(lcore_id) == 0)
>  			continue;
> @@ -2104,20 +2089,14 @@ main(int32_t argc, char **argv)
>  		else
>  			socket_id = 0;
> 
> +		/* mbuf_pool is initialised by the pool_init() function*/
>  		if (socket_ctx[socket_id].mbuf_pool)
>  			continue;
> 
> -		/* initilaze SPD */
> -		sp4_init(&socket_ctx[socket_id], socket_id);
> -
> -		sp6_init(&socket_ctx[socket_id], socket_id);
> -
> -		/* initilaze SAD */
> -		sa_init(&socket_ctx[socket_id], socket_id);
> -
> -		rt_init(&socket_ctx[socket_id], socket_id);
> -
>  		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
> +		session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
> +		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
> +			sess_sz);
>  	}
> 
>  	RTE_ETH_FOREACH_DEV(portid) {
> @@ -2135,7 +2114,11 @@ main(int32_t argc, char **argv)
>  		if ((enabled_port_mask & (1 << portid)) == 0)
>  			continue;
> 
> -		/* Start device */
> +		/*
> +		 * Start device
> +		 * note: device must be started before a flow rule
> +		 * can be installed.
> +		 */
>  		ret = rte_eth_dev_start(portid);
>  		if (ret < 0)
>  			rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
> @@ -2153,6 +2136,19 @@ main(int32_t argc, char **argv)
>  			RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback, NULL);
>  	}
> 
> +	/* Replicate each context per socket */
> +	for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
> +		socket_id = rte_socket_id_by_idx(i);
> +		if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
> +			(socket_ctx[socket_id].sa_in == NULL) &&
> +			(socket_ctx[socket_id].sa_out == NULL)) {
> +			sa_init(&socket_ctx[socket_id], socket_id);
> +			sp4_init(&socket_ctx[socket_id], socket_id);
> +			sp6_init(&socket_ctx[socket_id], socket_id);
> +			rt_init(&socket_ctx[socket_id], socket_id);
> +		}
> +	}
> +
>  	check_all_ports_link_status(enabled_port_mask);
> 
>  	/* launch per-lcore init on every lcore */
> diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
> index 7b85330..40a6123 100644
> --- a/examples/ipsec-secgw/ipsec.c
> +++ b/examples/ipsec-secgw/ipsec.c
> @@ -40,7 +40,7 @@ set_ipsec_conf(struct ipsec_sa *sa, struct rte_security_ipsec_xform *ipsec)
>  }
> 
>  int
> -create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
> +create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
>  {
>  	struct rte_cryptodev_info cdev_info;
>  	unsigned long cdev_id_qp = 0;


Actually looking at it a bit closer:

if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {

I think we need to remove that line above, otherwise cdev_id_qp
Would not be initialized for LOOKASIDE_PROTO.
BTW, as I can see, same problem exists in current code.


                ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
                                (void **)&cdev_id_qp);
                if (ret < 0) {
                        RTE_LOG(ERR, IPSEC,
                                "No cryptodev: core %u, cipher_algo %u, "
                                "auth_algo %u, aead_algo %u\n",
                                key.lcore_id,
                                key.cipher_algo,
                                key.auth_algo,
                                key.aead_algo);
                        return -1;
                }
        }

> @@ -108,23 +108,81 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
>  				"SEC Session init failed: err: %d\n", ret);
>  				return -1;
>  			}
> -		} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
> +		} else if (
> +			(sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) ||
> +			(sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
> +			) {
> +				RTE_LOG(ERR, IPSEC, "Inline not supported\n");
> +				return -1;
> +		}
> +	} else {
> +		sa->crypto_session = rte_cryptodev_sym_session_create(
> +				ipsec_ctx->session_pool);
> +		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
> +				sa->crypto_session, sa->xforms,
> +				ipsec_ctx->session_priv_pool);
> +
> +		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
> +				&cdev_info);
> +	}
> +
> +	sa->cdev_id_qp = cdev_id_qp;
> +
> +	return 0;
> +}
> +
> +int
> +create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
> +{
> +	unsigned long cdev_id_qp = 0;

This var is not really used in this function.

> +	int32_t ret = 0;
> +	struct rte_security_ctx *sec_ctx;
> +
> +	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on port %u\n",
> +		sa->spi, sa->portid);
> +
> +	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {

I think this always be true here and could be removed.


> +		struct rte_security_session_conf sess_conf = {
> +			.action_type = sa->type,
> +			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
> +			{.ipsec = {
> +				.spi = sa->spi,
> +				.salt = sa->salt,
> +				.options = { 0 },
> +				.direction = sa->direction,
> +				.proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
> +				.mode = (sa->flags == IP4_TUNNEL ||
> +						sa->flags == IP6_TUNNEL) ?
> +					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL :
> +					RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
> +			} },
> +			.crypto_xform = sa->xforms,
> +			.userdata = NULL,
> +		};
> +
> +		if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
>  			struct rte_flow_error err;
> -			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> -							rte_eth_dev_get_sec_ctx(
> -							sa->portid);
>  			const struct rte_security_capability *sec_cap;
>  			int ret = 0;
> 
> -			sa->sec_session = rte_security_session_create(ctx,
> -					&sess_conf, ipsec_ctx->session_priv_pool);
> +			sec_ctx = (struct rte_security_ctx *)
> +							rte_eth_dev_get_sec_ctx(
> +							sa->portid);
> +			if (sec_ctx == NULL) {
> +				RTE_LOG(ERR, IPSEC,
> +				" rte_eth_dev_get_sec_ctx failed\n");
> +				return -1;
> +			}
> +
> +			sa->sec_session = rte_security_session_create(sec_ctx,
> +					&sess_conf, skt_ctx->session_pool);
>  			if (sa->sec_session == NULL) {
>  				RTE_LOG(ERR, IPSEC,
>  				"SEC Session init failed: err: %d\n", ret);
>  				return -1;
>  			}
> 
> -			sec_cap = rte_security_capabilities_get(ctx);
> +			sec_cap = rte_security_capabilities_get(sec_ctx);
> 
>  			/* iterate until ESP tunnel*/
>  			while (sec_cap->action !=
> @@ -147,7 +205,7 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
>  			}
> 
>  			sa->ol_flags = sec_cap->ol_flags;
> -			sa->security_ctx = ctx;
> +			sa->security_ctx = sec_ctx;
>  			sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
> 
>  			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
> @@ -196,7 +254,7 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
>  				/* Try RSS. */
>  				sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
>  				sa->action[1].conf = &action_rss;
> -				eth_dev = ctx->device;
> +				eth_dev = sec_ctx->device;
>  				rte_eth_dev_rss_hash_conf_get(sa->portid,
>  							      &rss_conf);
>  				for (i = 0, j = 0;
> @@ -252,12 +310,12 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
>  			}
>  		} else if (sa->type ==
>  				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
> -			struct rte_security_ctx *ctx =
> -					(struct rte_security_ctx *)
> -					rte_eth_dev_get_sec_ctx(sa->portid);
>  			const struct rte_security_capability *sec_cap;
> 
> -			if (ctx == NULL) {
> +			sec_ctx = (struct rte_security_ctx *)
> +					rte_eth_dev_get_sec_ctx(sa->portid);
> +
> +			if (sec_ctx == NULL) {
>  				RTE_LOG(ERR, IPSEC,
>  				"Ethernet device doesn't have security features registered\n");
>  				return -1;
> @@ -279,15 +337,15 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
> 
>  			sess_conf.userdata = (void *) sa;
> 
> -			sa->sec_session = rte_security_session_create(ctx,
> -					&sess_conf, ipsec_ctx->session_pool);
> +			sa->sec_session = rte_security_session_create(sec_ctx,
> +					&sess_conf, skt_ctx->session_pool);
>  			if (sa->sec_session == NULL) {
>  				RTE_LOG(ERR, IPSEC,
>  				"SEC Session init failed: err: %d\n", ret);
>  				return -1;
>  			}
> 
> -			sec_cap = rte_security_capabilities_get(ctx);
> +			sec_cap = rte_security_capabilities_get(sec_ctx);
> 
>  			if (sec_cap == NULL) {
>  				RTE_LOG(ERR, IPSEC,
> @@ -316,17 +374,8 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
>  			}
> 
>  			sa->ol_flags = sec_cap->ol_flags;
> -			sa->security_ctx = ctx;
> +			sa->security_ctx = sec_ctx;
>  		}
> -	} else {
> -		sa->crypto_session = rte_cryptodev_sym_session_create(
> -				ipsec_ctx->session_pool);
> -		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
> -				sa->crypto_session, sa->xforms,
> -				ipsec_ctx->session_priv_pool);
> -
> -		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
> -				&cdev_info);
>  	}
>  	sa->cdev_id_qp = cdev_id_qp;
> 
> @@ -395,7 +444,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
>  			rte_prefetch0(&priv->sym_cop);
> 
>  			if ((unlikely(sa->sec_session == NULL)) &&
> -					create_session(ipsec_ctx, sa)) {
> +				create_lookaside_session(ipsec_ctx, sa)) {
>  				rte_pktmbuf_free(pkts[i]);
>  				continue;
>  			}
> @@ -414,7 +463,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
>  			rte_prefetch0(&priv->sym_cop);
> 
>  			if ((unlikely(sa->crypto_session == NULL)) &&
> -					create_session(ipsec_ctx, sa)) {
> +				create_lookaside_session(ipsec_ctx, sa)) {
>  				rte_pktmbuf_free(pkts[i]);
>  				continue;
>  			}
> @@ -429,12 +478,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
>  			}
>  			break;
>  		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
> -			if ((unlikely(sa->sec_session == NULL)) &&
> -					create_session(ipsec_ctx, sa)) {
> -				rte_pktmbuf_free(pkts[i]);
> -				continue;
> -			}
> -
> +			RTE_ASSERT(sa->sec_session != NULL);
>  			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
>  			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
>  				rte_security_set_pkt_metadata(
> @@ -442,17 +486,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
>  						sa->sec_session, pkts[i], NULL);
>  			continue;
>  		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
> +			RTE_ASSERT(sa->sec_session != NULL);
>  			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
>  			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
> 
>  			rte_prefetch0(&priv->sym_cop);
> -
> -			if ((unlikely(sa->sec_session == NULL)) &&
> -					create_session(ipsec_ctx, sa)) {
> -				rte_pktmbuf_free(pkts[i]);
> -				continue;
> -			}
> -
>  			rte_security_attach_session(&priv->cop,
>  					sa->sec_session);
> 
> diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
> index e9272d7..41bac0b 100644
> --- a/examples/ipsec-secgw/ipsec.h
> +++ b/examples/ipsec-secgw/ipsec.h
> @@ -312,6 +312,9 @@ void
>  enqueue_cop_burst(struct cdev_qp *cqp);
> 
>  int
> -create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
> +create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
> +
> +int
> +create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa);
> 
>  #endif /* __IPSEC_H__ */
> diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
> index 3f9cacb..868f1a2 100644
> --- a/examples/ipsec-secgw/ipsec_process.c
> +++ b/examples/ipsec-secgw/ipsec_process.c
> @@ -95,22 +95,23 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
>  	/* setup crypto section */
>  	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
>  		if (sa->crypto_session == NULL) {
> -			rc = create_session(ctx, sa);
> +			rc = create_lookaside_session(ctx, sa);
>  			if (rc != 0)
>  				return rc;
>  		}
>  		ss->crypto.ses = sa->crypto_session;
>  	/* setup session action type */
> -	} else {
> +	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
>  		if (sa->sec_session == NULL) {
> -			rc = create_session(ctx, sa);
> +			rc = create_lookaside_session(ctx, sa);
>  			if (rc != 0)
>  				return rc;
>  		}
>  		ss->security.ses = sa->sec_session;
>  		ss->security.ctx = sa->security_ctx;
>  		ss->security.ol_flags = sa->ol_flags;
> -	}
> +	} else
> +		RTE_ASSERT(0);
> 
>  	rc = rte_ipsec_session_prepare(ss);
>  	if (rc != 0)
> diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
> index 8d47d1d..e8e55bf 100644
> --- a/examples/ipsec-secgw/sa.c
> +++ b/examples/ipsec-secgw/sa.c
> @@ -777,11 +777,13 @@ check_eth_dev_caps(uint16_t portid, uint32_t inbound)
> 
>  static int
>  sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> -		uint32_t nb_entries, uint32_t inbound)
> +		uint32_t nb_entries, uint32_t inbound,
> +		struct socket_ctx *skt_ctx)
>  {
>  	struct ipsec_sa *sa;
>  	uint32_t i, idx;
>  	uint16_t iv_length, aad_length;
> +	int32_t rc;
> 
>  	/* for ESN upper 32 bits of SQN also need to be part of AAD */
>  	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
> @@ -834,6 +836,17 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> 
>  			sa->xforms = &sa_ctx->xf[idx].a;
> 
> +			if (sa->type ==
> +				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
> +				sa->type ==
> +				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
> +				rc = create_inline_session(skt_ctx, sa);
> +				if (rc != 0) {
> +					RTE_LOG(ERR, IPSEC_ESP,
> +						"create_inline_session() failed\n");
> +					return -EINVAL;
> +				}
> +			}
>  			print_one_sa_rule(sa, inbound);
>  		} else {
>  			switch (sa->cipher_algo) {
> @@ -909,16 +922,16 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> 
>  static inline int
>  sa_out_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> -		uint32_t nb_entries)
> +		uint32_t nb_entries, struct socket_ctx *skt_ctx)
>  {
> -	return sa_add_rules(sa_ctx, entries, nb_entries, 0);
> +	return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx);
>  }
> 
>  static inline int
>  sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> -		uint32_t nb_entries)
> +		uint32_t nb_entries, struct socket_ctx *skt_ctx)
>  {
> -	return sa_add_rules(sa_ctx, entries, nb_entries, 1);
> +	return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx);
>  }
> 
>  /*
> @@ -1012,10 +1025,12 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm, const struct ipsec_sa *ss,
>  	return 0;
>  }
> 
> -static void
> +static int
>  fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
>  	const struct ipsec_sa *lsa)
>  {
> +	int32_t rc = 0;
> +
>  	ss->sa = sa;
>  	ss->type = lsa->type;
> 
> @@ -1028,6 +1043,17 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
>  		ss->security.ctx = lsa->security_ctx;
>  		ss->security.ol_flags = lsa->ol_flags;
>  	}
> +
> +	if (ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
> +		ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
> +		if (ss->security.ses != NULL) {
> +			rc = rte_ipsec_session_prepare(ss);
> +			if (rc != 0)
> +				memset(ss, 0, sizeof(*ss));
> +		}
> +	}
> +
> +	return rc;
>  }
> 
>  /*
> @@ -1062,8 +1088,8 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
>  	if (rc < 0)
>  		return rc;
> 
> -	fill_ipsec_session(&lsa->ips, sa, lsa);
> -	return 0;
> +	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
> +	return rc;
>  }
> 
>  /*
> @@ -1166,7 +1192,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
>  				"context %s in socket %d\n", rte_errno,
>  				name, socket_id);
> 
> -		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in);
> +		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx);
> 
>  		if (app_sa_prm.enable != 0) {
>  			rc = ipsec_satbl_init(ctx->sa_in, sa_in, nb_sa_in,
> @@ -1186,7 +1212,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
>  				"context %s in socket %d\n", rte_errno,
>  				name, socket_id);
> 
> -		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out);
> +		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx);
> 
>  		if (app_sa_prm.enable != 0) {
>  			rc = ipsec_satbl_init(ctx->sa_out, sa_out, nb_sa_out,
> --
> 2.7.4

Re: [dpdk-dev] [PATCH v5 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Iremonger, Bernard]

Hi Konstantin,

> -----Original Message-----
> From: Ananyev, Konstantin
> Sent: Tuesday, June 11, 2019 4:29 PM
> To: Iremonger, Bernard <bernard.iremonger@intel.com>; dev@dpdk.org;
> akhil.goyal@nxp.com
> Cc: stable@dpdk.org
> Subject: RE: [PATCH v5 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline
> crypto
> 
> Hi Bernard,
> Few small comments below.
> Konstantin
> 
> > Inline crypto installs a flow rule in the NIC. This flow rule must be
> > installed before the first inbound packet is received.
> >
> > The create_session() function installs the flow rule,
> > create_session() has been refactored into create_inline_session() and
> > create_lookaside_session(). The create_inline_session() function uses
> > the socket_ctx data and is now called at initialisation in
> > sa_add_rules().
> >
> > The max_session_size() function has been added to calculate memory
> > requirements.
> >
> > The cryprodev_init() function has been refactored to drop calls to
> > rte_mempool_create() and to drop calculation of memory requirements.
> >
> > The main() function has been refactored to call max_session_size() and
> > to call session_pool_init() and session_priv_pool_init() earlier.
> > The ports are started now before adding a flow rule in main().
> > The sa_init(), sp4_init(), sp6_init() and rt_init() functions are now
> > called after the ports have been started.
> >
> > The rte_ipsec_session_prepare() function is called in
> > fill_ipsec_session() for inline which is called from the ipsec_sa_init() function.
> >
> > Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
> > Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample
> > application")
> > Cc: stable@dpdk.org
> >
> > Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> > ---
> >  examples/ipsec-secgw/ipsec-secgw.c   | 244 +++++++++++++++++--------------
> ----
> >  examples/ipsec-secgw/ipsec.c         | 122 ++++++++++++------
> >  examples/ipsec-secgw/ipsec.h         |   5 +-
> >  examples/ipsec-secgw/ipsec_process.c |   9 +-
> >  examples/ipsec-secgw/sa.c            |  46 +++++--
> >  5 files changed, 245 insertions(+), 181 deletions(-)
> >
> > diff --git a/examples/ipsec-secgw/ipsec-secgw.c
> > b/examples/ipsec-secgw/ipsec-secgw.c
> > index 6c626fa..24876ba 100644
> > --- a/examples/ipsec-secgw/ipsec-secgw.c
> > +++ b/examples/ipsec-secgw/ipsec-secgw.c
> > @@ -1628,7 +1628,7 @@ cryptodevs_init(void)
> >  	struct rte_cryptodev_config dev_conf;
> >  	struct rte_cryptodev_qp_conf qp_conf;
> >  	uint16_t idx, max_nb_qps, qp, i;
> > -	int16_t cdev_id, port_id;
> > +	int16_t cdev_id;
> >  	struct rte_hash_parameters params = { 0 };
> >
> >  	params.entries = CDEV_MAP_ENTRIES;
> > @@ -1651,45 +1651,6 @@ cryptodevs_init(void)
> >
> >  	printf("lcore/cryptodev/qp mappings:\n");
> >
> > -	uint32_t max_sess_sz = 0, sess_sz;
> > -	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
> > -		void *sec_ctx;
> > -
> > -		/* Get crypto priv session size */
> > -		sess_sz =
> rte_cryptodev_sym_get_private_session_size(cdev_id);
> > -		if (sess_sz > max_sess_sz)
> > -			max_sess_sz = sess_sz;
> > -
> > -		/*
> > -		 * If crypto device is security capable, need to check the
> > -		 * size of security session as well.
> > -		 */
> > -
> > -		/* Get security context of the crypto device */
> > -		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
> > -		if (sec_ctx == NULL)
> > -			continue;
> > -
> > -		/* Get size of security session */
> > -		sess_sz = rte_security_session_get_size(sec_ctx);
> > -		if (sess_sz > max_sess_sz)
> > -			max_sess_sz = sess_sz;
> > -	}
> > -	RTE_ETH_FOREACH_DEV(port_id) {
> > -		void *sec_ctx;
> > -
> > -		if ((enabled_port_mask & (1 << port_id)) == 0)
> > -			continue;
> > -
> > -		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
> > -		if (sec_ctx == NULL)
> > -			continue;
> > -
> > -		sess_sz = rte_security_session_get_size(sec_ctx);
> > -		if (sess_sz > max_sess_sz)
> > -			max_sess_sz = sess_sz;
> > -	}
> > -
> >  	idx = 0;
> >  	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
> >  		struct rte_cryptodev_info cdev_info; @@ -1727,45 +1688,6
> @@
> > cryptodevs_init(void)
> >  				"Device does not support at least %u "
> >  				"sessions", CDEV_MP_NB_OBJS);
> >
> > -		if (!socket_ctx[dev_conf.socket_id].session_pool) {
> > -			char mp_name[RTE_MEMPOOL_NAMESIZE];
> > -			struct rte_mempool *sess_mp;
> > -
> > -			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> > -					"sess_mp_%u", dev_conf.socket_id);
> > -			sess_mp = rte_cryptodev_sym_session_pool_create(
> > -					mp_name, CDEV_MP_NB_OBJS,
> > -					0, CDEV_MP_CACHE_SZ, 0,
> > -					dev_conf.socket_id);
> > -			socket_ctx[dev_conf.socket_id].session_pool =
> sess_mp;
> > -		}
> > -
> > -		if (!socket_ctx[dev_conf.socket_id].session_priv_pool) {
> > -			char mp_name[RTE_MEMPOOL_NAMESIZE];
> > -			struct rte_mempool *sess_mp;
> > -
> > -			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> > -					"sess_mp_priv_%u",
> dev_conf.socket_id);
> > -			sess_mp = rte_mempool_create(mp_name,
> > -					CDEV_MP_NB_OBJS,
> > -					max_sess_sz,
> > -					CDEV_MP_CACHE_SZ,
> > -					0, NULL, NULL, NULL,
> > -					NULL, dev_conf.socket_id,
> > -					0);
> > -			socket_ctx[dev_conf.socket_id].session_priv_pool =
> > -					sess_mp;
> > -		}
> > -
> > -		if (!socket_ctx[dev_conf.socket_id].session_priv_pool ||
> > -				!socket_ctx[dev_conf.socket_id].session_pool)
> > -			rte_exit(EXIT_FAILURE,
> > -				"Cannot create session pool on socket %d\n",
> > -				dev_conf.socket_id);
> > -		else
> > -			printf("Allocated session pool on socket %d\n",
> > -					dev_conf.socket_id);
> > -
> >  		if (rte_cryptodev_configure(cdev_id, &dev_conf))
> >  			rte_panic("Failed to initialize cryptodev %u\n",
> >  					cdev_id);
> > @@ -1786,39 +1708,6 @@ cryptodevs_init(void)
> >  					cdev_id);
> >  	}
> >
> > -	/* create session pools for eth devices that implement security */
> > -	RTE_ETH_FOREACH_DEV(port_id) {
> > -		if ((enabled_port_mask & (1 << port_id)) &&
> > -				rte_eth_dev_get_sec_ctx(port_id)) {
> > -			int socket_id = rte_eth_dev_socket_id(port_id);
> > -
> > -			if (!socket_ctx[socket_id].session_priv_pool) {
> > -				char mp_name[RTE_MEMPOOL_NAMESIZE];
> > -				struct rte_mempool *sess_mp;
> > -
> > -				snprintf(mp_name,
> RTE_MEMPOOL_NAMESIZE,
> > -						"sess_mp_%u", socket_id);
> > -				sess_mp = rte_mempool_create(mp_name,
> > -						(CDEV_MP_NB_OBJS * 2),
> > -						max_sess_sz,
> > -						CDEV_MP_CACHE_SZ,
> > -						0, NULL, NULL, NULL,
> > -						NULL, socket_id,
> > -						0);
> > -				if (sess_mp == NULL)
> > -					rte_exit(EXIT_FAILURE,
> > -						"Cannot create session pool "
> > -						"on socket %d\n", socket_id);
> > -				else
> > -					printf("Allocated session pool "
> > -						"on socket %d\n", socket_id);
> > -				socket_ctx[socket_id].session_priv_pool =
> > -						sess_mp;
> > -			}
> > -		}
> > -	}
> > -
> > -
> >  	printf("\n");
> >
> >  	return 0;
> > @@ -1984,6 +1873,99 @@ port_init(uint16_t portid, uint64_t
> req_rx_offloads, uint64_t req_tx_offloads)
> >  	printf("\n");
> >  }
> >
> > +static size_t
> > +max_session_size(void)
> > +{
> > +	size_t max_sz, sz;
> > +	void *sec_ctx;
> > +	int16_t cdev_id, port_id, n;
> > +
> > +	max_sz = 0;
> > +	n =  rte_cryptodev_count();
> > +	for (cdev_id = 0; cdev_id != n; cdev_id++) {
> > +		sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
> > +		if (sz > max_sz)
> > +			max_sz = sz;
> > +		/*
> > +		 * If crypto device is security capable, need to check the
> > +		 * size of security session as well.
> > +		 */
> > +
> > +		/* Get security context of the crypto device */
> > +		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
> > +		if (sec_ctx == NULL)
> > +			continue;
> > +
> > +		/* Get size of security session */
> > +		sz = rte_security_session_get_size(sec_ctx);
> > +		if (sz > max_sz)
> > +			max_sz = sz;
> > +	}
> > +
> > +	RTE_ETH_FOREACH_DEV(port_id) {
> > +		if ((enabled_port_mask & (1 << port_id)) == 0)
> > +			continue;
> > +
> > +		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
> > +		if (sec_ctx == NULL)
> > +			continue;
> > +
> > +		sz = rte_security_session_get_size(sec_ctx);
> > +		if (sz > max_sz)
> > +			max_sz = sz;
> > +	}
> > +
> > +	return max_sz;
> > +}
> > +
> > +static void
> > +session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t
> > +sess_sz) {
> > +	char mp_name[RTE_MEMPOOL_NAMESIZE];
> > +	struct rte_mempool *sess_mp;
> > +
> > +	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> > +			"sess_mp_%u", socket_id);
> > +	sess_mp = rte_cryptodev_sym_session_pool_create(
> > +			mp_name, CDEV_MP_NB_OBJS,
> > +			sess_sz, CDEV_MP_CACHE_SZ, 0,
> > +			socket_id);
> > +	ctx->session_pool = sess_mp;
> > +
> > +	if (ctx->session_pool == NULL)
> > +		rte_exit(EXIT_FAILURE,
> > +			"Cannot init session pool on socket %d\n", socket_id);
> > +	else
> > +		printf("Allocated session pool on socket %d\n",	socket_id);
> > +}
> > +
> > +static void
> > +session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
> > +	size_t sess_sz)
> > +{
> > +	char mp_name[RTE_MEMPOOL_NAMESIZE];
> > +	struct rte_mempool *sess_mp;
> > +
> > +	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> > +			"sess_mp_priv_%u", socket_id);
> > +	sess_mp = rte_mempool_create(mp_name,
> > +			CDEV_MP_NB_OBJS,
> > +			sess_sz,
> > +			CDEV_MP_CACHE_SZ,
> > +			0, NULL, NULL, NULL,
> > +			NULL, socket_id,
> > +			0);
> > +	ctx->session_priv_pool = sess_mp;
> > +
> > +	if (ctx->session_priv_pool == NULL)
> > +		rte_exit(EXIT_FAILURE,
> > +			"Cannot init session priv pool on socket %d\n",
> > +			socket_id);
> > +	else
> > +		printf("Allocated session priv pool on socket %d\n",
> > +			socket_id);
> > +}
> > +
> >  static void
> >  pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t
> > nb_mbuf)  { @@ -2064,9 +2046,11 @@ main(int32_t argc, char **argv)  {
> >  	int32_t ret;
> >  	uint32_t lcore_id;
> > +	uint32_t i;
> >  	uint8_t socket_id;
> >  	uint16_t portid;
> >  	uint64_t req_rx_offloads, req_tx_offloads;
> > +	size_t sess_sz;
> >
> >  	/* init EAL */
> >  	ret = rte_eal_init(argc, argv);
> > @@ -2094,7 +2078,8 @@ main(int32_t argc, char **argv)
> >
> >  	nb_lcores = rte_lcore_count();
> >
> > -	/* Replicate each context per socket */
> > +	sess_sz = max_session_size();
> > +
> >  	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
> >  		if (rte_lcore_is_enabled(lcore_id) == 0)
> >  			continue;
> > @@ -2104,20 +2089,14 @@ main(int32_t argc, char **argv)
> >  		else
> >  			socket_id = 0;
> >
> > +		/* mbuf_pool is initialised by the pool_init() function*/
> >  		if (socket_ctx[socket_id].mbuf_pool)
> >  			continue;
> >
> > -		/* initilaze SPD */
> > -		sp4_init(&socket_ctx[socket_id], socket_id);
> > -
> > -		sp6_init(&socket_ctx[socket_id], socket_id);
> > -
> > -		/* initilaze SAD */
> > -		sa_init(&socket_ctx[socket_id], socket_id);
> > -
> > -		rt_init(&socket_ctx[socket_id], socket_id);
> > -
> >  		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
> > +		session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
> > +		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
> > +			sess_sz);
> >  	}
> >
> >  	RTE_ETH_FOREACH_DEV(portid) {
> > @@ -2135,7 +2114,11 @@ main(int32_t argc, char **argv)
> >  		if ((enabled_port_mask & (1 << portid)) == 0)
> >  			continue;
> >
> > -		/* Start device */
> > +		/*
> > +		 * Start device
> > +		 * note: device must be started before a flow rule
> > +		 * can be installed.
> > +		 */
> >  		ret = rte_eth_dev_start(portid);
> >  		if (ret < 0)
> >  			rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
> > @@ -2153,6 +2136,19 @@ main(int32_t argc, char **argv)
> >  			RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback,
> NULL);
> >  	}
> >
> > +	/* Replicate each context per socket */
> > +	for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
> > +		socket_id = rte_socket_id_by_idx(i);
> > +		if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
> > +			(socket_ctx[socket_id].sa_in == NULL) &&
> > +			(socket_ctx[socket_id].sa_out == NULL)) {
> > +			sa_init(&socket_ctx[socket_id], socket_id);
> > +			sp4_init(&socket_ctx[socket_id], socket_id);
> > +			sp6_init(&socket_ctx[socket_id], socket_id);
> > +			rt_init(&socket_ctx[socket_id], socket_id);
> > +		}
> > +	}
> > +
> >  	check_all_ports_link_status(enabled_port_mask);
> >
> >  	/* launch per-lcore init on every lcore */ diff --git
> > a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c index
> > 7b85330..40a6123 100644
> > --- a/examples/ipsec-secgw/ipsec.c
> > +++ b/examples/ipsec-secgw/ipsec.c
> > @@ -40,7 +40,7 @@ set_ipsec_conf(struct ipsec_sa *sa, struct
> > rte_security_ipsec_xform *ipsec)  }
> >
> >  int
> > -create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
> > +create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa
> > +*sa)
> >  {
> >  	struct rte_cryptodev_info cdev_info;
> >  	unsigned long cdev_id_qp = 0;
> 
> 
> Actually looking at it a bit closer:
> 
> if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {
> 
> I think we need to remove that line above, otherwise cdev_id_qp Would not be
> initialized for LOOKASIDE_PROTO.
> BTW, as I can see, same problem exists in current code.

Will do.

>                 ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
>                                 (void **)&cdev_id_qp);
>                 if (ret < 0) {
>                         RTE_LOG(ERR, IPSEC,
>                                 "No cryptodev: core %u, cipher_algo %u, "
>                                 "auth_algo %u, aead_algo %u\n",
>                                 key.lcore_id,
>                                 key.cipher_algo,
>                                 key.auth_algo,
>                                 key.aead_algo);
>                         return -1;
>                 }
>         }
> 
> > @@ -108,23 +108,81 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct
> ipsec_sa *sa)
> >  				"SEC Session init failed: err: %d\n", ret);
> >  				return -1;
> >  			}
> > -		} else if (sa->type ==
> RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
> > +		} else if (
> > +			(sa->type ==
> RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) ||
> > +			(sa->type ==
> RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
> > +			) {
> > +				RTE_LOG(ERR, IPSEC, "Inline not supported\n");
> > +				return -1;
> > +		}
> > +	} else {
> > +		sa->crypto_session = rte_cryptodev_sym_session_create(
> > +				ipsec_ctx->session_pool);
> > +		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
> > +				sa->crypto_session, sa->xforms,
> > +				ipsec_ctx->session_priv_pool);
> > +
> > +		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
> > +				&cdev_info);
> > +	}
> > +
> > +	sa->cdev_id_qp = cdev_id_qp;
> > +
> > +	return 0;
> > +}
> > +
> > +int
> > +create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa
> > +*sa) {
> > +	unsigned long cdev_id_qp = 0;
> 
> This var is not really used in this function.

Will remove cdev_id_qp.
 
> > +	int32_t ret = 0;
> > +	struct rte_security_ctx *sec_ctx;
> > +
> > +	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on port
> %u\n",
> > +		sa->spi, sa->portid);
> > +
> > +	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
> 
> I think this always be true here and could be removed.

Will do.
 
> > +		struct rte_security_session_conf sess_conf = {
> > +			.action_type = sa->type,
> > +			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
> > +			{.ipsec = {
> > +				.spi = sa->spi,
> > +				.salt = sa->salt,
> > +				.options = { 0 },
> > +				.direction = sa->direction,
> > +				.proto =
> RTE_SECURITY_IPSEC_SA_PROTO_ESP,
> > +				.mode = (sa->flags == IP4_TUNNEL ||
> > +						sa->flags == IP6_TUNNEL) ?
> > +
> 	RTE_SECURITY_IPSEC_SA_MODE_TUNNEL :
> > +
> 	RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
> > +			} },
> > +			.crypto_xform = sa->xforms,
> > +			.userdata = NULL,
> > +		};
> > +
> > +		if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
> {
> >  			struct rte_flow_error err;
> > -			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > -
> 	rte_eth_dev_get_sec_ctx(
> > -							sa->portid);
> >  			const struct rte_security_capability *sec_cap;
> >  			int ret = 0;
> >
> > -			sa->sec_session = rte_security_session_create(ctx,
> > -					&sess_conf, ipsec_ctx-
> >session_priv_pool);
> > +			sec_ctx = (struct rte_security_ctx *)
> > +
> 	rte_eth_dev_get_sec_ctx(
> > +							sa->portid);
> > +			if (sec_ctx == NULL) {
> > +				RTE_LOG(ERR, IPSEC,
> > +				" rte_eth_dev_get_sec_ctx failed\n");
> > +				return -1;
> > +			}
> > +
> > +			sa->sec_session = rte_security_session_create(sec_ctx,
> > +					&sess_conf, skt_ctx->session_pool);
> >  			if (sa->sec_session == NULL) {
> >  				RTE_LOG(ERR, IPSEC,
> >  				"SEC Session init failed: err: %d\n", ret);
> >  				return -1;
> >  			}
> >
> > -			sec_cap = rte_security_capabilities_get(ctx);
> > +			sec_cap = rte_security_capabilities_get(sec_ctx);
> >
> >  			/* iterate until ESP tunnel*/
> >  			while (sec_cap->action !=
> > @@ -147,7 +205,7 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct
> ipsec_sa *sa)
> >  			}
> >
> >  			sa->ol_flags = sec_cap->ol_flags;
> > -			sa->security_ctx = ctx;
> > +			sa->security_ctx = sec_ctx;
> >  			sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
> >
> >  			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
> @@ -196,7 +254,7 @@
> > create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
> >  				/* Try RSS. */
> >  				sa->action[1].type =
> RTE_FLOW_ACTION_TYPE_RSS;
> >  				sa->action[1].conf = &action_rss;
> > -				eth_dev = ctx->device;
> > +				eth_dev = sec_ctx->device;
> >  				rte_eth_dev_rss_hash_conf_get(sa->portid,
> >  							      &rss_conf);
> >  				for (i = 0, j = 0;
> > @@ -252,12 +310,12 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct
> ipsec_sa *sa)
> >  			}
> >  		} else if (sa->type ==
> >
> 	RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
> > -			struct rte_security_ctx *ctx =
> > -					(struct rte_security_ctx *)
> > -					rte_eth_dev_get_sec_ctx(sa->portid);
> >  			const struct rte_security_capability *sec_cap;
> >
> > -			if (ctx == NULL) {
> > +			sec_ctx = (struct rte_security_ctx *)
> > +					rte_eth_dev_get_sec_ctx(sa->portid);
> > +
> > +			if (sec_ctx == NULL) {
> >  				RTE_LOG(ERR, IPSEC,
> >  				"Ethernet device doesn't have security features
> registered\n");
> >  				return -1;
> > @@ -279,15 +337,15 @@ create_session(struct ipsec_ctx *ipsec_ctx,
> > struct ipsec_sa *sa)
> >
> >  			sess_conf.userdata = (void *) sa;
> >
> > -			sa->sec_session = rte_security_session_create(ctx,
> > -					&sess_conf, ipsec_ctx->session_pool);
> > +			sa->sec_session = rte_security_session_create(sec_ctx,
> > +					&sess_conf, skt_ctx->session_pool);
> >  			if (sa->sec_session == NULL) {
> >  				RTE_LOG(ERR, IPSEC,
> >  				"SEC Session init failed: err: %d\n", ret);
> >  				return -1;
> >  			}
> >
> > -			sec_cap = rte_security_capabilities_get(ctx);
> > +			sec_cap = rte_security_capabilities_get(sec_ctx);
> >
> >  			if (sec_cap == NULL) {
> >  				RTE_LOG(ERR, IPSEC,
> > @@ -316,17 +374,8 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct
> ipsec_sa *sa)
> >  			}
> >
> >  			sa->ol_flags = sec_cap->ol_flags;
> > -			sa->security_ctx = ctx;
> > +			sa->security_ctx = sec_ctx;
> >  		}
> > -	} else {
> > -		sa->crypto_session = rte_cryptodev_sym_session_create(
> > -				ipsec_ctx->session_pool);
> > -		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
> > -				sa->crypto_session, sa->xforms,
> > -				ipsec_ctx->session_priv_pool);
> > -
> > -		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
> > -				&cdev_info);
> >  	}
> >  	sa->cdev_id_qp = cdev_id_qp;
> >
> > @@ -395,7 +444,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
> ipsec_ctx *ipsec_ctx,
> >  			rte_prefetch0(&priv->sym_cop);
> >
> >  			if ((unlikely(sa->sec_session == NULL)) &&
> > -					create_session(ipsec_ctx, sa)) {
> > +				create_lookaside_session(ipsec_ctx, sa)) {
> >  				rte_pktmbuf_free(pkts[i]);
> >  				continue;
> >  			}
> > @@ -414,7 +463,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
> ipsec_ctx *ipsec_ctx,
> >  			rte_prefetch0(&priv->sym_cop);
> >
> >  			if ((unlikely(sa->crypto_session == NULL)) &&
> > -					create_session(ipsec_ctx, sa)) {
> > +				create_lookaside_session(ipsec_ctx, sa)) {
> >  				rte_pktmbuf_free(pkts[i]);
> >  				continue;
> >  			}
> > @@ -429,12 +478,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
> ipsec_ctx *ipsec_ctx,
> >  			}
> >  			break;
> >  		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
> > -			if ((unlikely(sa->sec_session == NULL)) &&
> > -					create_session(ipsec_ctx, sa)) {
> > -				rte_pktmbuf_free(pkts[i]);
> > -				continue;
> > -			}
> > -
> > +			RTE_ASSERT(sa->sec_session != NULL);
> >  			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
> >  			if (sa->ol_flags &
> RTE_SECURITY_TX_OLOAD_NEED_MDATA)
> >  				rte_security_set_pkt_metadata(
> > @@ -442,17 +486,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
> ipsec_ctx *ipsec_ctx,
> >  						sa->sec_session, pkts[i],
> NULL);
> >  			continue;
> >  		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
> > +			RTE_ASSERT(sa->sec_session != NULL);
> >  			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
> >  			priv->cop.status =
> RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
> >
> >  			rte_prefetch0(&priv->sym_cop);
> > -
> > -			if ((unlikely(sa->sec_session == NULL)) &&
> > -					create_session(ipsec_ctx, sa)) {
> > -				rte_pktmbuf_free(pkts[i]);
> > -				continue;
> > -			}
> > -
> >  			rte_security_attach_session(&priv->cop,
> >  					sa->sec_session);
> >
> > diff --git a/examples/ipsec-secgw/ipsec.h
> > b/examples/ipsec-secgw/ipsec.h index e9272d7..41bac0b 100644
> > --- a/examples/ipsec-secgw/ipsec.h
> > +++ b/examples/ipsec-secgw/ipsec.h
> > @@ -312,6 +312,9 @@ void
> >  enqueue_cop_burst(struct cdev_qp *cqp);
> >
> >  int
> > -create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
> > +create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa
> > +*sa);
> > +
> > +int
> > +create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa
> > +*sa);
> >
> >  #endif /* __IPSEC_H__ */
> > diff --git a/examples/ipsec-secgw/ipsec_process.c
> > b/examples/ipsec-secgw/ipsec_process.c
> > index 3f9cacb..868f1a2 100644
> > --- a/examples/ipsec-secgw/ipsec_process.c
> > +++ b/examples/ipsec-secgw/ipsec_process.c
> > @@ -95,22 +95,23 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct
> ipsec_ctx *ctx,
> >  	/* setup crypto section */
> >  	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
> >  		if (sa->crypto_session == NULL) {
> > -			rc = create_session(ctx, sa);
> > +			rc = create_lookaside_session(ctx, sa);
> >  			if (rc != 0)
> >  				return rc;
> >  		}
> >  		ss->crypto.ses = sa->crypto_session;
> >  	/* setup session action type */
> > -	} else {
> > +	} else if (sa->type ==
> RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL)
> > +{
> >  		if (sa->sec_session == NULL) {
> > -			rc = create_session(ctx, sa);
> > +			rc = create_lookaside_session(ctx, sa);
> >  			if (rc != 0)
> >  				return rc;
> >  		}
> >  		ss->security.ses = sa->sec_session;
> >  		ss->security.ctx = sa->security_ctx;
> >  		ss->security.ol_flags = sa->ol_flags;
> > -	}
> > +	} else
> > +		RTE_ASSERT(0);
> >
> >  	rc = rte_ipsec_session_prepare(ss);
> >  	if (rc != 0)
> > diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
> > index 8d47d1d..e8e55bf 100644
> > --- a/examples/ipsec-secgw/sa.c
> > +++ b/examples/ipsec-secgw/sa.c
> > @@ -777,11 +777,13 @@ check_eth_dev_caps(uint16_t portid, uint32_t
> > inbound)
> >
> >  static int
> >  sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> > -		uint32_t nb_entries, uint32_t inbound)
> > +		uint32_t nb_entries, uint32_t inbound,
> > +		struct socket_ctx *skt_ctx)
> >  {
> >  	struct ipsec_sa *sa;
> >  	uint32_t i, idx;
> >  	uint16_t iv_length, aad_length;
> > +	int32_t rc;
> >
> >  	/* for ESN upper 32 bits of SQN also need to be part of AAD */
> >  	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0; @@
> > -834,6 +836,17 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct
> > ipsec_sa entries[],
> >
> >  			sa->xforms = &sa_ctx->xf[idx].a;
> >
> > +			if (sa->type ==
> > +
> 	RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
> > +				sa->type ==
> > +
> 	RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
> > +				rc = create_inline_session(skt_ctx, sa);
> > +				if (rc != 0) {
> > +					RTE_LOG(ERR, IPSEC_ESP,
> > +						"create_inline_session()
> failed\n");
> > +					return -EINVAL;
> > +				}
> > +			}
> >  			print_one_sa_rule(sa, inbound);
> >  		} else {
> >  			switch (sa->cipher_algo) {
> > @@ -909,16 +922,16 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct
> > ipsec_sa entries[],
> >
> >  static inline int
> >  sa_out_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> > -		uint32_t nb_entries)
> > +		uint32_t nb_entries, struct socket_ctx *skt_ctx)
> >  {
> > -	return sa_add_rules(sa_ctx, entries, nb_entries, 0);
> > +	return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx);
> >  }
> >
> >  static inline int
> >  sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> > -		uint32_t nb_entries)
> > +		uint32_t nb_entries, struct socket_ctx *skt_ctx)
> >  {
> > -	return sa_add_rules(sa_ctx, entries, nb_entries, 1);
> > +	return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx);
> >  }
> >
> >  /*
> > @@ -1012,10 +1025,12 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm,
> const struct ipsec_sa *ss,
> >  	return 0;
> >  }
> >
> > -static void
> > +static int
> >  fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
> >  	const struct ipsec_sa *lsa)
> >  {
> > +	int32_t rc = 0;
> > +
> >  	ss->sa = sa;
> >  	ss->type = lsa->type;
> >
> > @@ -1028,6 +1043,17 @@ fill_ipsec_session(struct rte_ipsec_session *ss,
> struct rte_ipsec_sa *sa,
> >  		ss->security.ctx = lsa->security_ctx;
> >  		ss->security.ol_flags = lsa->ol_flags;
> >  	}
> > +
> > +	if (ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
> > +		ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
> {
> > +		if (ss->security.ses != NULL) {
> > +			rc = rte_ipsec_session_prepare(ss);
> > +			if (rc != 0)
> > +				memset(ss, 0, sizeof(*ss));
> > +		}
> > +	}
> > +
> > +	return rc;
> >  }
> >
> >  /*
> > @@ -1062,8 +1088,8 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct
> rte_ipsec_sa *sa, uint32_t sa_size)
> >  	if (rc < 0)
> >  		return rc;
> >
> > -	fill_ipsec_session(&lsa->ips, sa, lsa);
> > -	return 0;
> > +	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
> > +	return rc;
> >  }
> >
> >  /*
> > @@ -1166,7 +1192,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
> >  				"context %s in socket %d\n", rte_errno,
> >  				name, socket_id);
> >
> > -		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in);
> > +		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx);
> >
> >  		if (app_sa_prm.enable != 0) {
> >  			rc = ipsec_satbl_init(ctx->sa_in, sa_in, nb_sa_in, @@ -
> 1186,7
> > +1212,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
> >  				"context %s in socket %d\n", rte_errno,
> >  				name, socket_id);
> >
> > -		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out);
> > +		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx);
> >
> >  		if (app_sa_prm.enable != 0) {
> >  			rc = ipsec_satbl_init(ctx->sa_out, sa_out, nb_sa_out,
> > --
> > 2.7.4

Thanks for the review.

Regards,

Bernard.

[dpdk-dev] [PATCH v6 0/2] examples/ipsec-secgw: fix 1st pkt dropped

[Bernard Iremonger]

This patchset fixes the issue of the first inbound packet
being dropped for inline crypto.

Changes in v6:
--------------
Rebased to latest master.
Minor changes to the following functions in ipsec.c:
create_lookaside_session()
create_inline_session()

Bernard Iremonger (2):
  examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
  examples/ipsec-secgw/test: fix inline test scripts

 examples/ipsec-secgw/ipsec-secgw.c           | 244 +++++++--------
 examples/ipsec-secgw/ipsec.c                 | 449 ++++++++++++++-------------
 examples/ipsec-secgw/ipsec.h                 |   5 +-
 examples/ipsec-secgw/ipsec_process.c         |   9 +-
 examples/ipsec-secgw/sa.c                    |  46 ++-
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 -
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 -
 7 files changed, 403 insertions(+), 370 deletions(-)

Changes in v5: 
-------------
The v2 patchset has been rebased to the latest master.
The v4 patchset has been dropped as it caused issues with the lookaside code
which we are unable to test.
 
Bernard Iremonger (2):
  examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
  examples/ipsec-secgw/test: fix inline test scripts

 examples/ipsec-secgw/ipsec-secgw.c           | 244 +++++++++++++--------------
 examples/ipsec-secgw/ipsec.c                 | 122 +++++++++-----
 examples/ipsec-secgw/ipsec.h                 |   5 +-
 examples/ipsec-secgw/ipsec_process.c         |   9 +-
 examples/ipsec-secgw/sa.c                    |  46 +++--
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 --
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 --
 7 files changed, 245 insertions(+), 201 deletions(-)

Changes in v2: 
-------------
The first three patches of the v1 have been squashed.
The commit message for the squashed patch has been updated.
Patches 4,5 and 6 of the v1 have been dropped from this patchset.
A patch to fix the test scripts has been added.

Bernard Iremonger (2):
  examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
  examples/ipsec-secgw/test: fix inline test scripts

 examples/ipsec-secgw/ipsec-secgw.c           | 243 +++++++++++++--------------
 examples/ipsec-secgw/ipsec.c                 | 123 +++++++++-----
 examples/ipsec-secgw/ipsec.h                 |   5 +-
 examples/ipsec-secgw/ipsec_process.c         |   9 +-
 examples/ipsec-secgw/sa.c                    |  46 +++--
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 --
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 --
 7 files changed, 246 insertions(+), 200 deletions(-)


-- 
2.7.4

[dpdk-dev] [PATCH v6 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Bernard Iremonger]

Inline crypto installs a flow rule in the NIC. This flow
rule must be installed before the first inbound packet is
received.

The create_session() function installs the flow rule,
create_session() has been refactored into create_inline_session()
and create_lookaside_session(). The create_inline_session() function
uses the socket_ctx data and is now called at initialisation in
sa_add_rules().

The max_session_size() function has been added to calculate memory
requirements.

The cryprodev_init() function has been refactored to drop calls to
rte_mempool_create() and to drop calculation of memory requirements.

The main() function has been refactored to call max_session_size() and
to call session_pool_init() and session_priv_pool_init() earlier.
The ports are started now before adding a flow rule in main().
The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
now called after the ports have been started.

The rte_ipsec_session_prepare() function is called in fill_ipsec_session()
for inline which is called from the ipsec_sa_init() function.

Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/ipsec-secgw.c   | 244 ++++++++++---------
 examples/ipsec-secgw/ipsec.c         | 449 +++++++++++++++++++----------------
 examples/ipsec-secgw/ipsec.h         |   5 +-
 examples/ipsec-secgw/ipsec_process.c |   9 +-
 examples/ipsec-secgw/sa.c            |  46 +++-
 5 files changed, 403 insertions(+), 350 deletions(-)

diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index 6c626fa..24876ba 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -1628,7 +1628,7 @@ cryptodevs_init(void)
 	struct rte_cryptodev_config dev_conf;
 	struct rte_cryptodev_qp_conf qp_conf;
 	uint16_t idx, max_nb_qps, qp, i;
-	int16_t cdev_id, port_id;
+	int16_t cdev_id;
 	struct rte_hash_parameters params = { 0 };
 
 	params.entries = CDEV_MAP_ENTRIES;
@@ -1651,45 +1651,6 @@ cryptodevs_init(void)
 
 	printf("lcore/cryptodev/qp mappings:\n");
 
-	uint32_t max_sess_sz = 0, sess_sz;
-	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
-		void *sec_ctx;
-
-		/* Get crypto priv session size */
-		sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-
-		/*
-		 * If crypto device is security capable, need to check the
-		 * size of security session as well.
-		 */
-
-		/* Get security context of the crypto device */
-		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		/* Get size of security session */
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-	RTE_ETH_FOREACH_DEV(port_id) {
-		void *sec_ctx;
-
-		if ((enabled_port_mask & (1 << port_id)) == 0)
-			continue;
-
-		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-
 	idx = 0;
 	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
 		struct rte_cryptodev_info cdev_info;
@@ -1727,45 +1688,6 @@ cryptodevs_init(void)
 				"Device does not support at least %u "
 				"sessions", CDEV_MP_NB_OBJS);
 
-		if (!socket_ctx[dev_conf.socket_id].session_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_%u", dev_conf.socket_id);
-			sess_mp = rte_cryptodev_sym_session_pool_create(
-					mp_name, CDEV_MP_NB_OBJS,
-					0, CDEV_MP_CACHE_SZ, 0,
-					dev_conf.socket_id);
-			socket_ctx[dev_conf.socket_id].session_pool = sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_priv_%u", dev_conf.socket_id);
-			sess_mp = rte_mempool_create(mp_name,
-					CDEV_MP_NB_OBJS,
-					max_sess_sz,
-					CDEV_MP_CACHE_SZ,
-					0, NULL, NULL, NULL,
-					NULL, dev_conf.socket_id,
-					0);
-			socket_ctx[dev_conf.socket_id].session_priv_pool =
-					sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool ||
-				!socket_ctx[dev_conf.socket_id].session_pool)
-			rte_exit(EXIT_FAILURE,
-				"Cannot create session pool on socket %d\n",
-				dev_conf.socket_id);
-		else
-			printf("Allocated session pool on socket %d\n",
-					dev_conf.socket_id);
-
 		if (rte_cryptodev_configure(cdev_id, &dev_conf))
 			rte_panic("Failed to initialize cryptodev %u\n",
 					cdev_id);
@@ -1786,39 +1708,6 @@ cryptodevs_init(void)
 					cdev_id);
 	}
 
-	/* create session pools for eth devices that implement security */
-	RTE_ETH_FOREACH_DEV(port_id) {
-		if ((enabled_port_mask & (1 << port_id)) &&
-				rte_eth_dev_get_sec_ctx(port_id)) {
-			int socket_id = rte_eth_dev_socket_id(port_id);
-
-			if (!socket_ctx[socket_id].session_priv_pool) {
-				char mp_name[RTE_MEMPOOL_NAMESIZE];
-				struct rte_mempool *sess_mp;
-
-				snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-						"sess_mp_%u", socket_id);
-				sess_mp = rte_mempool_create(mp_name,
-						(CDEV_MP_NB_OBJS * 2),
-						max_sess_sz,
-						CDEV_MP_CACHE_SZ,
-						0, NULL, NULL, NULL,
-						NULL, socket_id,
-						0);
-				if (sess_mp == NULL)
-					rte_exit(EXIT_FAILURE,
-						"Cannot create session pool "
-						"on socket %d\n", socket_id);
-				else
-					printf("Allocated session pool "
-						"on socket %d\n", socket_id);
-				socket_ctx[socket_id].session_priv_pool =
-						sess_mp;
-			}
-		}
-	}
-
-
 	printf("\n");
 
 	return 0;
@@ -1984,6 +1873,99 @@ port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads)
 	printf("\n");
 }
 
+static size_t
+max_session_size(void)
+{
+	size_t max_sz, sz;
+	void *sec_ctx;
+	int16_t cdev_id, port_id, n;
+
+	max_sz = 0;
+	n =  rte_cryptodev_count();
+	for (cdev_id = 0; cdev_id != n; cdev_id++) {
+		sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
+		if (sz > max_sz)
+			max_sz = sz;
+		/*
+		 * If crypto device is security capable, need to check the
+		 * size of security session as well.
+		 */
+
+		/* Get security context of the crypto device */
+		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		/* Get size of security session */
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	RTE_ETH_FOREACH_DEV(port_id) {
+		if ((enabled_port_mask & (1 << port_id)) == 0)
+			continue;
+
+		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	return max_sz;
+}
+
+static void
+session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_%u", socket_id);
+	sess_mp = rte_cryptodev_sym_session_pool_create(
+			mp_name, CDEV_MP_NB_OBJS,
+			sess_sz, CDEV_MP_CACHE_SZ, 0,
+			socket_id);
+	ctx->session_pool = sess_mp;
+
+	if (ctx->session_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session pool on socket %d\n", socket_id);
+	else
+		printf("Allocated session pool on socket %d\n",	socket_id);
+}
+
+static void
+session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
+	size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_priv_%u", socket_id);
+	sess_mp = rte_mempool_create(mp_name,
+			CDEV_MP_NB_OBJS,
+			sess_sz,
+			CDEV_MP_CACHE_SZ,
+			0, NULL, NULL, NULL,
+			NULL, socket_id,
+			0);
+	ctx->session_priv_pool = sess_mp;
+
+	if (ctx->session_priv_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session priv pool on socket %d\n",
+			socket_id);
+	else
+		printf("Allocated session priv pool on socket %d\n",
+			socket_id);
+}
+
 static void
 pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf)
 {
@@ -2064,9 +2046,11 @@ main(int32_t argc, char **argv)
 {
 	int32_t ret;
 	uint32_t lcore_id;
+	uint32_t i;
 	uint8_t socket_id;
 	uint16_t portid;
 	uint64_t req_rx_offloads, req_tx_offloads;
+	size_t sess_sz;
 
 	/* init EAL */
 	ret = rte_eal_init(argc, argv);
@@ -2094,7 +2078,8 @@ main(int32_t argc, char **argv)
 
 	nb_lcores = rte_lcore_count();
 
-	/* Replicate each context per socket */
+	sess_sz = max_session_size();
+
 	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
 		if (rte_lcore_is_enabled(lcore_id) == 0)
 			continue;
@@ -2104,20 +2089,14 @@ main(int32_t argc, char **argv)
 		else
 			socket_id = 0;
 
+		/* mbuf_pool is initialised by the pool_init() function*/
 		if (socket_ctx[socket_id].mbuf_pool)
 			continue;
 
-		/* initilaze SPD */
-		sp4_init(&socket_ctx[socket_id], socket_id);
-
-		sp6_init(&socket_ctx[socket_id], socket_id);
-
-		/* initilaze SAD */
-		sa_init(&socket_ctx[socket_id], socket_id);
-
-		rt_init(&socket_ctx[socket_id], socket_id);
-
 		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
+		session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
+		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
+			sess_sz);
 	}
 
 	RTE_ETH_FOREACH_DEV(portid) {
@@ -2135,7 +2114,11 @@ main(int32_t argc, char **argv)
 		if ((enabled_port_mask & (1 << portid)) == 0)
 			continue;
 
-		/* Start device */
+		/*
+		 * Start device
+		 * note: device must be started before a flow rule
+		 * can be installed.
+		 */
 		ret = rte_eth_dev_start(portid);
 		if (ret < 0)
 			rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
@@ -2153,6 +2136,19 @@ main(int32_t argc, char **argv)
 			RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback, NULL);
 	}
 
+	/* Replicate each context per socket */
+	for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
+		socket_id = rte_socket_id_by_idx(i);
+		if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
+			(socket_ctx[socket_id].sa_in == NULL) &&
+			(socket_ctx[socket_id].sa_out == NULL)) {
+			sa_init(&socket_ctx[socket_id], socket_id);
+			sp4_init(&socket_ctx[socket_id], socket_id);
+			sp6_init(&socket_ctx[socket_id], socket_id);
+			rt_init(&socket_ctx[socket_id], socket_id);
+		}
+	}
+
 	check_all_ports_link_status(enabled_port_mask);
 
 	/* launch per-lcore init on every lcore */
diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index 7b85330..c06ddd6 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -40,7 +40,7 @@ set_ipsec_conf(struct ipsec_sa *sa, struct rte_security_ipsec_xform *ipsec)
 }
 
 int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 {
 	struct rte_cryptodev_info cdev_info;
 	unsigned long cdev_id_qp = 0;
@@ -53,19 +53,17 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 	key.auth_algo = (uint8_t)sa->auth_algo;
 	key.aead_algo = (uint8_t)sa->aead_algo;
 
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
-				(void **)&cdev_id_qp);
-		if (ret < 0) {
-			RTE_LOG(ERR, IPSEC,
+	ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
+			(void **)&cdev_id_qp);
+	if (ret < 0) {
+		RTE_LOG(ERR, IPSEC,
 				"No cryptodev: core %u, cipher_algo %u, "
 				"auth_algo %u, aead_algo %u\n",
 				key.lcore_id,
 				key.cipher_algo,
 				key.auth_algo,
 				key.aead_algo);
-			return -1;
-		}
+		return -1;
 	}
 
 	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on cryptodev "
@@ -108,227 +106,267 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 				"SEC Session init failed: err: %d\n", ret);
 				return -1;
 			}
-		} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
-			struct rte_flow_error err;
-			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
-							rte_eth_dev_get_sec_ctx(
-							sa->portid);
-			const struct rte_security_capability *sec_cap;
-			int ret = 0;
-
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_priv_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"SEC Session init failed: err: %d\n", ret);
+		} else if (
+			(sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) ||
+			(sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
+			) {
+				RTE_LOG(ERR, IPSEC, "Inline not supported\n");
 				return -1;
-			}
-
-			sec_cap = rte_security_capabilities_get(ctx);
+		}
+	} else {
+		sa->crypto_session = rte_cryptodev_sym_session_create(
+				ipsec_ctx->session_pool);
+		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
+				sa->crypto_session, sa->xforms,
+				ipsec_ctx->session_priv_pool);
 
-			/* iterate until ESP tunnel*/
-			while (sec_cap->action !=
-					RTE_SECURITY_ACTION_TYPE_NONE) {
+		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
+				&cdev_info);
+	}
 
-				if (sec_cap->action == sa->type &&
-				    sec_cap->protocol ==
-					RTE_SECURITY_PROTOCOL_IPSEC &&
-				    sec_cap->ipsec.mode ==
-					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
-				    sec_cap->ipsec.direction == sa->direction)
-					break;
-				sec_cap++;
-			}
+	sa->cdev_id_qp = cdev_id_qp;
 
-			if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
-				RTE_LOG(ERR, IPSEC,
-				"No suitable security capability found\n");
-				return -1;
-			}
+	return 0;
+}
 
-			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
-			sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
-
-			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
-			sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
-			if (sa->flags & IP6_TUNNEL) {
-				sa->pattern[1].spec = &sa->ipv6_spec;
-				memcpy(sa->ipv6_spec.hdr.dst_addr,
-					sa->dst.ip.ip6.ip6_b, 16);
-				memcpy(sa->ipv6_spec.hdr.src_addr,
-				       sa->src.ip.ip6.ip6_b, 16);
-			} else {
-				sa->pattern[1].spec = &sa->ipv4_spec;
-				sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
-				sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
-			}
+int
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
+{
+	int32_t ret = 0;
+	struct rte_security_ctx *sec_ctx;
+	struct rte_security_session_conf sess_conf = {
+		.action_type = sa->type,
+		.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
+		{.ipsec = {
+			.spi = sa->spi,
+			.salt = sa->salt,
+			.options = { 0 },
+			.direction = sa->direction,
+			.proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
+			.mode = (sa->flags == IP4_TUNNEL ||
+					sa->flags == IP6_TUNNEL) ?
+					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL :
+					RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
+		} },
+		.crypto_xform = sa->xforms,
+		.userdata = NULL,
+	};
+
+	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on port %u\n",
+		sa->spi, sa->portid);
+
+	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		struct rte_flow_error err;
+		const struct rte_security_capability *sec_cap;
+		int ret = 0;
+
+		sec_ctx = (struct rte_security_ctx *)
+					rte_eth_dev_get_sec_ctx(
+					sa->portid);
+		if (sec_ctx == NULL) {
+			RTE_LOG(ERR, IPSEC,
+				" rte_eth_dev_get_sec_ctx failed\n");
+			return -1;
+		}
 
-			sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
-			sa->pattern[2].spec = &sa->esp_spec;
-			sa->pattern[2].mask = &rte_flow_item_esp_mask;
-			sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
+		sa->sec_session = rte_security_session_create(sec_ctx,
+				&sess_conf, skt_ctx->session_pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC,
+				"SEC Session init failed: err: %d\n", ret);
+			return -1;
+		}
 
-			sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
+		sec_cap = rte_security_capabilities_get(sec_ctx);
+
+		/* iterate until ESP tunnel*/
+		while (sec_cap->action != RTE_SECURITY_ACTION_TYPE_NONE) {
+			if (sec_cap->action == sa->type &&
+			    sec_cap->protocol ==
+				RTE_SECURITY_PROTOCOL_IPSEC &&
+			    sec_cap->ipsec.mode ==
+				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
+			    sec_cap->ipsec.direction == sa->direction)
+				break;
+			sec_cap++;
+		}
 
-			sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
-			sa->action[0].conf = sa->sec_session;
+		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
+			RTE_LOG(ERR, IPSEC,
+				"No suitable security capability found\n");
+			return -1;
+		}
 
-			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
+		sa->ol_flags = sec_cap->ol_flags;
+		sa->security_ctx = sec_ctx;
+		sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
+
+		sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
+		sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
+		if (sa->flags & IP6_TUNNEL) {
+			sa->pattern[1].spec = &sa->ipv6_spec;
+			memcpy(sa->ipv6_spec.hdr.dst_addr,
+				sa->dst.ip.ip6.ip6_b, 16);
+			memcpy(sa->ipv6_spec.hdr.src_addr,
+			       sa->src.ip.ip6.ip6_b, 16);
+		} else {
+			sa->pattern[1].spec = &sa->ipv4_spec;
+			sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
+			sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
+		}
 
-			sa->attr.egress = (sa->direction ==
-					RTE_SECURITY_IPSEC_SA_DIR_EGRESS);
-			sa->attr.ingress = (sa->direction ==
-					RTE_SECURITY_IPSEC_SA_DIR_INGRESS);
-			if (sa->attr.ingress) {
-				uint8_t rss_key[40];
-				struct rte_eth_rss_conf rss_conf = {
-					.rss_key = rss_key,
-					.rss_key_len = 40,
-				};
-				struct rte_eth_dev *eth_dev;
-				uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
-				struct rte_flow_action_rss action_rss;
-				unsigned int i;
-				unsigned int j;
-
-				sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
-				/* Try RSS. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
-				sa->action[1].conf = &action_rss;
-				eth_dev = ctx->device;
-				rte_eth_dev_rss_hash_conf_get(sa->portid,
-							      &rss_conf);
-				for (i = 0, j = 0;
-				     i < eth_dev->data->nb_rx_queues; ++i)
-					if (eth_dev->data->rx_queues[i])
-						queue[j++] = i;
-				action_rss = (struct rte_flow_action_rss){
+		sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
+		sa->pattern[2].spec = &sa->esp_spec;
+		sa->pattern[2].mask = &rte_flow_item_esp_mask;
+		sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
+
+		sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
+
+		sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
+		sa->action[0].conf = sa->sec_session;
+
+		sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
+
+		sa->attr.egress = (sa->direction ==
+				RTE_SECURITY_IPSEC_SA_DIR_EGRESS);
+		sa->attr.ingress = (sa->direction ==
+				RTE_SECURITY_IPSEC_SA_DIR_INGRESS);
+		if (sa->attr.ingress) {
+			uint8_t rss_key[40];
+			struct rte_eth_rss_conf rss_conf = {
+				.rss_key = rss_key,
+				.rss_key_len = 40,
+			};
+			struct rte_eth_dev *eth_dev;
+			uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
+			struct rte_flow_action_rss action_rss;
+			unsigned int i;
+			unsigned int j;
+
+			sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
+			/* Try RSS. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
+			sa->action[1].conf = &action_rss;
+			eth_dev = sec_ctx->device;
+			rte_eth_dev_rss_hash_conf_get(sa->portid, &rss_conf);
+			for (i = 0, j = 0;
+			     i < eth_dev->data->nb_rx_queues; ++i)
+				if (eth_dev->data->rx_queues[i])
+					queue[j++] = i;
+
+			action_rss = (struct rte_flow_action_rss){
 					.types = rss_conf.rss_hf,
 					.key_len = rss_conf.rss_key_len,
 					.queue_num = j,
 					.key = rss_key,
 					.queue = queue,
-				};
-				ret = rte_flow_validate(sa->portid, &sa->attr,
-							sa->pattern, sa->action,
-							&err);
-				if (!ret)
-					goto flow_create;
-				/* Try Queue. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_QUEUE;
-				sa->action[1].conf =
-					&(struct rte_flow_action_queue){
-					.index = 0,
-				};
-				ret = rte_flow_validate(sa->portid, &sa->attr,
-							sa->pattern, sa->action,
-							&err);
-				/* Try End. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
-				sa->action[1].conf = NULL;
-				ret = rte_flow_validate(sa->portid, &sa->attr,
-							sa->pattern, sa->action,
-							&err);
-				if (ret)
-					goto flow_create_failure;
-			} else if (sa->attr.egress &&
-				   (sa->ol_flags &
+			};
+			ret = rte_flow_validate(sa->portid, &sa->attr,
+						sa->pattern, sa->action,
+						&err);
+			if (!ret)
+				goto flow_create;
+			/* Try Queue. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_QUEUE;
+			sa->action[1].conf =
+				&(struct rte_flow_action_queue){
+				.index = 0,
+			};
+			ret = rte_flow_validate(sa->portid, &sa->attr,
+						sa->pattern, sa->action,
+						&err);
+			/* Try End. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
+			sa->action[1].conf = NULL;
+			ret = rte_flow_validate(sa->portid, &sa->attr,
+						sa->pattern, sa->action,
+						&err);
+			if (ret)
+				goto flow_create_failure;
+		} else if (sa->attr.egress &&
+			   (sa->ol_flags &
 				    RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
-				sa->action[1].type =
+			sa->action[1].type =
 					RTE_FLOW_ACTION_TYPE_PASSTHRU;
-				sa->action[2].type =
+			sa->action[2].type =
 					RTE_FLOW_ACTION_TYPE_END;
-			}
+		}
 flow_create:
-			sa->flow = rte_flow_create(sa->portid,
+		sa->flow = rte_flow_create(sa->portid,
 				&sa->attr, sa->pattern, sa->action, &err);
-			if (sa->flow == NULL) {
+		if (sa->flow == NULL) {
 flow_create_failure:
-				RTE_LOG(ERR, IPSEC,
-					"Failed to create ipsec flow msg: %s\n",
-					err.message);
-				return -1;
-			}
-		} else if (sa->type ==
-				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
-			struct rte_security_ctx *ctx =
-					(struct rte_security_ctx *)
-					rte_eth_dev_get_sec_ctx(sa->portid);
-			const struct rte_security_capability *sec_cap;
-
-			if (ctx == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"Ethernet device doesn't have security features registered\n");
-				return -1;
-			}
-
-			/* Set IPsec parameters in conf */
-			set_ipsec_conf(sa, &(sess_conf.ipsec));
+			RTE_LOG(ERR, IPSEC,
+				"Failed to create ipsec flow msg: %s\n",
+				err.message);
+			return -1;
+		}
+	} else if (sa->type ==	RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
+		const struct rte_security_capability *sec_cap;
 
-			/* Save SA as userdata for the security session. When
-			 * the packet is received, this userdata will be
-			 * retrieved using the metadata from the packet.
-			 *
-			 * The PMD is expected to set similar metadata for other
-			 * operations, like rte_eth_event, which are tied to
-			 * security session. In such cases, the userdata could
-			 * be obtained to uniquely identify the security
-			 * parameters denoted.
-			 */
+		sec_ctx = (struct rte_security_ctx *)
+				rte_eth_dev_get_sec_ctx(sa->portid);
 
-			sess_conf.userdata = (void *) sa;
+		if (sec_ctx == NULL) {
+			RTE_LOG(ERR, IPSEC,
+				"Ethernet device doesn't have security features registered\n");
+			return -1;
+		}
 
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
+		/* Set IPsec parameters in conf */
+		set_ipsec_conf(sa, &(sess_conf.ipsec));
+
+		/* Save SA as userdata for the security session. When
+		 * the packet is received, this userdata will be
+		 * retrieved using the metadata from the packet.
+		 *
+		 * The PMD is expected to set similar metadata for other
+		 * operations, like rte_eth_event, which are tied to
+		 * security session. In such cases, the userdata could
+		 * be obtained to uniquely identify the security
+		 * parameters denoted.
+		 */
+
+		sess_conf.userdata = (void *) sa;
+
+		sa->sec_session = rte_security_session_create(sec_ctx,
+					&sess_conf, skt_ctx->session_pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
-				return -1;
-			}
-
-			sec_cap = rte_security_capabilities_get(ctx);
+			return -1;
+		}
 
-			if (sec_cap == NULL) {
-				RTE_LOG(ERR, IPSEC,
+		sec_cap = rte_security_capabilities_get(sec_ctx);
+		if (sec_cap == NULL) {
+			RTE_LOG(ERR, IPSEC,
 				"No capabilities registered\n");
-				return -1;
-			}
+			return -1;
+		}
 
-			/* iterate until ESP tunnel*/
-			while (sec_cap->action !=
-					RTE_SECURITY_ACTION_TYPE_NONE) {
-
-				if (sec_cap->action == sa->type &&
-				    sec_cap->protocol ==
-					RTE_SECURITY_PROTOCOL_IPSEC &&
-				    sec_cap->ipsec.mode ==
-					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
-				    sec_cap->ipsec.direction == sa->direction)
-					break;
-				sec_cap++;
-			}
+		/* iterate until ESP tunnel*/
+		while (sec_cap->action != RTE_SECURITY_ACTION_TYPE_NONE) {
+			if (sec_cap->action == sa->type &&
+			    sec_cap->protocol ==
+				RTE_SECURITY_PROTOCOL_IPSEC &&
+			    sec_cap->ipsec.mode ==
+				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
+			    sec_cap->ipsec.direction == sa->direction)
+				break;
+			sec_cap++;
+		}
 
-			if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
-				RTE_LOG(ERR, IPSEC,
+		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
+			RTE_LOG(ERR, IPSEC,
 				"No suitable security capability found\n");
-				return -1;
-			}
-
-			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
+			return -1;
 		}
-	} else {
-		sa->crypto_session = rte_cryptodev_sym_session_create(
-				ipsec_ctx->session_pool);
-		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
-				sa->crypto_session, sa->xforms,
-				ipsec_ctx->session_priv_pool);
 
-		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
-				&cdev_info);
+		sa->ol_flags = sec_cap->ol_flags;
+		sa->security_ctx = sec_ctx;
 	}
-	sa->cdev_id_qp = cdev_id_qp;
+
+	sa->cdev_id_qp = 0;
 
 	return 0;
 }
@@ -395,7 +433,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			rte_prefetch0(&priv->sym_cop);
 
 			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
+				create_lookaside_session(ipsec_ctx, sa)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -414,7 +452,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			rte_prefetch0(&priv->sym_cop);
 
 			if ((unlikely(sa->crypto_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
+				create_lookaside_session(ipsec_ctx, sa)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -429,12 +467,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 			break;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
+			RTE_ASSERT(sa->sec_session != NULL);
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
 			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
@@ -442,17 +475,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 						sa->sec_session, pkts[i], NULL);
 			continue;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
+			RTE_ASSERT(sa->sec_session != NULL);
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			rte_security_attach_session(&priv->cop,
 					sa->sec_session);
 
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index e9272d7..41bac0b 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -312,6 +312,9 @@ void
 enqueue_cop_burst(struct cdev_qp *cqp);
 
 int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+
+int
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa);
 
 #endif /* __IPSEC_H__ */
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index 3f9cacb..868f1a2 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -95,22 +95,23 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
 	/* setup crypto section */
 	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
 		if (sa->crypto_session == NULL) {
-			rc = create_session(ctx, sa);
+			rc = create_lookaside_session(ctx, sa);
 			if (rc != 0)
 				return rc;
 		}
 		ss->crypto.ses = sa->crypto_session;
 	/* setup session action type */
-	} else {
+	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
 		if (sa->sec_session == NULL) {
-			rc = create_session(ctx, sa);
+			rc = create_lookaside_session(ctx, sa);
 			if (rc != 0)
 				return rc;
 		}
 		ss->security.ses = sa->sec_session;
 		ss->security.ctx = sa->security_ctx;
 		ss->security.ol_flags = sa->ol_flags;
-	}
+	} else
+		RTE_ASSERT(0);
 
 	rc = rte_ipsec_session_prepare(ss);
 	if (rc != 0)
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index 8d47d1d..e8e55bf 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -777,11 +777,13 @@ check_eth_dev_caps(uint16_t portid, uint32_t inbound)
 
 static int
 sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries, uint32_t inbound)
+		uint32_t nb_entries, uint32_t inbound,
+		struct socket_ctx *skt_ctx)
 {
 	struct ipsec_sa *sa;
 	uint32_t i, idx;
 	uint16_t iv_length, aad_length;
+	int32_t rc;
 
 	/* for ESN upper 32 bits of SQN also need to be part of AAD */
 	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
@@ -834,6 +836,17 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			sa->xforms = &sa_ctx->xf[idx].a;
 
+			if (sa->type ==
+				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
+				sa->type ==
+				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+				rc = create_inline_session(skt_ctx, sa);
+				if (rc != 0) {
+					RTE_LOG(ERR, IPSEC_ESP,
+						"create_inline_session() failed\n");
+					return -EINVAL;
+				}
+			}
 			print_one_sa_rule(sa, inbound);
 		} else {
 			switch (sa->cipher_algo) {
@@ -909,16 +922,16 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 static inline int
 sa_out_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 0);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx);
 }
 
 static inline int
 sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 1);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx);
 }
 
 /*
@@ -1012,10 +1025,12 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm, const struct ipsec_sa *ss,
 	return 0;
 }
 
-static void
+static int
 fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 	const struct ipsec_sa *lsa)
 {
+	int32_t rc = 0;
+
 	ss->sa = sa;
 	ss->type = lsa->type;
 
@@ -1028,6 +1043,17 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 		ss->security.ctx = lsa->security_ctx;
 		ss->security.ol_flags = lsa->ol_flags;
 	}
+
+	if (ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+		ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
+		if (ss->security.ses != NULL) {
+			rc = rte_ipsec_session_prepare(ss);
+			if (rc != 0)
+				memset(ss, 0, sizeof(*ss));
+		}
+	}
+
+	return rc;
 }
 
 /*
@@ -1062,8 +1088,8 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	fill_ipsec_session(&lsa->ips, sa, lsa);
-	return 0;
+	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
+	return rc;
 }
 
 /*
@@ -1166,7 +1192,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in);
+		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx);
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_in, sa_in, nb_sa_in,
@@ -1186,7 +1212,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out);
+		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx);
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_out, sa_out, nb_sa_out,
-- 
2.7.4

[dpdk-dev] [PATCH v6 2/2] examples/ipsec-secgw/test: fix inline test scripts

[Bernard Iremonger]

Remove workaround in tun_aesgcm_defs.sh and trs_aesgcm_defs.sh
to get around the bug where the first inbound packet is dropped
for inline crypto.

Fixes: 929784452094 ("examples/ipsec-secgw: add scripts for functional test")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh | 10 ----------
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh | 10 ----------
 2 files changed, 20 deletions(-)

diff --git a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
index a4d902b..8382d3d 100755
--- a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
@@ -33,11 +33,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -68,9 +63,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV6}
 }
diff --git a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
index 1764ef6..8ae6532 100755
--- a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
@@ -35,11 +35,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -72,9 +67,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping6 -c 1 ${LOCAL_IPV6}
 }
-- 
2.7.4

Re: [dpdk-dev] [PATCH v6 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Ananyev, Konstantin]

> -----Original Message-----
> From: Iremonger, Bernard
> Sent: Wednesday, June 12, 2019 3:52 PM
> To: dev@dpdk.org; Ananyev, Konstantin <konstantin.ananyev@intel.com>; akhil.goyal@nxp.com
> Cc: Iremonger, Bernard <bernard.iremonger@intel.com>; stable@dpdk.org
> Subject: [PATCH v6 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
> 
> Inline crypto installs a flow rule in the NIC. This flow
> rule must be installed before the first inbound packet is
> received.
> 
> The create_session() function installs the flow rule,
> create_session() has been refactored into create_inline_session()
> and create_lookaside_session(). The create_inline_session() function
> uses the socket_ctx data and is now called at initialisation in
> sa_add_rules().
> 
> The max_session_size() function has been added to calculate memory
> requirements.
> 
> The cryprodev_init() function has been refactored to drop calls to
> rte_mempool_create() and to drop calculation of memory requirements.
> 
> The main() function has been refactored to call max_session_size() and
> to call session_pool_init() and session_priv_pool_init() earlier.
> The ports are started now before adding a flow rule in main().
> The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
> now called after the ports have been started.
> 
> The rte_ipsec_session_prepare() function is called in fill_ipsec_session()
> for inline which is called from the ipsec_sa_init() function.
> 
> Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
> Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
> Cc: stable@dpdk.org
> 
> Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> ---

Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
Tested-by: Konstantin Ananyev <konstantin.ananyev@intel.com>

> --
> 2.7.4

Re: [dpdk-dev] [PATCH v6 2/2] examples/ipsec-secgw/test: fix inline test scripts

[Ananyev, Konstantin]

> -----Original Message-----
> From: Iremonger, Bernard
> Sent: Wednesday, June 12, 2019 3:52 PM
> To: dev@dpdk.org; Ananyev, Konstantin <konstantin.ananyev@intel.com>; akhil.goyal@nxp.com
> Cc: Iremonger, Bernard <bernard.iremonger@intel.com>; stable@dpdk.org
> Subject: [PATCH v6 2/2] examples/ipsec-secgw/test: fix inline test scripts
> 
> Remove workaround in tun_aesgcm_defs.sh and trs_aesgcm_defs.sh
> to get around the bug where the first inbound packet is dropped
> for inline crypto.
> 
> Fixes: 929784452094 ("examples/ipsec-secgw: add scripts for functional test")
> Cc: stable@dpdk.org
> 
> Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> ---

Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
Tested-by: Konstantin Ananyev <konstantin.ananyev@intel.com>

> --
> 2.7.4

Re: [dpdk-dev] [PATCH v6 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Akhil Goyal]

Hi Bernard,

> >
> > Inline crypto installs a flow rule in the NIC. This flow
> > rule must be installed before the first inbound packet is
> > received.
> >
> > The create_session() function installs the flow rule,
> > create_session() has been refactored into create_inline_session()
> > and create_lookaside_session(). The create_inline_session() function
> > uses the socket_ctx data and is now called at initialisation in
> > sa_add_rules().
> >
> > The max_session_size() function has been added to calculate memory
> > requirements.
> >
> > The cryprodev_init() function has been refactored to drop calls to
> > rte_mempool_create() and to drop calculation of memory requirements.
> >
> > The main() function has been refactored to call max_session_size() and
> > to call session_pool_init() and session_priv_pool_init() earlier.
> > The ports are started now before adding a flow rule in main().
> > The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
> > now called after the ports have been started.
> >
> > The rte_ipsec_session_prepare() function is called in fill_ipsec_session()
> > for inline which is called from the ipsec_sa_init() function.
> >
> > Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
> > Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
> > Cc: stable@dpdk.org
> >
> > Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> > ---
> 
> Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
> Tested-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
> 

This patch need to be rebased.

However, I will try to work on your v5 in next few days if I get time. If I don't get time to fix that, will need you to send a rebased version of this patch.

Thanks,
Akhil

Re: [dpdk-dev] [PATCH v6 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Iremonger, Bernard]

Hi Akhil,

> -----Original Message-----
> From: Akhil Goyal [mailto:akhil.goyal@nxp.com]
> Sent: Wednesday, July 3, 2019 11:05 AM
> To: Ananyev, Konstantin <konstantin.ananyev@intel.com>; Iremonger,
> Bernard <bernard.iremonger@intel.com>; dev@dpdk.org
> Cc: stable@dpdk.org
> Subject: RE: [PATCH v6 1/2] examples/ipsec-secgw: fix 1st pkt dropped for
> inline crypto
> 
> Hi Bernard,
> 
> > >
> > > Inline crypto installs a flow rule in the NIC. This flow rule must
> > > be installed before the first inbound packet is received.
> > >
> > > The create_session() function installs the flow rule,
> > > create_session() has been refactored into create_inline_session()
> > > and create_lookaside_session(). The create_inline_session() function
> > > uses the socket_ctx data and is now called at initialisation in
> > > sa_add_rules().
> > >
> > > The max_session_size() function has been added to calculate memory
> > > requirements.
> > >
> > > The cryprodev_init() function has been refactored to drop calls to
> > > rte_mempool_create() and to drop calculation of memory requirements.
> > >
> > > The main() function has been refactored to call max_session_size()
> > > and to call session_pool_init() and session_priv_pool_init() earlier.
> > > The ports are started now before adding a flow rule in main().
> > > The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
> > > now called after the ports have been started.
> > >
> > > The rte_ipsec_session_prepare() function is called in
> > > fill_ipsec_session() for inline which is called from the ipsec_sa_init()
> function.
> > >
> > > Fixes: ec17993a145a ("examples/ipsec-secgw: support security
> > > offload")
> > > Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample
> > > application")
> > > Cc: stable@dpdk.org
> > >
> > > Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> > > ---
> >
> > Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
> > Tested-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
> >
> 
> This patch need to be rebased.
> 
> However, I will try to work on your v5 in next few days if I get time. If I don't
> get time to fix that, will need you to send a rebased version of this patch.
> 
> Thanks,
> Akhil
> 

v6 is the latest version
I will send a v7 rebased on 19.08.rc1

Regards,

Bernard

Re: [dpdk-dev] [PATCH v6 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Akhil Goyal]

> 
> Hi Akhil,
> 
> >
> > Hi Bernard,
> >
> > > >
> > > > Inline crypto installs a flow rule in the NIC. This flow rule must
> > > > be installed before the first inbound packet is received.
> > > >
> > > > The create_session() function installs the flow rule,
> > > > create_session() has been refactored into create_inline_session()
> > > > and create_lookaside_session(). The create_inline_session() function
> > > > uses the socket_ctx data and is now called at initialisation in
> > > > sa_add_rules().
> > > >
> > > > The max_session_size() function has been added to calculate memory
> > > > requirements.
> > > >
> > > > The cryprodev_init() function has been refactored to drop calls to
> > > > rte_mempool_create() and to drop calculation of memory requirements.
> > > >
> > > > The main() function has been refactored to call max_session_size()
> > > > and to call session_pool_init() and session_priv_pool_init() earlier.
> > > > The ports are started now before adding a flow rule in main().
> > > > The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
> > > > now called after the ports have been started.
> > > >
> > > > The rte_ipsec_session_prepare() function is called in
> > > > fill_ipsec_session() for inline which is called from the ipsec_sa_init()
> > function.
> > > >
> > > > Fixes: ec17993a145a ("examples/ipsec-secgw: support security
> > > > offload")
> > > > Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample
> > > > application")
> > > > Cc: stable@dpdk.org
> > > >
> > > > Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> > > > ---
> > >
> > > Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
> > > Tested-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
> > >
> >
> > This patch need to be rebased.
> >
> > However, I will try to work on your v5 in next few days if I get time. If I don't
> > get time to fix that, will need you to send a rebased version of this patch.
> >
> > Thanks,
> > Akhil
> >
> 
> v6 is the latest version
> I will send a v7 rebased on 19.08.rc1

I was talking about your previous which was for both lookaside and inline cases to create sessions at init time.
If I get time, I will fix that or will ask you to rebase this one.
> 
> Regards,
> 
> Bernard
> 

Re: [dpdk-dev] [PATCH v6 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Iremonger, Bernard]

Hi Akhil,

> -----Original Message-----
> From: Akhil Goyal [mailto:akhil.goyal@nxp.com]
> Sent: Wednesday, July 3, 2019 11:18 AM
> To: Iremonger, Bernard <bernard.iremonger@intel.com>; Ananyev,
> Konstantin <konstantin.ananyev@intel.com>; dev@dpdk.org
> Cc: stable@dpdk.org
> Subject: RE: [PATCH v6 1/2] examples/ipsec-secgw: fix 1st pkt dropped for
> inline crypto
> 
> 
> 
> >
> > Hi Akhil,
> >
> > >
> > > Hi Bernard,
> > >
> > > > >
> > > > > Inline crypto installs a flow rule in the NIC. This flow rule
> > > > > must be installed before the first inbound packet is received.
> > > > >
> > > > > The create_session() function installs the flow rule,
> > > > > create_session() has been refactored into
> > > > > create_inline_session() and create_lookaside_session(). The
> > > > > create_inline_session() function uses the socket_ctx data and is
> > > > > now called at initialisation in sa_add_rules().
> > > > >
> > > > > The max_session_size() function has been added to calculate
> > > > > memory requirements.
> > > > >
> > > > > The cryprodev_init() function has been refactored to drop calls
> > > > > to
> > > > > rte_mempool_create() and to drop calculation of memory
> requirements.
> > > > >
> > > > > The main() function has been refactored to call
> > > > > max_session_size() and to call session_pool_init() and
> session_priv_pool_init() earlier.
> > > > > The ports are started now before adding a flow rule in main().
> > > > > The sa_init(), sp4_init(), sp6_init() and rt_init() functions
> > > > > are now called after the ports have been started.
> > > > >
> > > > > The rte_ipsec_session_prepare() function is called in
> > > > > fill_ipsec_session() for inline which is called from the
> > > > > ipsec_sa_init()
> > > function.
> > > > >
> > > > > Fixes: ec17993a145a ("examples/ipsec-secgw: support security
> > > > > offload")
> > > > > Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample
> > > > > application")
> > > > > Cc: stable@dpdk.org
> > > > >
> > > > > Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> > > > > ---
> > > >
> > > > Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
> > > > Tested-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
> > > >
> > >
> > > This patch need to be rebased.
> > >
> > > However, I will try to work on your v5 in next few days if I get
> > > time. If I don't get time to fix that, will need you to send a rebased
> version of this patch.
> > >
> > > Thanks,
> > > Akhil
> > >
> >
> > v6 is the latest version
> > I will send a v7 rebased on 19.08.rc1
> 
> I was talking about your previous which was for both lookaside and inline
> cases to create sessions at init time.
> If I get time, I will fix that or will ask you to rebase this one.
> >
> > Regards,
> >
> > Bernard
> >

They previous one was v4, where we could not test the lookaside code.
Should I rebase the v4 and send as v7?

Regards,

Bernard.

Re: [dpdk-dev] [PATCH v6 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Akhil Goyal]

> 
> Hi Akhil,
> 
> >
> >
> >
> > >
> > > Hi Akhil,
> > >
> > > >
> > > > Hi Bernard,
> > > >
> > > > > >
> > > > > > Inline crypto installs a flow rule in the NIC. This flow rule
> > > > > > must be installed before the first inbound packet is received.
> > > > > >
> > > > > > The create_session() function installs the flow rule,
> > > > > > create_session() has been refactored into
> > > > > > create_inline_session() and create_lookaside_session(). The
> > > > > > create_inline_session() function uses the socket_ctx data and is
> > > > > > now called at initialisation in sa_add_rules().
> > > > > >
> > > > > > The max_session_size() function has been added to calculate
> > > > > > memory requirements.
> > > > > >
> > > > > > The cryprodev_init() function has been refactored to drop calls
> > > > > > to
> > > > > > rte_mempool_create() and to drop calculation of memory
> > requirements.
> > > > > >
> > > > > > The main() function has been refactored to call
> > > > > > max_session_size() and to call session_pool_init() and
> > session_priv_pool_init() earlier.
> > > > > > The ports are started now before adding a flow rule in main().
> > > > > > The sa_init(), sp4_init(), sp6_init() and rt_init() functions
> > > > > > are now called after the ports have been started.
> > > > > >
> > > > > > The rte_ipsec_session_prepare() function is called in
> > > > > > fill_ipsec_session() for inline which is called from the
> > > > > > ipsec_sa_init()
> > > > function.
> > > > > >
> > > > > > Fixes: ec17993a145a ("examples/ipsec-secgw: support security
> > > > > > offload")
> > > > > > Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample
> > > > > > application")
> > > > > > Cc: stable@dpdk.org
> > > > > >
> > > > > > Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> > > > > > ---
> > > > >
> > > > > Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
> > > > > Tested-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
> > > > >
> > > >
> > > > This patch need to be rebased.
> > > >
> > > > However, I will try to work on your v5 in next few days if I get
> > > > time. If I don't get time to fix that, will need you to send a rebased
> > version of this patch.
> > > >
> > > > Thanks,
> > > > Akhil
> > > >
> > >
> > > v6 is the latest version
> > > I will send a v7 rebased on 19.08.rc1
> >
> > I was talking about your previous which was for both lookaside and inline
> > cases to create sessions at init time.
> > If I get time, I will fix that or will ask you to rebase this one.
> > >
> > > Regards,
> > >
> > > Bernard
> > >
> 
> They previous one was v4, where we could not test the lookaside code.
> Should I rebase the v4 and send as v7?
> 
I will let you know, if your v4 can work well or your v6 will be rebased and applied.


> Regards,
> 
> Bernard.
> 

[dpdk-dev] [PATCH v7 0/2] examples/ipsec-secgw: fix 1st pkt dropped

[Bernard Iremonger]

This patchset fixes the issue of the first inbound packet
being dropped for inline crypto.

Changes in v7:
--------------
Rebased to DPDK 19.08-rc1

Changes in v6:
--------------
Rebased to latest master.
Minor changes to the following functions in ipsec.c:
create_lookaside_session()
create_inline_session()

Bernard Iremonger (2):
  examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
  examples/ipsec-secgw/test: fix inline test scripts

 examples/ipsec-secgw/ipsec-secgw.c           | 244 +++++++--------
 examples/ipsec-secgw/ipsec.c                 | 449 ++++++++++++++-------------
 examples/ipsec-secgw/ipsec.h                 |   5 +-
 examples/ipsec-secgw/ipsec_process.c         |   9 +-
 examples/ipsec-secgw/sa.c                    |  46 ++-
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 -
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 -
 7 files changed, 403 insertions(+), 370 deletions(-)

Changes in v5:
-------------
The v2 patchset has been rebased to the latest master.
The v4 patchset has been dropped as it caused issues with the lookaside code
which we are unable to test.

Bernard Iremonger (2):
  examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
  examples/ipsec-secgw/test: fix inline test scripts

 examples/ipsec-secgw/ipsec-secgw.c           | 244 +++++++++++++--------------
 examples/ipsec-secgw/ipsec.c                 | 122 +++++++++-----
 examples/ipsec-secgw/ipsec.h                 |   5 +-
 examples/ipsec-secgw/ipsec_process.c         |   9 +-
 examples/ipsec-secgw/sa.c                    |  46 +++--
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 --
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 --
 7 files changed, 245 insertions(+), 201 deletions(-)

Changes in v2:
-------------
The first three patches of the v1 have been squashed.
The commit message for the squashed patch has been updated.
Patches 4,5 and 6 of the v1 have been dropped from this patchset.
A patch to fix the test scripts has been added.

Bernard Iremonger (2):
  examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
  examples/ipsec-secgw/test: fix inline test scripts

 examples/ipsec-secgw/ipsec-secgw.c           | 244 +++++++-------
 examples/ipsec-secgw/ipsec.c                 | 456 ++++++++++++++-------------
 examples/ipsec-secgw/ipsec.h                 |   5 +-
 examples/ipsec-secgw/ipsec_process.c         |   9 +-
 examples/ipsec-secgw/sa.c                    |  46 ++-
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 -
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 -
 7 files changed, 405 insertions(+), 375 deletions(-)

-- 
2.7.4

[dpdk-dev] [PATCH v7 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Bernard Iremonger]

Inline crypto installs a flow rule in the NIC. This flow
rule must be installed before the first inbound packet is
received.

The create_session() function installs the flow rule,
create_session() has been refactored into create_inline_session()
and create_lookaside_session(). The create_inline_session() function
uses the socket_ctx data and is now called at initialisation in
sa_add_rules().

The max_session_size() function has been added to calculate memory
requirements.

The cryprodev_init() function has been refactored to drop calls to
rte_mempool_create() and to drop calculation of memory requirements.

The main() function has been refactored to call max_session_size() and
to call session_pool_init() and session_priv_pool_init() earlier.
The ports are started now before adding a flow rule in main().
The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
now called after the ports have been started.

The rte_ipsec_session_prepare() function is called in fill_ipsec_session()
for inline which is called from the ipsec_sa_init() function.

Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
---
 examples/ipsec-secgw/ipsec-secgw.c   | 244 +++++++++----------
 examples/ipsec-secgw/ipsec.c         | 456 ++++++++++++++++++-----------------
 examples/ipsec-secgw/ipsec.h         |   5 +-
 examples/ipsec-secgw/ipsec_process.c |   9 +-
 examples/ipsec-secgw/sa.c            |  46 +++-
 5 files changed, 405 insertions(+), 355 deletions(-)

diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index b1ecbb9..f4819ce 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -1802,7 +1802,7 @@ cryptodevs_init(void)
 	struct rte_cryptodev_config dev_conf;
 	struct rte_cryptodev_qp_conf qp_conf;
 	uint16_t idx, max_nb_qps, qp, i;
-	int16_t cdev_id, port_id;
+	int16_t cdev_id;
 	struct rte_hash_parameters params = { 0 };
 
 	const uint64_t mseg_flag = multi_seg_required() ?
@@ -1828,45 +1828,6 @@ cryptodevs_init(void)
 
 	printf("lcore/cryptodev/qp mappings:\n");
 
-	uint32_t max_sess_sz = 0, sess_sz;
-	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
-		void *sec_ctx;
-
-		/* Get crypto priv session size */
-		sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-
-		/*
-		 * If crypto device is security capable, need to check the
-		 * size of security session as well.
-		 */
-
-		/* Get security context of the crypto device */
-		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		/* Get size of security session */
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-	RTE_ETH_FOREACH_DEV(port_id) {
-		void *sec_ctx;
-
-		if ((enabled_port_mask & (1 << port_id)) == 0)
-			continue;
-
-		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-
 	idx = 0;
 	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
 		struct rte_cryptodev_info cdev_info;
@@ -1911,45 +1872,6 @@ cryptodevs_init(void)
 				"Device does not support at least %u "
 				"sessions", CDEV_MP_NB_OBJS);
 
-		if (!socket_ctx[dev_conf.socket_id].session_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_%u", dev_conf.socket_id);
-			sess_mp = rte_cryptodev_sym_session_pool_create(
-					mp_name, CDEV_MP_NB_OBJS,
-					0, CDEV_MP_CACHE_SZ, 0,
-					dev_conf.socket_id);
-			socket_ctx[dev_conf.socket_id].session_pool = sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_priv_%u", dev_conf.socket_id);
-			sess_mp = rte_mempool_create(mp_name,
-					CDEV_MP_NB_OBJS,
-					max_sess_sz,
-					CDEV_MP_CACHE_SZ,
-					0, NULL, NULL, NULL,
-					NULL, dev_conf.socket_id,
-					0);
-			socket_ctx[dev_conf.socket_id].session_priv_pool =
-					sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool ||
-				!socket_ctx[dev_conf.socket_id].session_pool)
-			rte_exit(EXIT_FAILURE,
-				"Cannot create session pool on socket %d\n",
-				dev_conf.socket_id);
-		else
-			printf("Allocated session pool on socket %d\n",
-					dev_conf.socket_id);
-
 		if (rte_cryptodev_configure(cdev_id, &dev_conf))
 			rte_panic("Failed to initialize cryptodev %u\n",
 					cdev_id);
@@ -1970,39 +1892,6 @@ cryptodevs_init(void)
 					cdev_id);
 	}
 
-	/* create session pools for eth devices that implement security */
-	RTE_ETH_FOREACH_DEV(port_id) {
-		if ((enabled_port_mask & (1 << port_id)) &&
-				rte_eth_dev_get_sec_ctx(port_id)) {
-			int socket_id = rte_eth_dev_socket_id(port_id);
-
-			if (!socket_ctx[socket_id].session_priv_pool) {
-				char mp_name[RTE_MEMPOOL_NAMESIZE];
-				struct rte_mempool *sess_mp;
-
-				snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-						"sess_mp_%u", socket_id);
-				sess_mp = rte_mempool_create(mp_name,
-						(CDEV_MP_NB_OBJS * 2),
-						max_sess_sz,
-						CDEV_MP_CACHE_SZ,
-						0, NULL, NULL, NULL,
-						NULL, socket_id,
-						0);
-				if (sess_mp == NULL)
-					rte_exit(EXIT_FAILURE,
-						"Cannot create session pool "
-						"on socket %d\n", socket_id);
-				else
-					printf("Allocated session pool "
-						"on socket %d\n", socket_id);
-				socket_ctx[socket_id].session_priv_pool =
-						sess_mp;
-			}
-		}
-	}
-
-
 	printf("\n");
 
 	return 0;
@@ -2174,6 +2063,99 @@ port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads)
 	printf("\n");
 }
 
+static size_t
+max_session_size(void)
+{
+	size_t max_sz, sz;
+	void *sec_ctx;
+	int16_t cdev_id, port_id, n;
+
+	max_sz = 0;
+	n =  rte_cryptodev_count();
+	for (cdev_id = 0; cdev_id != n; cdev_id++) {
+		sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
+		if (sz > max_sz)
+			max_sz = sz;
+		/*
+		 * If crypto device is security capable, need to check the
+		 * size of security session as well.
+		 */
+
+		/* Get security context of the crypto device */
+		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		/* Get size of security session */
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	RTE_ETH_FOREACH_DEV(port_id) {
+		if ((enabled_port_mask & (1 << port_id)) == 0)
+			continue;
+
+		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	return max_sz;
+}
+
+static void
+session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_%u", socket_id);
+	sess_mp = rte_cryptodev_sym_session_pool_create(
+			mp_name, CDEV_MP_NB_OBJS,
+			sess_sz, CDEV_MP_CACHE_SZ, 0,
+			socket_id);
+	ctx->session_pool = sess_mp;
+
+	if (ctx->session_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session pool on socket %d\n", socket_id);
+	else
+		printf("Allocated session pool on socket %d\n",	socket_id);
+}
+
+static void
+session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
+	size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_priv_%u", socket_id);
+	sess_mp = rte_mempool_create(mp_name,
+			CDEV_MP_NB_OBJS,
+			sess_sz,
+			CDEV_MP_CACHE_SZ,
+			0, NULL, NULL, NULL,
+			NULL, socket_id,
+			0);
+	ctx->session_priv_pool = sess_mp;
+
+	if (ctx->session_priv_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session priv pool on socket %d\n",
+			socket_id);
+	else
+		printf("Allocated session priv pool on socket %d\n",
+			socket_id);
+}
+
 static void
 pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf)
 {
@@ -2397,9 +2379,11 @@ main(int32_t argc, char **argv)
 {
 	int32_t ret;
 	uint32_t lcore_id;
+	uint32_t i;
 	uint8_t socket_id;
 	uint16_t portid;
 	uint64_t req_rx_offloads, req_tx_offloads;
+	size_t sess_sz;
 
 	/* init EAL */
 	ret = rte_eal_init(argc, argv);
@@ -2427,7 +2411,8 @@ main(int32_t argc, char **argv)
 
 	nb_lcores = rte_lcore_count();
 
-	/* Replicate each context per socket */
+	sess_sz = max_session_size();
+
 	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
 		if (rte_lcore_is_enabled(lcore_id) == 0)
 			continue;
@@ -2437,20 +2422,14 @@ main(int32_t argc, char **argv)
 		else
 			socket_id = 0;
 
+		/* mbuf_pool is initialised by the pool_init() function*/
 		if (socket_ctx[socket_id].mbuf_pool)
 			continue;
 
-		/* initilaze SPD */
-		sp4_init(&socket_ctx[socket_id], socket_id);
-
-		sp6_init(&socket_ctx[socket_id], socket_id);
-
-		/* initilaze SAD */
-		sa_init(&socket_ctx[socket_id], socket_id);
-
-		rt_init(&socket_ctx[socket_id], socket_id);
-
 		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
+		session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
+		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
+			sess_sz);
 	}
 
 	RTE_ETH_FOREACH_DEV(portid) {
@@ -2468,7 +2447,11 @@ main(int32_t argc, char **argv)
 		if ((enabled_port_mask & (1 << portid)) == 0)
 			continue;
 
-		/* Start device */
+		/*
+		 * Start device
+		 * note: device must be started before a flow rule
+		 * can be installed.
+		 */
 		ret = rte_eth_dev_start(portid);
 		if (ret < 0)
 			rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
@@ -2493,6 +2476,19 @@ main(int32_t argc, char **argv)
 			rte_exit(EXIT_FAILURE, "failed at reassemble init");
 	}
 
+	/* Replicate each context per socket */
+	for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
+		socket_id = rte_socket_id_by_idx(i);
+		if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
+			(socket_ctx[socket_id].sa_in == NULL) &&
+			(socket_ctx[socket_id].sa_out == NULL)) {
+			sa_init(&socket_ctx[socket_id], socket_id);
+			sp4_init(&socket_ctx[socket_id], socket_id);
+			sp6_init(&socket_ctx[socket_id], socket_id);
+			rt_init(&socket_ctx[socket_id], socket_id);
+		}
+	}
+
 	check_all_ports_link_status(enabled_port_mask);
 
 	/* launch per-lcore init on every lcore */
diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index d1cbdc3..0f23cb1 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -40,7 +40,7 @@ set_ipsec_conf(struct ipsec_sa *sa, struct rte_security_ipsec_xform *ipsec)
 }
 
 int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 {
 	struct rte_cryptodev_info cdev_info;
 	unsigned long cdev_id_qp = 0;
@@ -53,19 +53,17 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 	key.auth_algo = (uint8_t)sa->auth_algo;
 	key.aead_algo = (uint8_t)sa->aead_algo;
 
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
-				(void **)&cdev_id_qp);
-		if (ret < 0) {
-			RTE_LOG(ERR, IPSEC,
+	ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
+			(void **)&cdev_id_qp);
+	if (ret < 0) {
+		RTE_LOG(ERR, IPSEC,
 				"No cryptodev: core %u, cipher_algo %u, "
 				"auth_algo %u, aead_algo %u\n",
 				key.lcore_id,
 				key.cipher_algo,
 				key.auth_algo,
 				key.aead_algo);
-			return -1;
-		}
+		return -1;
 	}
 
 	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on cryptodev "
@@ -107,231 +105,268 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 				"SEC Session init failed: err: %d\n", ret);
 				return -1;
 			}
-		} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
-			struct rte_flow_error err;
-			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
-							rte_eth_dev_get_sec_ctx(
-							sa->portid);
-			const struct rte_security_capability *sec_cap;
-			int ret = 0;
-
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_priv_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"SEC Session init failed: err: %d\n", ret);
-				return -1;
-			}
-
-			sec_cap = rte_security_capabilities_get(ctx);
+		} else {
+			RTE_LOG(ERR, IPSEC, "Inline not supported\n");
+			return -1;
+		}
+	} else {
+		sa->crypto_session = rte_cryptodev_sym_session_create(
+				ipsec_ctx->session_pool);
+		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
+				sa->crypto_session, sa->xforms,
+				ipsec_ctx->session_priv_pool);
 
-			/* iterate until ESP tunnel*/
-			while (sec_cap->action !=
-					RTE_SECURITY_ACTION_TYPE_NONE) {
+		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
+				&cdev_info);
+	}
 
-				if (sec_cap->action == sa->type &&
-				    sec_cap->protocol ==
-					RTE_SECURITY_PROTOCOL_IPSEC &&
-				    sec_cap->ipsec.mode ==
-					sess_conf.ipsec.mode &&
-				    sec_cap->ipsec.direction == sa->direction)
-					break;
-				sec_cap++;
-			}
+	sa->cdev_id_qp = cdev_id_qp;
 
-			if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
-				RTE_LOG(ERR, IPSEC,
-				"No suitable security capability found\n");
-				return -1;
-			}
+	return 0;
+}
 
-			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
-			sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
-
-			if (IS_IP6(sa->flags)) {
-				sa->pattern[1].mask = &rte_flow_item_ipv6_mask;
-				sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV6;
-				sa->pattern[1].spec = &sa->ipv6_spec;
-
-				memcpy(sa->ipv6_spec.hdr.dst_addr,
-					sa->dst.ip.ip6.ip6_b, 16);
-				memcpy(sa->ipv6_spec.hdr.src_addr,
-				       sa->src.ip.ip6.ip6_b, 16);
-			} else if (IS_IP4(sa->flags)) {
-				sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
-				sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
-				sa->pattern[1].spec = &sa->ipv4_spec;
-
-				sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
-				sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
-			}
+int
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
+{
+	int32_t ret = 0;
+	struct rte_security_ctx *sec_ctx;
+	struct rte_security_session_conf sess_conf = {
+		.action_type = sa->type,
+		.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
+		{.ipsec = {
+			.spi = sa->spi,
+			.salt = sa->salt,
+			.options = { 0 },
+			.direction = sa->direction,
+			.proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
+			.mode = (sa->flags == IP4_TUNNEL ||
+					sa->flags == IP6_TUNNEL) ?
+					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL :
+					RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
+		} },
+		.crypto_xform = sa->xforms,
+		.userdata = NULL,
+	};
+
+	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on port %u\n",
+		sa->spi, sa->portid);
+
+	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		struct rte_flow_error err;
+		const struct rte_security_capability *sec_cap;
+		int ret = 0;
+
+		sec_ctx = (struct rte_security_ctx *)
+					rte_eth_dev_get_sec_ctx(
+					sa->portid);
+		if (sec_ctx == NULL) {
+			RTE_LOG(ERR, IPSEC,
+				" rte_eth_dev_get_sec_ctx failed\n");
+			return -1;
+		}
 
-			sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
-			sa->pattern[2].spec = &sa->esp_spec;
-			sa->pattern[2].mask = &rte_flow_item_esp_mask;
-			sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
+		sa->sec_session = rte_security_session_create(sec_ctx,
+				&sess_conf, skt_ctx->session_pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC,
+				"SEC Session init failed: err: %d\n", ret);
+			return -1;
+		}
 
-			sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
+		sec_cap = rte_security_capabilities_get(sec_ctx);
+
+		/* iterate until ESP tunnel*/
+		while (sec_cap->action != RTE_SECURITY_ACTION_TYPE_NONE) {
+			if (sec_cap->action == sa->type &&
+			    sec_cap->protocol ==
+				RTE_SECURITY_PROTOCOL_IPSEC &&
+			    sec_cap->ipsec.mode ==
+				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
+			    sec_cap->ipsec.direction == sa->direction)
+				break;
+			sec_cap++;
+		}
 
-			sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
-			sa->action[0].conf = sa->sec_session;
+		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
+			RTE_LOG(ERR, IPSEC,
+				"No suitable security capability found\n");
+			return -1;
+		}
 
-			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
+		sa->ol_flags = sec_cap->ol_flags;
+		sa->security_ctx = sec_ctx;
+		sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
+
+		if (IS_IP6(sa->flags)) {
+			sa->pattern[1].mask = &rte_flow_item_ipv6_mask;
+			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV6;
+			sa->pattern[1].spec = &sa->ipv6_spec;
+
+			memcpy(sa->ipv6_spec.hdr.dst_addr,
+				sa->dst.ip.ip6.ip6_b, 16);
+			memcpy(sa->ipv6_spec.hdr.src_addr,
+			       sa->src.ip.ip6.ip6_b, 16);
+		} else if (IS_IP4(sa->flags)) {
+			sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
+			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
+			sa->pattern[1].spec = &sa->ipv4_spec;
+
+			sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
+			sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
+		}
 
-			sa->attr.egress = (sa->direction ==
-					RTE_SECURITY_IPSEC_SA_DIR_EGRESS);
-			sa->attr.ingress = (sa->direction ==
-					RTE_SECURITY_IPSEC_SA_DIR_INGRESS);
-			if (sa->attr.ingress) {
-				uint8_t rss_key[40];
-				struct rte_eth_rss_conf rss_conf = {
-					.rss_key = rss_key,
-					.rss_key_len = 40,
-				};
-				struct rte_eth_dev *eth_dev;
-				uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
-				struct rte_flow_action_rss action_rss;
-				unsigned int i;
-				unsigned int j;
-
-				sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
-				/* Try RSS. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
-				sa->action[1].conf = &action_rss;
-				eth_dev = ctx->device;
-				rte_eth_dev_rss_hash_conf_get(sa->portid,
-							      &rss_conf);
-				for (i = 0, j = 0;
-				     i < eth_dev->data->nb_rx_queues; ++i)
-					if (eth_dev->data->rx_queues[i])
-						queue[j++] = i;
-				action_rss = (struct rte_flow_action_rss){
+		sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
+		sa->pattern[2].spec = &sa->esp_spec;
+		sa->pattern[2].mask = &rte_flow_item_esp_mask;
+		sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
+
+		sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
+
+		sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
+		sa->action[0].conf = sa->sec_session;
+
+		sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
+
+		sa->attr.egress = (sa->direction ==
+				RTE_SECURITY_IPSEC_SA_DIR_EGRESS);
+		sa->attr.ingress = (sa->direction ==
+				RTE_SECURITY_IPSEC_SA_DIR_INGRESS);
+		if (sa->attr.ingress) {
+			uint8_t rss_key[40];
+			struct rte_eth_rss_conf rss_conf = {
+				.rss_key = rss_key,
+				.rss_key_len = 40,
+			};
+			struct rte_eth_dev *eth_dev;
+			uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
+			struct rte_flow_action_rss action_rss;
+			unsigned int i;
+			unsigned int j;
+
+			sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
+			/* Try RSS. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
+			sa->action[1].conf = &action_rss;
+			eth_dev = sec_ctx->device;
+			rte_eth_dev_rss_hash_conf_get(sa->portid, &rss_conf);
+			for (i = 0, j = 0;
+			     i < eth_dev->data->nb_rx_queues; ++i)
+				if (eth_dev->data->rx_queues[i])
+					queue[j++] = i;
+
+			action_rss = (struct rte_flow_action_rss){
 					.types = rss_conf.rss_hf,
 					.key_len = rss_conf.rss_key_len,
 					.queue_num = j,
 					.key = rss_key,
 					.queue = queue,
-				};
-				ret = rte_flow_validate(sa->portid, &sa->attr,
-							sa->pattern, sa->action,
-							&err);
-				if (!ret)
-					goto flow_create;
-				/* Try Queue. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_QUEUE;
-				sa->action[1].conf =
-					&(struct rte_flow_action_queue){
-					.index = 0,
-				};
-				ret = rte_flow_validate(sa->portid, &sa->attr,
-							sa->pattern, sa->action,
-							&err);
-				/* Try End. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
-				sa->action[1].conf = NULL;
-				ret = rte_flow_validate(sa->portid, &sa->attr,
-							sa->pattern, sa->action,
-							&err);
-				if (ret)
-					goto flow_create_failure;
-			} else if (sa->attr.egress &&
-				   (sa->ol_flags &
+			};
+			ret = rte_flow_validate(sa->portid, &sa->attr,
+						sa->pattern, sa->action,
+						&err);
+			if (!ret)
+				goto flow_create;
+			/* Try Queue. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_QUEUE;
+			sa->action[1].conf =
+				&(struct rte_flow_action_queue){
+				.index = 0,
+			};
+			ret = rte_flow_validate(sa->portid, &sa->attr,
+						sa->pattern, sa->action,
+						&err);
+			/* Try End. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
+			sa->action[1].conf = NULL;
+			ret = rte_flow_validate(sa->portid, &sa->attr,
+						sa->pattern, sa->action,
+						&err);
+			if (ret)
+				goto flow_create_failure;
+		} else if (sa->attr.egress &&
+			   (sa->ol_flags &
 				    RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
-				sa->action[1].type =
+			sa->action[1].type =
 					RTE_FLOW_ACTION_TYPE_PASSTHRU;
-				sa->action[2].type =
+			sa->action[2].type =
 					RTE_FLOW_ACTION_TYPE_END;
-			}
+		}
 flow_create:
-			sa->flow = rte_flow_create(sa->portid,
+		sa->flow = rte_flow_create(sa->portid,
 				&sa->attr, sa->pattern, sa->action, &err);
-			if (sa->flow == NULL) {
+		if (sa->flow == NULL) {
 flow_create_failure:
-				RTE_LOG(ERR, IPSEC,
-					"Failed to create ipsec flow msg: %s\n",
-					err.message);
-				return -1;
-			}
-		} else if (sa->type ==
-				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
-			struct rte_security_ctx *ctx =
-					(struct rte_security_ctx *)
-					rte_eth_dev_get_sec_ctx(sa->portid);
-			const struct rte_security_capability *sec_cap;
-
-			if (ctx == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"Ethernet device doesn't have security features registered\n");
-				return -1;
-			}
-
-			/* Set IPsec parameters in conf */
-			set_ipsec_conf(sa, &(sess_conf.ipsec));
+			RTE_LOG(ERR, IPSEC,
+				"Failed to create ipsec flow msg: %s\n",
+				err.message);
+			return -1;
+		}
+	} else if (sa->type ==	RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
+		const struct rte_security_capability *sec_cap;
 
-			/* Save SA as userdata for the security session. When
-			 * the packet is received, this userdata will be
-			 * retrieved using the metadata from the packet.
-			 *
-			 * The PMD is expected to set similar metadata for other
-			 * operations, like rte_eth_event, which are tied to
-			 * security session. In such cases, the userdata could
-			 * be obtained to uniquely identify the security
-			 * parameters denoted.
-			 */
+		sec_ctx = (struct rte_security_ctx *)
+				rte_eth_dev_get_sec_ctx(sa->portid);
 
-			sess_conf.userdata = (void *) sa;
+		if (sec_ctx == NULL) {
+			RTE_LOG(ERR, IPSEC,
+				"Ethernet device doesn't have security features registered\n");
+			return -1;
+		}
 
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
+		/* Set IPsec parameters in conf */
+		set_ipsec_conf(sa, &(sess_conf.ipsec));
+
+		/* Save SA as userdata for the security session. When
+		 * the packet is received, this userdata will be
+		 * retrieved using the metadata from the packet.
+		 *
+		 * The PMD is expected to set similar metadata for other
+		 * operations, like rte_eth_event, which are tied to
+		 * security session. In such cases, the userdata could
+		 * be obtained to uniquely identify the security
+		 * parameters denoted.
+		 */
+
+		sess_conf.userdata = (void *) sa;
+
+		sa->sec_session = rte_security_session_create(sec_ctx,
+					&sess_conf, skt_ctx->session_pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
-				return -1;
-			}
-
-			sec_cap = rte_security_capabilities_get(ctx);
+			return -1;
+		}
 
-			if (sec_cap == NULL) {
-				RTE_LOG(ERR, IPSEC,
+		sec_cap = rte_security_capabilities_get(sec_ctx);
+		if (sec_cap == NULL) {
+			RTE_LOG(ERR, IPSEC,
 				"No capabilities registered\n");
-				return -1;
-			}
+			return -1;
+		}
 
-			/* iterate until ESP tunnel*/
-			while (sec_cap->action !=
-					RTE_SECURITY_ACTION_TYPE_NONE) {
-
-				if (sec_cap->action == sa->type &&
-				    sec_cap->protocol ==
-					RTE_SECURITY_PROTOCOL_IPSEC &&
-				    sec_cap->ipsec.mode ==
-					sess_conf.ipsec.mode &&
-				    sec_cap->ipsec.direction == sa->direction)
-					break;
-				sec_cap++;
-			}
+		/* iterate until ESP tunnel*/
+		while (sec_cap->action !=
+				RTE_SECURITY_ACTION_TYPE_NONE) {
+			if (sec_cap->action == sa->type &&
+			    sec_cap->protocol ==
+				RTE_SECURITY_PROTOCOL_IPSEC &&
+			    sec_cap->ipsec.mode ==
+				sess_conf.ipsec.mode &&
+			    sec_cap->ipsec.direction == sa->direction)
+				break;
+			sec_cap++;
+		}
 
-			if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
-				RTE_LOG(ERR, IPSEC,
+		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
+			RTE_LOG(ERR, IPSEC,
 				"No suitable security capability found\n");
-				return -1;
-			}
-
-			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
+			return -1;
 		}
-	} else {
-		sa->crypto_session = rte_cryptodev_sym_session_create(
-				ipsec_ctx->session_pool);
-		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
-				sa->crypto_session, sa->xforms,
-				ipsec_ctx->session_priv_pool);
 
-		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
-				&cdev_info);
+		sa->ol_flags = sec_cap->ol_flags;
+		sa->security_ctx = sec_ctx;
 	}
-	sa->cdev_id_qp = cdev_id_qp;
+	sa->cdev_id_qp = 0;
 
 	return 0;
 }
@@ -398,7 +433,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			rte_prefetch0(&priv->sym_cop);
 
 			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
+				create_lookaside_session(ipsec_ctx, sa)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -417,7 +452,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			rte_prefetch0(&priv->sym_cop);
 
 			if ((unlikely(sa->crypto_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
+				create_lookaside_session(ipsec_ctx, sa)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -432,12 +467,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 			break;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
+			RTE_ASSERT(sa->sec_session != NULL);
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
 			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
@@ -445,17 +475,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 						sa->sec_session, pkts[i], NULL);
 			continue;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
+			RTE_ASSERT(sa->sec_session != NULL);
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			rte_security_attach_session(&priv->cop,
 					sa->sec_session);
 
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index 6e48466..1efa6e4 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -338,6 +338,9 @@ void
 enqueue_cop_burst(struct cdev_qp *cqp);
 
 int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+
+int
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa);
 
 #endif /* __IPSEC_H__ */
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index 3f9cacb..868f1a2 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -95,22 +95,23 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
 	/* setup crypto section */
 	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
 		if (sa->crypto_session == NULL) {
-			rc = create_session(ctx, sa);
+			rc = create_lookaside_session(ctx, sa);
 			if (rc != 0)
 				return rc;
 		}
 		ss->crypto.ses = sa->crypto_session;
 	/* setup session action type */
-	} else {
+	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
 		if (sa->sec_session == NULL) {
-			rc = create_session(ctx, sa);
+			rc = create_lookaside_session(ctx, sa);
 			if (rc != 0)
 				return rc;
 		}
 		ss->security.ses = sa->sec_session;
 		ss->security.ctx = sa->security_ctx;
 		ss->security.ol_flags = sa->ol_flags;
-	}
+	} else
+		RTE_ASSERT(0);
 
 	rc = rte_ipsec_session_prepare(ss);
 	if (rc != 0)
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index d700c8e..c3cf3bd 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -868,12 +868,14 @@ sa_add_address_inline_crypto(struct ipsec_sa *sa)
 
 static int
 sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries, uint32_t inbound)
+		uint32_t nb_entries, uint32_t inbound,
+		struct socket_ctx *skt_ctx)
 {
 	struct ipsec_sa *sa;
 	uint32_t i, idx;
 	uint16_t iv_length, aad_length;
 	int inline_status;
+	int32_t rc;
 
 	/* for ESN upper 32 bits of SQN also need to be part of AAD */
 	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
@@ -936,6 +938,17 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			sa->xforms = &sa_ctx->xf[idx].a;
 
+			if (sa->type ==
+				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
+				sa->type ==
+				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+				rc = create_inline_session(skt_ctx, sa);
+				if (rc != 0) {
+					RTE_LOG(ERR, IPSEC_ESP,
+						"create_inline_session() failed\n");
+					return -EINVAL;
+				}
+			}
 			print_one_sa_rule(sa, inbound);
 		} else {
 			switch (sa->cipher_algo) {
@@ -1011,16 +1024,16 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 static inline int
 sa_out_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 0);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx);
 }
 
 static inline int
 sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 1);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx);
 }
 
 /*
@@ -1086,10 +1099,12 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm, const struct ipsec_sa *ss,
 	return 0;
 }
 
-static void
+static int
 fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 	const struct ipsec_sa *lsa)
 {
+	int32_t rc = 0;
+
 	ss->sa = sa;
 	ss->type = lsa->type;
 
@@ -1102,6 +1117,17 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 		ss->security.ctx = lsa->security_ctx;
 		ss->security.ol_flags = lsa->ol_flags;
 	}
+
+	if (ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+		ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
+		if (ss->security.ses != NULL) {
+			rc = rte_ipsec_session_prepare(ss);
+			if (rc != 0)
+				memset(ss, 0, sizeof(*ss));
+		}
+	}
+
+	return rc;
 }
 
 /*
@@ -1136,8 +1162,8 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	fill_ipsec_session(&lsa->ips, sa, lsa);
-	return 0;
+	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
+	return rc;
 }
 
 /*
@@ -1240,7 +1266,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in);
+		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx);
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_in, sa_in, nb_sa_in,
@@ -1260,7 +1286,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out);
+		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx);
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_out, sa_out, nb_sa_out,
-- 
2.7.4

[dpdk-dev] [PATCH v7 2/2] examples/ipsec-secgw/test: fix inline test scripts

[Bernard Iremonger]

Remove workaround in tun_aesgcm_defs.sh and trs_aesgcm_defs.sh
to get around the bug where the first inbound packet is dropped
for inline crypto.

Fixes: 929784452094 ("examples/ipsec-secgw: add scripts for functional test")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
---
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh | 10 ----------
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh | 10 ----------
 2 files changed, 20 deletions(-)

diff --git a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
index a4d902b..8382d3d 100755
--- a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
@@ -33,11 +33,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -68,9 +63,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV6}
 }
diff --git a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
index 1764ef6..8ae6532 100755
--- a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
@@ -35,11 +35,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -72,9 +67,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping6 -c 1 ${LOCAL_IPV6}
 }
-- 
2.7.4

Re: [dpdk-dev] [PATCH v7 0/2] examples/ipsec-secgw: fix 1st pkt dropped

[Akhil Goyal]

> -----Original Message-----
> From: Bernard Iremonger <bernard.iremonger@intel.com>
> Sent: Wednesday, July 10, 2019 4:53 PM
> To: dev@dpdk.org; konstantin.ananyev@intel.com; Akhil Goyal
> <akhil.goyal@nxp.com>
> Cc: Bernard Iremonger <bernard.iremonger@intel.com>
> Subject: [PATCH v7 0/2] examples/ipsec-secgw: fix 1st pkt dropped
> 
> This patchset fixes the issue of the first inbound packet
> being dropped for inline crypto.
> 
> Changes in v7:
> --------------
> Rebased to DPDK 19.08-rc1
> 
> Changes in v6:
> --------------
> Rebased to latest master.
> Minor changes to the following functions in ipsec.c:
> create_lookaside_session()
> create_inline_session()
> 
> Bernard Iremonger (2):
>   examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
>   examples/ipsec-secgw/test: fix inline test scripts
> 
>  examples/ipsec-secgw/ipsec-secgw.c           | 244 +++++++--------
>  examples/ipsec-secgw/ipsec.c                 | 449 ++++++++++++++-------------
>  examples/ipsec-secgw/ipsec.h                 |   5 +-
>  examples/ipsec-secgw/ipsec_process.c         |   9 +-
>  examples/ipsec-secgw/sa.c                    |  46 ++-
>  examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 -
>  examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 -
>  7 files changed, 403 insertions(+), 370 deletions(-)
> 
> Changes in v5:
> -------------
> The v2 patchset has been rebased to the latest master.
> The v4 patchset has been dropped as it caused issues with the lookaside code
> which we are unable to test.
> 
> Bernard Iremonger (2):
>   examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
>   examples/ipsec-secgw/test: fix inline test scripts
> 
>  examples/ipsec-secgw/ipsec-secgw.c           | 244 +++++++++++++--------------
>  examples/ipsec-secgw/ipsec.c                 | 122 +++++++++-----
>  examples/ipsec-secgw/ipsec.h                 |   5 +-
>  examples/ipsec-secgw/ipsec_process.c         |   9 +-
>  examples/ipsec-secgw/sa.c                    |  46 +++--
>  examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 --
>  examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 --
>  7 files changed, 245 insertions(+), 201 deletions(-)
> 
> Changes in v2:
> -------------
> The first three patches of the v1 have been squashed.
> The commit message for the squashed patch has been updated.
> Patches 4,5 and 6 of the v1 have been dropped from this patchset.
> A patch to fix the test scripts has been added.
> 
> Bernard Iremonger (2):
>   examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
>   examples/ipsec-secgw/test: fix inline test scripts
> 
>  examples/ipsec-secgw/ipsec-secgw.c           | 244 +++++++-------
>  examples/ipsec-secgw/ipsec.c                 | 456 ++++++++++++++-------------
>  examples/ipsec-secgw/ipsec.h                 |   5 +-
>  examples/ipsec-secgw/ipsec_process.c         |   9 +-
>  examples/ipsec-secgw/sa.c                    |  46 ++-
>  examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 -
>  examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 -
>  7 files changed, 405 insertions(+), 375 deletions(-)
> 
> --
> 2.7.4

It was rebased manually, so please verify.
Applied to dpdk-next-crypto

Thanks.

Re: [dpdk-dev] [PATCH v7 0/2] examples/ipsec-secgw: fix 1st pkt dropped

[Iremonger, Bernard]

Hi Akhil,

<snip>

> > Subject: [PATCH v7 0/2] examples/ipsec-secgw: fix 1st pkt dropped
> >
> > This patchset fixes the issue of the first inbound packet being
> > dropped for inline crypto.
> >
> > Changes in v7:
> > --------------
> > Rebased to DPDK 19.08-rc1
> >
> > Changes in v6:
> > --------------
> > Rebased to latest master.
> > Minor changes to the following functions in ipsec.c:
> > create_lookaside_session()
> > create_inline_session()
> >
> > Bernard Iremonger (2):
> >   examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
> >   examples/ipsec-secgw/test: fix inline test scripts
> >
> >  examples/ipsec-secgw/ipsec-secgw.c           | 244 +++++++--------
> >  examples/ipsec-secgw/ipsec.c                 | 449 ++++++++++++++-------------
> >  examples/ipsec-secgw/ipsec.h                 |   5 +-
> >  examples/ipsec-secgw/ipsec_process.c         |   9 +-
> >  examples/ipsec-secgw/sa.c                    |  46 ++-
> >  examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 -
> > examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 -
> >  7 files changed, 403 insertions(+), 370 deletions(-)
> >
> > Changes in v5:
> > -------------
> > The v2 patchset has been rebased to the latest master.
> > The v4 patchset has been dropped as it caused issues with the
> > lookaside code which we are unable to test.
> >
> > Bernard Iremonger (2):
> >   examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
> >   examples/ipsec-secgw/test: fix inline test scripts
> >
> >  examples/ipsec-secgw/ipsec-secgw.c           | 244 +++++++++++++-----------
> ---
> >  examples/ipsec-secgw/ipsec.c                 | 122 +++++++++-----
> >  examples/ipsec-secgw/ipsec.h                 |   5 +-
> >  examples/ipsec-secgw/ipsec_process.c         |   9 +-
> >  examples/ipsec-secgw/sa.c                    |  46 +++--
> >  examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 --
> > examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 --
> >  7 files changed, 245 insertions(+), 201 deletions(-)
> >
> > Changes in v2:
> > -------------
> > The first three patches of the v1 have been squashed.
> > The commit message for the squashed patch has been updated.
> > Patches 4,5 and 6 of the v1 have been dropped from this patchset.
> > A patch to fix the test scripts has been added.
> >
> > Bernard Iremonger (2):
> >   examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
> >   examples/ipsec-secgw/test: fix inline test scripts
> >
> >  examples/ipsec-secgw/ipsec-secgw.c           | 244 +++++++-------
> >  examples/ipsec-secgw/ipsec.c                 | 456 ++++++++++++++-------------
> >  examples/ipsec-secgw/ipsec.h                 |   5 +-
> >  examples/ipsec-secgw/ipsec_process.c         |   9 +-
> >  examples/ipsec-secgw/sa.c                    |  46 ++-
> >  examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 -
> > examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 -
> >  7 files changed, 405 insertions(+), 375 deletions(-)
> >
> > --
> > 2.7.4
> 
> It was rebased manually, so please verify.
> Applied to dpdk-next-crypto
> 
> Thanks.

I will verify.

Regards,

Bernard.

Re: [dpdk-dev] [PATCH v7 0/2] examples/ipsec-secgw: fix 1st pkt dropped

[Iremonger, Bernard]

Hi Akhil,

<snip>

> > > Subject: [PATCH v7 0/2] examples/ipsec-secgw: fix 1st pkt dropped
> > >
> > > This patchset fixes the issue of the first inbound packet being
> > > dropped for inline crypto.
> > >
> > > Changes in v7:
> > > --------------
> > > Rebased to DPDK 19.08-rc1
> > >
> > > Changes in v6:
> > > --------------
> > > Rebased to latest master.
> > > Minor changes to the following functions in ipsec.c:
> > > create_lookaside_session()
> > > create_inline_session()
> > >
> > > Bernard Iremonger (2):
> > >   examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
> > >   examples/ipsec-secgw/test: fix inline test scripts
> > >
> > >  examples/ipsec-secgw/ipsec-secgw.c           | 244 +++++++--------
> > >  examples/ipsec-secgw/ipsec.c                 | 449 ++++++++++++++------------
> -
> > >  examples/ipsec-secgw/ipsec.h                 |   5 +-
> > >  examples/ipsec-secgw/ipsec_process.c         |   9 +-
> > >  examples/ipsec-secgw/sa.c                    |  46 ++-
> > >  examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 -
> > > examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 -
> > >  7 files changed, 403 insertions(+), 370 deletions(-)
> > >
> > > Changes in v5:
> > > -------------
> > > The v2 patchset has been rebased to the latest master.
> > > The v4 patchset has been dropped as it caused issues with the
> > > lookaside code which we are unable to test.
> > >
> > > Bernard Iremonger (2):
> > >   examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
> > >   examples/ipsec-secgw/test: fix inline test scripts
> > >
> > >  examples/ipsec-secgw/ipsec-secgw.c           | 244 +++++++++++++---------
> --
> > ---
> > >  examples/ipsec-secgw/ipsec.c                 | 122 +++++++++-----
> > >  examples/ipsec-secgw/ipsec.h                 |   5 +-
> > >  examples/ipsec-secgw/ipsec_process.c         |   9 +-
> > >  examples/ipsec-secgw/sa.c                    |  46 +++--
> > >  examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 --
> > > examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 --
> > >  7 files changed, 245 insertions(+), 201 deletions(-)
> > >
> > > Changes in v2:
> > > -------------
> > > The first three patches of the v1 have been squashed.
> > > The commit message for the squashed patch has been updated.
> > > Patches 4,5 and 6 of the v1 have been dropped from this patchset.
> > > A patch to fix the test scripts has been added.
> > >
> > > Bernard Iremonger (2):
> > >   examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
> > >   examples/ipsec-secgw/test: fix inline test scripts
> > >
> > >  examples/ipsec-secgw/ipsec-secgw.c           | 244 +++++++-------
> > >  examples/ipsec-secgw/ipsec.c                 | 456 ++++++++++++++------------
> -
> > >  examples/ipsec-secgw/ipsec.h                 |   5 +-
> > >  examples/ipsec-secgw/ipsec_process.c         |   9 +-
> > >  examples/ipsec-secgw/sa.c                    |  46 ++-
> > >  examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 -
> > > examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 -
> > >  7 files changed, 405 insertions(+), 375 deletions(-)
> > >
> > > --
> > > 2.7.4
> >
> > It was rebased manually, so please verify.
> > Applied to dpdk-next-crypto
> >
> > Thanks.
> 
> I will verify.
> 
> Regards,
> 
> Bernard.
> 
> 
Our tests are passing fine with the rebased patches.

Regards,

Bernard.

Re: [dpdk-dev] [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Iremonger, Bernard]

Hi Akhil,

<snip>

> Subject: RE: [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for
> inline crypto
> 
> 
> 
> > > > > > > > Subject: RE: [PATCH v4 1/2] examples/ipsec-secgw: fix 1st
> > > > > > > > packet
> > > > dropped
> > > > > > for
> > > > > > > > inline crypto
> > > > > > > >
> > > > > > > > Hi Bernard,
> > > > > > > >
> > > > > > > > > -       RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on
> > > > > > cryptodev "
> > > > > > > > > -                       "%u qp %u\n", sa->spi,
> > > > > > > > > -                       ipsec_ctx->tbl[cdev_id_qp].id,
> > > > > > > > > -                       ipsec_ctx->tbl[cdev_id_qp].qp);
> > > > > > > > > +       if ((sa == NULL) || (pool == NULL))
> > > > > > > > > +               return -EINVAL;
> > > > > > > > >
> > > > > > > > > -       if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
> > > > > > > > > -               struct rte_security_session_conf sess_conf = {
> > > > > > > > > +       struct rte_security_session_conf sess_conf = {
> > > > > > > > >                         .action_type = sa->type,
> > > > > > > > >                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
> > > > > > > > >                         {.ipsec = { @@ -90,247 +65,340
> > > > > > > > > @@ create_session(struct ipsec_ctx *ipsec_ctx,
> > > > > > struct
> > > > > > > > > ipsec_sa *sa)
> > > > > > > > >                         } },
> > > > > > > > >                         .crypto_xform = sa->xforms,
> > > > > > > > >                         .userdata = NULL,
> > > > > > > > > -
> > > > > > > > >                 };
> > > > > > > > >
> > > > > > > > > -               if (sa->type ==
> > > > > > > > RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL)
> > > > > > > > > {
> > > > > > > > > -                       struct rte_security_ctx *ctx = (struct rte_security_ctx
> *)
> > > > > > > > > -                                                       rte_cryptodev_get_sec_ctx(
> > > > > > > > > -                                                       ipsec_ctx->tbl[cdev_id_qp].id);
> > > > > > > > > -
> > > > > > > > > -                       /* Set IPsec parameters in conf */
> > > > > > > > > -                       set_ipsec_conf(sa, &(sess_conf.ipsec));
> > > > > > > > > -
> > > > > > > > > -                       sa->sec_session = rte_security_session_create(ctx,
> > > > > > > > > -                                       &sess_conf, ipsec_ctx->session_pool);
> > > > > > > > > -                       if (sa->sec_session == NULL) {
> > > > > > > > > -                               RTE_LOG(ERR, IPSEC,
> > > > > > > > > -                               "SEC Session init failed: err: %d\n", ret);
> > > > > > > > > -                               return -1;
> > > > > > > > > -                       }
> > > > > > > > > -               } else if (sa->type ==
> > > > > > RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
> > > > > > > > {
> > > > > > > > > -                       struct rte_flow_error err;
> > > > > > > > > -                       struct rte_security_ctx *ctx = (struct rte_security_ctx
> *)
> > > > > > > > > -                                                       rte_eth_dev_get_sec_ctx(
> > > > > > > > > -                                                       sa->portid);
> > > > > > > > > -                       const struct rte_security_capability *sec_cap;
> > > > > > > > > -                       int ret = 0;
> > > > > > > > > -
> > > > > > > > > -                       sa->sec_session = rte_security_session_create(ctx,
> > > > > > > > > -                                       &sess_conf, ipsec_ctx->session_pool);
> > > > > > > > > -                       if (sa->sec_session == NULL) {
> > > > > > > > > -                               RTE_LOG(ERR, IPSEC,
> > > > > > > > > -                               "SEC Session init failed: err: %d\n", ret);
> > > > > > > > > -                               return -1;
> > > > > > > > > -                       }
> > > > > > > > > +       if (sa->type ==
> > > > > > RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > > > > > > > > +               ctx = (struct rte_security_ctx *)
> > > > > > > > > +
> > > > > > > > > + rte_eth_dev_get_sec_ctx(sa->portid);
> > > > > > > >
> > > > > > > > This is breaking the lookaside mode. Ctx was retrieved
> > > > > > > > using the
> > > > ipsec_ctx-
> > > > > > >tbl
> > > > > > > > struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > > > > > > 				rte_cryptodev_get_sec_ctx(
> > > > > > > > 				ipsec_ctx->tbl[cdev_id_qp].id);
> > > > > > > >
> > > > > > > > I am looking into it, but I don't have time left to get it integrated in
> RC2.
> > > > So
> > > > > > this
> > > > > > > > has to be pushed to RC3
> > > > > > >
> > > > > > > It looks like there are multiple issues in this patch wrt
> > > > > > > lookaside and none
> > > > cases.
> > > > > > Only the inline cases seem to be working.
> > > > > > >
> > > > > > > 1. the patch removes the cdev_mapping concept completely.
> > > > > > > Cdev_id_qp is
> > > > > > not getting used.
> > > > > >
> > > > > > Not exactly.
> > > > > > cdev_id_qp is still setup, and is still used to decide to
> > > > > > which crypto-dev to enqueuer the crypto-op:
> > > > > > ipsec_enqueue(...)
> > > > > > {
> > > > > >    ...
> > > > > >    enqueue_cop(&ipsec_ctx->tbl[sa->cdev_id_qp], &priv->cop);
> > > > >
> > > > > I don't see anybody filling "sa->cdev_id_qp". Please let me know
> > > > > if I have
> > > > missed it somewhere.
> > > > > It is memset to 0 I guess.
> > > >
> > > >
> > > > Yep, true we lost it somewhere during the rework.
> > > >
> > > > >
> > > > > >
> > > > > >
> > > > > > Same in ipsec_process().
> > > > > >
> > > > > > For initialization, yes cdev_id_qp is not used anymore.
> > > > > > As discussed here:
> > > > > >
> > > > https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2F
> > > > mail
> > > > s.dp
> > > > > > dk.org%2Farchives%2Fdev%2F2019-
> > > > > >
> > > >
> > >
> March%2F127725.html&amp;data=02%7C01%7Cakhil.goyal%40nxp.com%7C04
> > > > > >
> > > >
> > >
> 194193cfc04c0b629008d6c7eea247%7C686ea1d3bc2b4c6fa92cd99c5c301635%
> > > > > >
> > > >
> > >
> 7C0%7C0%7C636916225072561313&amp;sdata=ga9IiqhYRWOz9QkRDIXNiigInk
> > > > > > soVGgu1E5EetqvE%2FA%3D&amp;reserved=0
> > > > > >
> > > > > > I think the problem you are hitting with lookaside-proto is
> > > > > > that for it we use 2 different values here:
> > > > > > a) In create_sec_session we use portid (it also should be
> > > > > > rte_cryptodev_get_sec_ctx() here)
> > > > > >     if (sa->type ==
> RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > > > > >                 ctx = (struct rte_security_ctx *)
> > > > > >
> > > > > > rte_eth_dev_get_sec_ctx(sa->portid);
> > > > > It should be rte_cryptodev_get_sec_ctx in the first place. And
> > > > > it needs a
> > > > cdev_id as input based on the cdev mapping done while initializing
> > > > > the cryptodev and neither the portid and nor cdev_id_qp.
> > > > > > b) in enqueue() we use cdev_id_qp
> > > > > >
> > > > > > Right now these values could be different.
> > > > > > As I understand we need to make sure that fro lookaside-proto
> > > > > > cdev_id_qp
> > > > ==
> > > > > > portid provided by user, correct?
> > > > > No it is not the case. Right now for lookaside there is no use
> > > > > of portid in case
> > > > of lookaside case.
> > > > > As the cdev/qp/core mappings are managed internally and the user
> > > > > cannot
> > > > tweak it from cfg file.
> > > > >
> > > >
> > > >
> > > > Hmm, then at least that line  is wrong here:
> > > >
> https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdoc.
> > > > dpdk
> > > >
> > >
> .org%2Fguides%2Fsample_app_ug%2Fipsec_secgw.html&amp;data=02%7C01%
> > > >
> > >
> 7Cakhil.goyal%40nxp.com%7Cb1e931a9967943887a0c08d6c7f49d28%7C686ea
> > > >
> > >
> 1d3bc2b4c6fa92cd99c5c301635%7C0%7C0%7C636916250754452306&amp;sda
> > > >
> > >
> ta=tNjHCO0O2rfh8shhQXu93qrM0Hr0OZVXUVhMcsg53dw%3D&amp;reserved=
> > > > 0
> > > >
> > > > sa out 5 cipher_algo aes-128-cbc cipher_key
> > > > 0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0 \ auth_algo sha1-hmac auth_key
> > > > 0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0 \ mode ipv4-tunnel src
> > > > 172.16.1.5 dst 172.16.2.5 \ type lookaside-protocol-offload
> > > > port_id 4
> > > >
> > > > And probably:
> > > > "Port/device ID of the ethernet/crypto accelerator for which the
> > > > SA is configured."
> > > > Need to be rephrased to remove crypto accelerator notice.
> > > Yes.
> > >
> > > >
> > > > Another question - why you guys don't consider using portid for
> > > > lookaside-
> > > proto?
> > > > As I can see add_mapping(function that  fills cdev_id_qp) doesn't
> > > > bother to check which rte_security protocols are supported (only
> > > > crypto capabilities are checked).
> > > > So not sure does current code will work ok with a mix of
> > > > lookaside- none/lookaside-proto devices.
> > > > Forcing user to specify crypto-dev id for lookaside-proto (via
> > > > portid or so) will simplify things significantly.
> > > I will think about it. Actually initially when portid was
> > > introduced, the intent was same but it did not work well.
> > > Because there may be a case of a cryptodev with multiple queues, so
> > > there will be a mapping done internally in the application and
> > > matching it with the user provided portid will be difficult.
> > >
> > > I believe that this could be done but would need some rework.
> > >
> > > >
> > > > Konstantin
> >
> > Could we consider going back to the V2 version of this patch set which
> > fixed the issue with the inline code and left the lookaside code unchanged?
> 
> Actually I have the same thoughts here:
> considering the problems with lookaside-proto, it seems more pragmatic to fix it
> as was suggested in V2:
> split create_session() into 2 functions, and invoke create_session_inline() at
> startup, while keeping create_session_lookaside() invocation at runtime.
> At least it would fix the issue in a clean way.
> If later ipsec-secgw will be reworked to allow lookaside session creation at init
> time, we can merge them back.
> Konstantin
> 
> >
> > We are not in a position test any changes to the lookaside code.
> >
> > Regards,
> >
> > Bernard.
> >

Both Konstantin and I have proposed using the V2 patch set  as a starting point to fix this issue instead of the V4 patch set which has caused issues with the lookaside code.
Have you an opinion on this proposal?

Regards,

Bernard.

Re: [dpdk-dev] [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Ananyev, Konstantin]

> > > > > > > Subject: RE: [PATCH v4 1/2] examples/ipsec-secgw: fix 1st
> > > > > > > packet
> > > dropped
> > > > > for
> > > > > > > inline crypto
> > > > > > >
> > > > > > > Hi Bernard,
> > > > > > >
> > > > > > > > -       RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on
> > > > > cryptodev "
> > > > > > > > -                       "%u qp %u\n", sa->spi,
> > > > > > > > -                       ipsec_ctx->tbl[cdev_id_qp].id,
> > > > > > > > -                       ipsec_ctx->tbl[cdev_id_qp].qp);
> > > > > > > > +       if ((sa == NULL) || (pool == NULL))
> > > > > > > > +               return -EINVAL;
> > > > > > > >
> > > > > > > > -       if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
> > > > > > > > -               struct rte_security_session_conf sess_conf = {
> > > > > > > > +       struct rte_security_session_conf sess_conf = {
> > > > > > > >                         .action_type = sa->type,
> > > > > > > >                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
> > > > > > > >                         {.ipsec = { @@ -90,247 +65,340 @@
> > > > > > > > create_session(struct ipsec_ctx *ipsec_ctx,
> > > > > struct
> > > > > > > > ipsec_sa *sa)
> > > > > > > >                         } },
> > > > > > > >                         .crypto_xform = sa->xforms,
> > > > > > > >                         .userdata = NULL,
> > > > > > > > -
> > > > > > > >                 };
> > > > > > > >
> > > > > > > > -               if (sa->type ==
> > > > > > > RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL)
> > > > > > > > {
> > > > > > > > -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > > > > > > -                                                       rte_cryptodev_get_sec_ctx(
> > > > > > > > -                                                       ipsec_ctx->tbl[cdev_id_qp].id);
> > > > > > > > -
> > > > > > > > -                       /* Set IPsec parameters in conf */
> > > > > > > > -                       set_ipsec_conf(sa, &(sess_conf.ipsec));
> > > > > > > > -
> > > > > > > > -                       sa->sec_session = rte_security_session_create(ctx,
> > > > > > > > -                                       &sess_conf, ipsec_ctx->session_pool);
> > > > > > > > -                       if (sa->sec_session == NULL) {
> > > > > > > > -                               RTE_LOG(ERR, IPSEC,
> > > > > > > > -                               "SEC Session init failed: err: %d\n", ret);
> > > > > > > > -                               return -1;
> > > > > > > > -                       }
> > > > > > > > -               } else if (sa->type ==
> > > > > RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
> > > > > > > {
> > > > > > > > -                       struct rte_flow_error err;
> > > > > > > > -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > > > > > > -                                                       rte_eth_dev_get_sec_ctx(
> > > > > > > > -                                                       sa->portid);
> > > > > > > > -                       const struct rte_security_capability *sec_cap;
> > > > > > > > -                       int ret = 0;
> > > > > > > > -
> > > > > > > > -                       sa->sec_session = rte_security_session_create(ctx,
> > > > > > > > -                                       &sess_conf, ipsec_ctx->session_pool);
> > > > > > > > -                       if (sa->sec_session == NULL) {
> > > > > > > > -                               RTE_LOG(ERR, IPSEC,
> > > > > > > > -                               "SEC Session init failed: err: %d\n", ret);
> > > > > > > > -                               return -1;
> > > > > > > > -                       }
> > > > > > > > +       if (sa->type ==
> > > > > RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > > > > > > > +               ctx = (struct rte_security_ctx *)
> > > > > > > > +
> > > > > > > > + rte_eth_dev_get_sec_ctx(sa->portid);
> > > > > > >
> > > > > > > This is breaking the lookaside mode. Ctx was retrieved using
> > > > > > > the
> > > ipsec_ctx-
> > > > > >tbl
> > > > > > > struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > > > > > 				rte_cryptodev_get_sec_ctx(
> > > > > > > 				ipsec_ctx->tbl[cdev_id_qp].id);
> > > > > > >
> > > > > > > I am looking into it, but I don't have time left to get it integrated in RC2.
> > > So
> > > > > this
> > > > > > > has to be pushed to RC3
> > > > > >
> > > > > > It looks like there are multiple issues in this patch wrt
> > > > > > lookaside and none
> > > cases.
> > > > > Only the inline cases seem to be working.
> > > > > >
> > > > > > 1. the patch removes the cdev_mapping concept completely.
> > > > > > Cdev_id_qp is
> > > > > not getting used.
> > > > >
> > > > > Not exactly.
> > > > > cdev_id_qp is still setup, and is still used to decide to which
> > > > > crypto-dev to enqueuer the crypto-op:
> > > > > ipsec_enqueue(...)
> > > > > {
> > > > >    ...
> > > > >    enqueue_cop(&ipsec_ctx->tbl[sa->cdev_id_qp], &priv->cop);
> > > >
> > > > I don't see anybody filling "sa->cdev_id_qp". Please let me know if
> > > > I have
> > > missed it somewhere.
> > > > It is memset to 0 I guess.
> > >
> > >
> > > Yep, true we lost it somewhere during the rework.
> > >
> > > >
> > > > >
> > > > >
> > > > > Same in ipsec_process().
> > > > >
> > > > > For initialization, yes cdev_id_qp is not used anymore.
> > > > > As discussed here:
> > > > >
> > > https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fmail
> > > s.dp
> > > > > dk.org%2Farchives%2Fdev%2F2019-
> > > > >
> > >
> > March%2F127725.html&data=02%7C01%7Cakhil.goyal%40nxp.com%7C04
> > > > >
> > >
> > 194193cfc04c0b629008d6c7eea247%7C686ea1d3bc2b4c6fa92cd99c5c301635%
> > > > >
> > >
> > 7C0%7C0%7C636916225072561313&sdata=ga9IiqhYRWOz9QkRDIXNiigInk
> > > > > soVGgu1E5EetqvE%2FA%3D&reserved=0
> > > > >
> > > > > I think the problem you are hitting with lookaside-proto is that
> > > > > for it we use 2 different values here:
> > > > > a) In create_sec_session we use portid (it also should be
> > > > > rte_cryptodev_get_sec_ctx() here)
> > > > >     if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > > > >                 ctx = (struct rte_security_ctx *)
> > > > >
> > > > > rte_eth_dev_get_sec_ctx(sa->portid);
> > > > It should be rte_cryptodev_get_sec_ctx in the first place. And it
> > > > needs a
> > > cdev_id as input based on the cdev mapping done while initializing
> > > > the cryptodev and neither the portid and nor cdev_id_qp.
> > > > > b) in enqueue() we use cdev_id_qp
> > > > >
> > > > > Right now these values could be different.
> > > > > As I understand we need to make sure that fro lookaside-proto
> > > > > cdev_id_qp
> > > ==
> > > > > portid provided by user, correct?
> > > > No it is not the case. Right now for lookaside there is no use of
> > > > portid in case
> > > of lookaside case.
> > > > As the cdev/qp/core mappings are managed internally and the user
> > > > cannot
> > > tweak it from cfg file.
> > > >
> > >
> > >
> > > Hmm, then at least that line  is wrong here:
> > > https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdoc.
> > > dpdk
> > >
> > .org%2Fguides%2Fsample_app_ug%2Fipsec_secgw.html&data=02%7C01%
> > >
> > 7Cakhil.goyal%40nxp.com%7Cb1e931a9967943887a0c08d6c7f49d28%7C686ea
> > >
> > 1d3bc2b4c6fa92cd99c5c301635%7C0%7C0%7C636916250754452306&sda
> > >
> > ta=tNjHCO0O2rfh8shhQXu93qrM0Hr0OZVXUVhMcsg53dw%3D&reserved=
> > > 0
> > >
> > > sa out 5 cipher_algo aes-128-cbc cipher_key
> > > 0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0 \ auth_algo sha1-hmac auth_key
> > > 0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0 \ mode ipv4-tunnel src
> > > 172.16.1.5 dst 172.16.2.5 \ type lookaside-protocol-offload port_id 4
> > >
> > > And probably:
> > > "Port/device ID of the ethernet/crypto accelerator for which the SA is
> > > configured."
> > > Need to be rephrased to remove crypto accelerator notice.
> > Yes.
> >
> > >
> > > Another question - why you guys don't consider using portid for lookaside-
> > proto?
> > > As I can see add_mapping(function that  fills cdev_id_qp) doesn't
> > > bother to check which rte_security protocols are supported (only
> > > crypto capabilities are checked).
> > > So not sure does current code will work ok with a mix of lookaside-
> > > none/lookaside-proto devices.
> > > Forcing user to specify crypto-dev id for lookaside-proto (via portid
> > > or so) will simplify things significantly.
> > I will think about it. Actually initially when portid was introduced, the intent was
> > same but it did not work well.
> > Because there may be a case of a cryptodev with multiple queues, so there will
> > be a mapping done internally in the application and matching it with the user
> > provided portid will be difficult.
> >
> > I believe that this could be done but would need some rework.
> >
> > >
> > > Konstantin
> 
> Could we consider going back to the V2 version of this patch set which fixed the issue with the inline code and left the lookaside code
> unchanged?

Actually I have the same thoughts here:
considering the problems with lookaside-proto, it seems more pragmatic 
to fix it as was suggested in V2:
split create_session() into 2 functions, and invoke create_session_inline() at startup,
while keeping create_session_lookaside() invocation at runtime.
At least it would fix the issue in a clean way.
If later ipsec-secgw will be reworked to allow lookaside session creation at init time,
we can merge them back. 
Konstantin 

> 
> We are not in a position test any changes to the lookaside code.
> 
> Regards,
> 
> Bernard.
> 

Re: [dpdk-dev] [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Ananyev, Konstantin]

> > > > > > > Subject: RE: [PATCH v4 1/2] examples/ipsec-secgw: fix 1st
> > > > > > > packet
> > > dropped
> > > > > for
> > > > > > > inline crypto
> > > > > > >
> > > > > > > Hi Bernard,
> > > > > > >
> > > > > > > > -       RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on
> > > > > cryptodev "
> > > > > > > > -                       "%u qp %u\n", sa->spi,
> > > > > > > > -                       ipsec_ctx->tbl[cdev_id_qp].id,
> > > > > > > > -                       ipsec_ctx->tbl[cdev_id_qp].qp);
> > > > > > > > +       if ((sa == NULL) || (pool == NULL))
> > > > > > > > +               return -EINVAL;
> > > > > > > >
> > > > > > > > -       if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
> > > > > > > > -               struct rte_security_session_conf sess_conf = {
> > > > > > > > +       struct rte_security_session_conf sess_conf = {
> > > > > > > >                         .action_type = sa->type,
> > > > > > > >                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
> > > > > > > >                         {.ipsec = { @@ -90,247 +65,340 @@
> > > > > > > > create_session(struct ipsec_ctx *ipsec_ctx,
> > > > > struct
> > > > > > > > ipsec_sa *sa)
> > > > > > > >                         } },
> > > > > > > >                         .crypto_xform = sa->xforms,
> > > > > > > >                         .userdata = NULL,
> > > > > > > > -
> > > > > > > >                 };
> > > > > > > >
> > > > > > > > -               if (sa->type ==
> > > > > > > RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL)
> > > > > > > > {
> > > > > > > > -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > > > > > > -                                                       rte_cryptodev_get_sec_ctx(
> > > > > > > > -                                                       ipsec_ctx->tbl[cdev_id_qp].id);
> > > > > > > > -
> > > > > > > > -                       /* Set IPsec parameters in conf */
> > > > > > > > -                       set_ipsec_conf(sa, &(sess_conf.ipsec));
> > > > > > > > -
> > > > > > > > -                       sa->sec_session = rte_security_session_create(ctx,
> > > > > > > > -                                       &sess_conf, ipsec_ctx->session_pool);
> > > > > > > > -                       if (sa->sec_session == NULL) {
> > > > > > > > -                               RTE_LOG(ERR, IPSEC,
> > > > > > > > -                               "SEC Session init failed: err: %d\n", ret);
> > > > > > > > -                               return -1;
> > > > > > > > -                       }
> > > > > > > > -               } else if (sa->type ==
> > > > > RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
> > > > > > > {
> > > > > > > > -                       struct rte_flow_error err;
> > > > > > > > -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > > > > > > -                                                       rte_eth_dev_get_sec_ctx(
> > > > > > > > -                                                       sa->portid);
> > > > > > > > -                       const struct rte_security_capability *sec_cap;
> > > > > > > > -                       int ret = 0;
> > > > > > > > -
> > > > > > > > -                       sa->sec_session = rte_security_session_create(ctx,
> > > > > > > > -                                       &sess_conf, ipsec_ctx->session_pool);
> > > > > > > > -                       if (sa->sec_session == NULL) {
> > > > > > > > -                               RTE_LOG(ERR, IPSEC,
> > > > > > > > -                               "SEC Session init failed: err: %d\n", ret);
> > > > > > > > -                               return -1;
> > > > > > > > -                       }
> > > > > > > > +       if (sa->type ==
> > > > > RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > > > > > > > +               ctx = (struct rte_security_ctx *)
> > > > > > > > +
> > > > > > > > + rte_eth_dev_get_sec_ctx(sa->portid);
> > > > > > >
> > > > > > > This is breaking the lookaside mode. Ctx was retrieved using
> > > > > > > the
> > > ipsec_ctx-
> > > > > >tbl
> > > > > > > struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > > > > > 				rte_cryptodev_get_sec_ctx(
> > > > > > > 				ipsec_ctx->tbl[cdev_id_qp].id);
> > > > > > >
> > > > > > > I am looking into it, but I don't have time left to get it integrated in RC2.
> > > So
> > > > > this
> > > > > > > has to be pushed to RC3
> > > > > >
> > > > > > It looks like there are multiple issues in this patch wrt
> > > > > > lookaside and none
> > > cases.
> > > > > Only the inline cases seem to be working.
> > > > > >
> > > > > > 1. the patch removes the cdev_mapping concept completely.
> > > > > > Cdev_id_qp is
> > > > > not getting used.
> > > > >
> > > > > Not exactly.
> > > > > cdev_id_qp is still setup, and is still used to decide to which
> > > > > crypto-dev to enqueuer the crypto-op:
> > > > > ipsec_enqueue(...)
> > > > > {
> > > > >    ...
> > > > >    enqueue_cop(&ipsec_ctx->tbl[sa->cdev_id_qp], &priv->cop);
> > > >
> > > > I don't see anybody filling "sa->cdev_id_qp". Please let me know if
> > > > I have
> > > missed it somewhere.
> > > > It is memset to 0 I guess.
> > >
> > >
> > > Yep, true we lost it somewhere during the rework.
> > >
> > > >
> > > > >
> > > > >
> > > > > Same in ipsec_process().
> > > > >
> > > > > For initialization, yes cdev_id_qp is not used anymore.
> > > > > As discussed here:
> > > > >
> > > https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fmail
> > > s.dp
> > > > > dk.org%2Farchives%2Fdev%2F2019-
> > > > >
> > >
> > March%2F127725.html&data=02%7C01%7Cakhil.goyal%40nxp.com%7C04
> > > > >
> > >
> > 194193cfc04c0b629008d6c7eea247%7C686ea1d3bc2b4c6fa92cd99c5c301635%
> > > > >
> > >
> > 7C0%7C0%7C636916225072561313&sdata=ga9IiqhYRWOz9QkRDIXNiigInk
> > > > > soVGgu1E5EetqvE%2FA%3D&reserved=0
> > > > >
> > > > > I think the problem you are hitting with lookaside-proto is that
> > > > > for it we use 2 different values here:
> > > > > a) In create_sec_session we use portid (it also should be
> > > > > rte_cryptodev_get_sec_ctx() here)
> > > > >     if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > > > >                 ctx = (struct rte_security_ctx *)
> > > > >
> > > > > rte_eth_dev_get_sec_ctx(sa->portid);
> > > > It should be rte_cryptodev_get_sec_ctx in the first place. And it
> > > > needs a
> > > cdev_id as input based on the cdev mapping done while initializing
> > > > the cryptodev and neither the portid and nor cdev_id_qp.
> > > > > b) in enqueue() we use cdev_id_qp
> > > > >
> > > > > Right now these values could be different.
> > > > > As I understand we need to make sure that fro lookaside-proto
> > > > > cdev_id_qp
> > > ==
> > > > > portid provided by user, correct?
> > > > No it is not the case. Right now for lookaside there is no use of
> > > > portid in case
> > > of lookaside case.
> > > > As the cdev/qp/core mappings are managed internally and the user
> > > > cannot
> > > tweak it from cfg file.
> > > >
> > >
> > >
> > > Hmm, then at least that line  is wrong here:
> > > https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdoc.
> > > dpdk
> > >
> > .org%2Fguides%2Fsample_app_ug%2Fipsec_secgw.html&data=02%7C01%
> > >
> > 7Cakhil.goyal%40nxp.com%7Cb1e931a9967943887a0c08d6c7f49d28%7C686ea
> > >
> > 1d3bc2b4c6fa92cd99c5c301635%7C0%7C0%7C636916250754452306&sda
> > >
> > ta=tNjHCO0O2rfh8shhQXu93qrM0Hr0OZVXUVhMcsg53dw%3D&reserved=
> > > 0
> > >
> > > sa out 5 cipher_algo aes-128-cbc cipher_key
> > > 0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0 \ auth_algo sha1-hmac auth_key
> > > 0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0 \ mode ipv4-tunnel src
> > > 172.16.1.5 dst 172.16.2.5 \ type lookaside-protocol-offload port_id 4
> > >
> > > And probably:
> > > "Port/device ID of the ethernet/crypto accelerator for which the SA is
> > > configured."
> > > Need to be rephrased to remove crypto accelerator notice.
> > Yes.
> >
> > >
> > > Another question - why you guys don't consider using portid for lookaside-
> > proto?
> > > As I can see add_mapping(function that  fills cdev_id_qp) doesn't
> > > bother to check which rte_security protocols are supported (only
> > > crypto capabilities are checked).
> > > So not sure does current code will work ok with a mix of lookaside-
> > > none/lookaside-proto devices.
> > > Forcing user to specify crypto-dev id for lookaside-proto (via portid
> > > or so) will simplify things significantly.
> > I will think about it. Actually initially when portid was introduced, the intent was
> > same but it did not work well.
> > Because there may be a case of a cryptodev with multiple queues, so there will
> > be a mapping done internally in the application and matching it with the user
> > provided portid will be difficult.
> >
> > I believe that this could be done but would need some rework.
> >
> > >
> > > Konstantin
> 
> Could we consider going back to the V2 version of this patch set which fixed the issue with the inline code and left the lookaside code
> unchanged?

Actually I have the same thoughts here:
considering the problems with lookaside-proto, it seems more pragmatic 
to fix it as was suggested in V2:
split create_session() into 2 functions, and invoke create_session_inline() at startup,
while keeping create_session_lookaside() invocation at runtime.
At least it would fix the issue in a clean way.
If later ipsec-secgw will be reworked to allow lookaside session creation at init time,
we can merge them back. 
Konstantin 

> 
> We are not in a position test any changes to the lookaside code.
> 
> Regards,
> 
> Bernard.
> 

Re: [dpdk-dev] [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Iremonger, Bernard]

Hi Akhil

<snip>

> > > > > > Subject: RE: [PATCH v4 1/2] examples/ipsec-secgw: fix 1st
> > > > > > packet
> > dropped
> > > > for
> > > > > > inline crypto
> > > > > >
> > > > > > Hi Bernard,
> > > > > >
> > > > > > > -       RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on
> > > > cryptodev "
> > > > > > > -                       "%u qp %u\n", sa->spi,
> > > > > > > -                       ipsec_ctx->tbl[cdev_id_qp].id,
> > > > > > > -                       ipsec_ctx->tbl[cdev_id_qp].qp);
> > > > > > > +       if ((sa == NULL) || (pool == NULL))
> > > > > > > +               return -EINVAL;
> > > > > > >
> > > > > > > -       if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
> > > > > > > -               struct rte_security_session_conf sess_conf = {
> > > > > > > +       struct rte_security_session_conf sess_conf = {
> > > > > > >                         .action_type = sa->type,
> > > > > > >                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
> > > > > > >                         {.ipsec = { @@ -90,247 +65,340 @@
> > > > > > > create_session(struct ipsec_ctx *ipsec_ctx,
> > > > struct
> > > > > > > ipsec_sa *sa)
> > > > > > >                         } },
> > > > > > >                         .crypto_xform = sa->xforms,
> > > > > > >                         .userdata = NULL,
> > > > > > > -
> > > > > > >                 };
> > > > > > >
> > > > > > > -               if (sa->type ==
> > > > > > RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL)
> > > > > > > {
> > > > > > > -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > > > > > -                                                       rte_cryptodev_get_sec_ctx(
> > > > > > > -                                                       ipsec_ctx->tbl[cdev_id_qp].id);
> > > > > > > -
> > > > > > > -                       /* Set IPsec parameters in conf */
> > > > > > > -                       set_ipsec_conf(sa, &(sess_conf.ipsec));
> > > > > > > -
> > > > > > > -                       sa->sec_session = rte_security_session_create(ctx,
> > > > > > > -                                       &sess_conf, ipsec_ctx->session_pool);
> > > > > > > -                       if (sa->sec_session == NULL) {
> > > > > > > -                               RTE_LOG(ERR, IPSEC,
> > > > > > > -                               "SEC Session init failed: err: %d\n", ret);
> > > > > > > -                               return -1;
> > > > > > > -                       }
> > > > > > > -               } else if (sa->type ==
> > > > RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
> > > > > > {
> > > > > > > -                       struct rte_flow_error err;
> > > > > > > -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > > > > > -                                                       rte_eth_dev_get_sec_ctx(
> > > > > > > -                                                       sa->portid);
> > > > > > > -                       const struct rte_security_capability *sec_cap;
> > > > > > > -                       int ret = 0;
> > > > > > > -
> > > > > > > -                       sa->sec_session = rte_security_session_create(ctx,
> > > > > > > -                                       &sess_conf, ipsec_ctx->session_pool);
> > > > > > > -                       if (sa->sec_session == NULL) {
> > > > > > > -                               RTE_LOG(ERR, IPSEC,
> > > > > > > -                               "SEC Session init failed: err: %d\n", ret);
> > > > > > > -                               return -1;
> > > > > > > -                       }
> > > > > > > +       if (sa->type ==
> > > > RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > > > > > > +               ctx = (struct rte_security_ctx *)
> > > > > > > +
> > > > > > > + rte_eth_dev_get_sec_ctx(sa->portid);
> > > > > >
> > > > > > This is breaking the lookaside mode. Ctx was retrieved using
> > > > > > the
> > ipsec_ctx-
> > > > >tbl
> > > > > > struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > > > > 				rte_cryptodev_get_sec_ctx(
> > > > > > 				ipsec_ctx->tbl[cdev_id_qp].id);
> > > > > >
> > > > > > I am looking into it, but I don't have time left to get it integrated in RC2.
> > So
> > > > this
> > > > > > has to be pushed to RC3
> > > > >
> > > > > It looks like there are multiple issues in this patch wrt
> > > > > lookaside and none
> > cases.
> > > > Only the inline cases seem to be working.
> > > > >
> > > > > 1. the patch removes the cdev_mapping concept completely.
> > > > > Cdev_id_qp is
> > > > not getting used.
> > > >
> > > > Not exactly.
> > > > cdev_id_qp is still setup, and is still used to decide to which
> > > > crypto-dev to enqueuer the crypto-op:
> > > > ipsec_enqueue(...)
> > > > {
> > > >    ...
> > > >    enqueue_cop(&ipsec_ctx->tbl[sa->cdev_id_qp], &priv->cop);
> > >
> > > I don't see anybody filling "sa->cdev_id_qp". Please let me know if
> > > I have
> > missed it somewhere.
> > > It is memset to 0 I guess.
> >
> >
> > Yep, true we lost it somewhere during the rework.
> >
> > >
> > > >
> > > >
> > > > Same in ipsec_process().
> > > >
> > > > For initialization, yes cdev_id_qp is not used anymore.
> > > > As discussed here:
> > > >
> > https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fmail
> > s.dp
> > > > dk.org%2Farchives%2Fdev%2F2019-
> > > >
> >
> March%2F127725.html&amp;data=02%7C01%7Cakhil.goyal%40nxp.com%7C04
> > > >
> >
> 194193cfc04c0b629008d6c7eea247%7C686ea1d3bc2b4c6fa92cd99c5c301635%
> > > >
> >
> 7C0%7C0%7C636916225072561313&amp;sdata=ga9IiqhYRWOz9QkRDIXNiigInk
> > > > soVGgu1E5EetqvE%2FA%3D&amp;reserved=0
> > > >
> > > > I think the problem you are hitting with lookaside-proto is that
> > > > for it we use 2 different values here:
> > > > a) In create_sec_session we use portid (it also should be
> > > > rte_cryptodev_get_sec_ctx() here)
> > > >     if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > > >                 ctx = (struct rte_security_ctx *)
> > > >
> > > > rte_eth_dev_get_sec_ctx(sa->portid);
> > > It should be rte_cryptodev_get_sec_ctx in the first place. And it
> > > needs a
> > cdev_id as input based on the cdev mapping done while initializing
> > > the cryptodev and neither the portid and nor cdev_id_qp.
> > > > b) in enqueue() we use cdev_id_qp
> > > >
> > > > Right now these values could be different.
> > > > As I understand we need to make sure that fro lookaside-proto
> > > > cdev_id_qp
> > ==
> > > > portid provided by user, correct?
> > > No it is not the case. Right now for lookaside there is no use of
> > > portid in case
> > of lookaside case.
> > > As the cdev/qp/core mappings are managed internally and the user
> > > cannot
> > tweak it from cfg file.
> > >
> >
> >
> > Hmm, then at least that line  is wrong here:
> > https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdoc.
> > dpdk
> >
> .org%2Fguides%2Fsample_app_ug%2Fipsec_secgw.html&amp;data=02%7C01%
> >
> 7Cakhil.goyal%40nxp.com%7Cb1e931a9967943887a0c08d6c7f49d28%7C686ea
> >
> 1d3bc2b4c6fa92cd99c5c301635%7C0%7C0%7C636916250754452306&amp;sda
> >
> ta=tNjHCO0O2rfh8shhQXu93qrM0Hr0OZVXUVhMcsg53dw%3D&amp;reserved=
> > 0
> >
> > sa out 5 cipher_algo aes-128-cbc cipher_key
> > 0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0 \ auth_algo sha1-hmac auth_key
> > 0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0 \ mode ipv4-tunnel src
> > 172.16.1.5 dst 172.16.2.5 \ type lookaside-protocol-offload port_id 4
> >
> > And probably:
> > "Port/device ID of the ethernet/crypto accelerator for which the SA is
> > configured."
> > Need to be rephrased to remove crypto accelerator notice.
> Yes.
> 
> >
> > Another question - why you guys don't consider using portid for lookaside-
> proto?
> > As I can see add_mapping(function that  fills cdev_id_qp) doesn't
> > bother to check which rte_security protocols are supported (only
> > crypto capabilities are checked).
> > So not sure does current code will work ok with a mix of lookaside-
> > none/lookaside-proto devices.
> > Forcing user to specify crypto-dev id for lookaside-proto (via portid
> > or so) will simplify things significantly.
> I will think about it. Actually initially when portid was introduced, the intent was
> same but it did not work well.
> Because there may be a case of a cryptodev with multiple queues, so there will
> be a mapping done internally in the application and matching it with the user
> provided portid will be difficult.
> 
> I believe that this could be done but would need some rework.
> 
> >
> > Konstantin

Could we consider going back to the V2 version of this patch set which fixed the issue with the inline code and left the lookaside code unchanged?

We are not in a position test any changes to the lookaside code.

Regards,

Bernard.

Re: [dpdk-dev] [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Iremonger, Bernard]

Hi Akhil

<snip>

> > > > > > Subject: RE: [PATCH v4 1/2] examples/ipsec-secgw: fix 1st
> > > > > > packet
> > dropped
> > > > for
> > > > > > inline crypto
> > > > > >
> > > > > > Hi Bernard,
> > > > > >
> > > > > > > -       RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on
> > > > cryptodev "
> > > > > > > -                       "%u qp %u\n", sa->spi,
> > > > > > > -                       ipsec_ctx->tbl[cdev_id_qp].id,
> > > > > > > -                       ipsec_ctx->tbl[cdev_id_qp].qp);
> > > > > > > +       if ((sa == NULL) || (pool == NULL))
> > > > > > > +               return -EINVAL;
> > > > > > >
> > > > > > > -       if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
> > > > > > > -               struct rte_security_session_conf sess_conf = {
> > > > > > > +       struct rte_security_session_conf sess_conf = {
> > > > > > >                         .action_type = sa->type,
> > > > > > >                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
> > > > > > >                         {.ipsec = { @@ -90,247 +65,340 @@
> > > > > > > create_session(struct ipsec_ctx *ipsec_ctx,
> > > > struct
> > > > > > > ipsec_sa *sa)
> > > > > > >                         } },
> > > > > > >                         .crypto_xform = sa->xforms,
> > > > > > >                         .userdata = NULL,
> > > > > > > -
> > > > > > >                 };
> > > > > > >
> > > > > > > -               if (sa->type ==
> > > > > > RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL)
> > > > > > > {
> > > > > > > -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > > > > > -                                                       rte_cryptodev_get_sec_ctx(
> > > > > > > -                                                       ipsec_ctx->tbl[cdev_id_qp].id);
> > > > > > > -
> > > > > > > -                       /* Set IPsec parameters in conf */
> > > > > > > -                       set_ipsec_conf(sa, &(sess_conf.ipsec));
> > > > > > > -
> > > > > > > -                       sa->sec_session = rte_security_session_create(ctx,
> > > > > > > -                                       &sess_conf, ipsec_ctx->session_pool);
> > > > > > > -                       if (sa->sec_session == NULL) {
> > > > > > > -                               RTE_LOG(ERR, IPSEC,
> > > > > > > -                               "SEC Session init failed: err: %d\n", ret);
> > > > > > > -                               return -1;
> > > > > > > -                       }
> > > > > > > -               } else if (sa->type ==
> > > > RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
> > > > > > {
> > > > > > > -                       struct rte_flow_error err;
> > > > > > > -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > > > > > -                                                       rte_eth_dev_get_sec_ctx(
> > > > > > > -                                                       sa->portid);
> > > > > > > -                       const struct rte_security_capability *sec_cap;
> > > > > > > -                       int ret = 0;
> > > > > > > -
> > > > > > > -                       sa->sec_session = rte_security_session_create(ctx,
> > > > > > > -                                       &sess_conf, ipsec_ctx->session_pool);
> > > > > > > -                       if (sa->sec_session == NULL) {
> > > > > > > -                               RTE_LOG(ERR, IPSEC,
> > > > > > > -                               "SEC Session init failed: err: %d\n", ret);
> > > > > > > -                               return -1;
> > > > > > > -                       }
> > > > > > > +       if (sa->type ==
> > > > RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > > > > > > +               ctx = (struct rte_security_ctx *)
> > > > > > > +
> > > > > > > + rte_eth_dev_get_sec_ctx(sa->portid);
> > > > > >
> > > > > > This is breaking the lookaside mode. Ctx was retrieved using
> > > > > > the
> > ipsec_ctx-
> > > > >tbl
> > > > > > struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > > > > 				rte_cryptodev_get_sec_ctx(
> > > > > > 				ipsec_ctx->tbl[cdev_id_qp].id);
> > > > > >
> > > > > > I am looking into it, but I don't have time left to get it integrated in RC2.
> > So
> > > > this
> > > > > > has to be pushed to RC3
> > > > >
> > > > > It looks like there are multiple issues in this patch wrt
> > > > > lookaside and none
> > cases.
> > > > Only the inline cases seem to be working.
> > > > >
> > > > > 1. the patch removes the cdev_mapping concept completely.
> > > > > Cdev_id_qp is
> > > > not getting used.
> > > >
> > > > Not exactly.
> > > > cdev_id_qp is still setup, and is still used to decide to which
> > > > crypto-dev to enqueuer the crypto-op:
> > > > ipsec_enqueue(...)
> > > > {
> > > >    ...
> > > >    enqueue_cop(&ipsec_ctx->tbl[sa->cdev_id_qp], &priv->cop);
> > >
> > > I don't see anybody filling "sa->cdev_id_qp". Please let me know if
> > > I have
> > missed it somewhere.
> > > It is memset to 0 I guess.
> >
> >
> > Yep, true we lost it somewhere during the rework.
> >
> > >
> > > >
> > > >
> > > > Same in ipsec_process().
> > > >
> > > > For initialization, yes cdev_id_qp is not used anymore.
> > > > As discussed here:
> > > >
> > https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fmail
> > s.dp
> > > > dk.org%2Farchives%2Fdev%2F2019-
> > > >
> >
> March%2F127725.html&amp;data=02%7C01%7Cakhil.goyal%40nxp.com%7C04
> > > >
> >
> 194193cfc04c0b629008d6c7eea247%7C686ea1d3bc2b4c6fa92cd99c5c301635%
> > > >
> >
> 7C0%7C0%7C636916225072561313&amp;sdata=ga9IiqhYRWOz9QkRDIXNiigInk
> > > > soVGgu1E5EetqvE%2FA%3D&amp;reserved=0
> > > >
> > > > I think the problem you are hitting with lookaside-proto is that
> > > > for it we use 2 different values here:
> > > > a) In create_sec_session we use portid (it also should be
> > > > rte_cryptodev_get_sec_ctx() here)
> > > >     if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > > >                 ctx = (struct rte_security_ctx *)
> > > >
> > > > rte_eth_dev_get_sec_ctx(sa->portid);
> > > It should be rte_cryptodev_get_sec_ctx in the first place. And it
> > > needs a
> > cdev_id as input based on the cdev mapping done while initializing
> > > the cryptodev and neither the portid and nor cdev_id_qp.
> > > > b) in enqueue() we use cdev_id_qp
> > > >
> > > > Right now these values could be different.
> > > > As I understand we need to make sure that fro lookaside-proto
> > > > cdev_id_qp
> > ==
> > > > portid provided by user, correct?
> > > No it is not the case. Right now for lookaside there is no use of
> > > portid in case
> > of lookaside case.
> > > As the cdev/qp/core mappings are managed internally and the user
> > > cannot
> > tweak it from cfg file.
> > >
> >
> >
> > Hmm, then at least that line  is wrong here:
> > https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdoc.
> > dpdk
> >
> .org%2Fguides%2Fsample_app_ug%2Fipsec_secgw.html&amp;data=02%7C01%
> >
> 7Cakhil.goyal%40nxp.com%7Cb1e931a9967943887a0c08d6c7f49d28%7C686ea
> >
> 1d3bc2b4c6fa92cd99c5c301635%7C0%7C0%7C636916250754452306&amp;sda
> >
> ta=tNjHCO0O2rfh8shhQXu93qrM0Hr0OZVXUVhMcsg53dw%3D&amp;reserved=
> > 0
> >
> > sa out 5 cipher_algo aes-128-cbc cipher_key
> > 0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0 \ auth_algo sha1-hmac auth_key
> > 0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0 \ mode ipv4-tunnel src
> > 172.16.1.5 dst 172.16.2.5 \ type lookaside-protocol-offload port_id 4
> >
> > And probably:
> > "Port/device ID of the ethernet/crypto accelerator for which the SA is
> > configured."
> > Need to be rephrased to remove crypto accelerator notice.
> Yes.
> 
> >
> > Another question - why you guys don't consider using portid for lookaside-
> proto?
> > As I can see add_mapping(function that  fills cdev_id_qp) doesn't
> > bother to check which rte_security protocols are supported (only
> > crypto capabilities are checked).
> > So not sure does current code will work ok with a mix of lookaside-
> > none/lookaside-proto devices.
> > Forcing user to specify crypto-dev id for lookaside-proto (via portid
> > or so) will simplify things significantly.
> I will think about it. Actually initially when portid was introduced, the intent was
> same but it did not work well.
> Because there may be a case of a cryptodev with multiple queues, so there will
> be a mapping done internally in the application and matching it with the user
> provided portid will be difficult.
> 
> I believe that this could be done but would need some rework.
> 
> >
> > Konstantin

Could we consider going back to the V2 version of this patch set which fixed the issue with the inline code and left the lookaside code unchanged?

We are not in a position test any changes to the lookaside code.

Regards,

Bernard.

Re: [dpdk-dev] [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Akhil Goyal]

> -----Original Message-----
> From: Ananyev, Konstantin <konstantin.ananyev@intel.com>
> Sent: Tuesday, April 23, 2019 7:35 PM
> To: Akhil Goyal <akhil.goyal@nxp.com>; Iremonger, Bernard
> <bernard.iremonger@intel.com>; dev@dpdk.org
> Cc: stable@dpdk.org
> Subject: RE: [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for
> inline crypto
> 
> 
> 
> > >
> > > >
> > > > > -----Original Message-----
> > > > > From: Akhil Goyal
> > > > > Sent: Thursday, April 18, 2019 7:21 PM
> > > > > To: Bernard Iremonger <bernard.iremonger@intel.com>; dev@dpdk.org;
> > > > > konstantin.ananyev@intel.com
> > > > > Cc: stable@dpdk.org
> > > > > Subject: RE: [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet
> dropped
> > > for
> > > > > inline crypto
> > > > >
> > > > > Hi Bernard,
> > > > >
> > > > > > -       RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on
> > > cryptodev "
> > > > > > -                       "%u qp %u\n", sa->spi,
> > > > > > -                       ipsec_ctx->tbl[cdev_id_qp].id,
> > > > > > -                       ipsec_ctx->tbl[cdev_id_qp].qp);
> > > > > > +       if ((sa == NULL) || (pool == NULL))
> > > > > > +               return -EINVAL;
> > > > > >
> > > > > > -       if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
> > > > > > -               struct rte_security_session_conf sess_conf = {
> > > > > > +       struct rte_security_session_conf sess_conf = {
> > > > > >                         .action_type = sa->type,
> > > > > >                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
> > > > > >                         {.ipsec = {
> > > > > > @@ -90,247 +65,340 @@ create_session(struct ipsec_ctx *ipsec_ctx,
> > > struct
> > > > > > ipsec_sa *sa)
> > > > > >                         } },
> > > > > >                         .crypto_xform = sa->xforms,
> > > > > >                         .userdata = NULL,
> > > > > > -
> > > > > >                 };
> > > > > >
> > > > > > -               if (sa->type ==
> > > > > RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL)
> > > > > > {
> > > > > > -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > > > > -                                                       rte_cryptodev_get_sec_ctx(
> > > > > > -                                                       ipsec_ctx->tbl[cdev_id_qp].id);
> > > > > > -
> > > > > > -                       /* Set IPsec parameters in conf */
> > > > > > -                       set_ipsec_conf(sa, &(sess_conf.ipsec));
> > > > > > -
> > > > > > -                       sa->sec_session = rte_security_session_create(ctx,
> > > > > > -                                       &sess_conf, ipsec_ctx->session_pool);
> > > > > > -                       if (sa->sec_session == NULL) {
> > > > > > -                               RTE_LOG(ERR, IPSEC,
> > > > > > -                               "SEC Session init failed: err: %d\n", ret);
> > > > > > -                               return -1;
> > > > > > -                       }
> > > > > > -               } else if (sa->type ==
> > > RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
> > > > > {
> > > > > > -                       struct rte_flow_error err;
> > > > > > -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > > > > -                                                       rte_eth_dev_get_sec_ctx(
> > > > > > -                                                       sa->portid);
> > > > > > -                       const struct rte_security_capability *sec_cap;
> > > > > > -                       int ret = 0;
> > > > > > -
> > > > > > -                       sa->sec_session = rte_security_session_create(ctx,
> > > > > > -                                       &sess_conf, ipsec_ctx->session_pool);
> > > > > > -                       if (sa->sec_session == NULL) {
> > > > > > -                               RTE_LOG(ERR, IPSEC,
> > > > > > -                               "SEC Session init failed: err: %d\n", ret);
> > > > > > -                               return -1;
> > > > > > -                       }
> > > > > > +       if (sa->type ==
> > > RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > > > > > +               ctx = (struct rte_security_ctx *)
> > > > > > +                               rte_eth_dev_get_sec_ctx(sa->portid);
> > > > >
> > > > > This is breaking the lookaside mode. Ctx was retrieved using the
> ipsec_ctx-
> > > >tbl
> > > > > struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > > > 				rte_cryptodev_get_sec_ctx(
> > > > > 				ipsec_ctx->tbl[cdev_id_qp].id);
> > > > >
> > > > > I am looking into it, but I don't have time left to get it integrated in RC2.
> So
> > > this
> > > > > has to be pushed to RC3
> > > >
> > > > It looks like there are multiple issues in this patch wrt lookaside and none
> cases.
> > > Only the inline cases seem to be working.
> > > >
> > > > 1. the patch removes the cdev_mapping concept completely. Cdev_id_qp is
> > > not getting used.
> > >
> > > Not exactly.
> > > cdev_id_qp is still setup, and is still used to decide to which crypto-dev to
> > > enqueuer the crypto-op:
> > > ipsec_enqueue(...)
> > > {
> > >    ...
> > >    enqueue_cop(&ipsec_ctx->tbl[sa->cdev_id_qp], &priv->cop);
> >
> > I don't see anybody filling "sa->cdev_id_qp". Please let me know if I have
> missed it somewhere.
> > It is memset to 0 I guess.
> 
> 
> Yep, true we lost it somewhere during the rework.
> 
> >
> > >
> > >
> > > Same in ipsec_process().
> > >
> > > For initialization, yes cdev_id_qp is not used anymore.
> > > As discussed here:
> > >
> https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fmails.dp
> > > dk.org%2Farchives%2Fdev%2F2019-
> > >
> March%2F127725.html&amp;data=02%7C01%7Cakhil.goyal%40nxp.com%7C04
> > >
> 194193cfc04c0b629008d6c7eea247%7C686ea1d3bc2b4c6fa92cd99c5c301635%
> > >
> 7C0%7C0%7C636916225072561313&amp;sdata=ga9IiqhYRWOz9QkRDIXNiigInk
> > > soVGgu1E5EetqvE%2FA%3D&amp;reserved=0
> > >
> > > I think the problem you are hitting with lookaside-proto is that for it
> > > we use 2 different values here:
> > > a) In create_sec_session we use portid (it also should be
> > > rte_cryptodev_get_sec_ctx() here)
> > >     if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > >                 ctx = (struct rte_security_ctx *)
> > >                                 rte_eth_dev_get_sec_ctx(sa->portid);
> > It should be rte_cryptodev_get_sec_ctx in the first place. And it needs a
> cdev_id as input based on the cdev mapping done while initializing
> > the cryptodev and neither the portid and nor cdev_id_qp.
> > > b) in enqueue() we use cdev_id_qp
> > >
> > > Right now these values could be different.
> > > As I understand we need to make sure that fro lookaside-proto cdev_id_qp
> ==
> > > portid provided by user, correct?
> > No it is not the case. Right now for lookaside there is no use of portid in case
> of lookaside case.
> > As the cdev/qp/core mappings are managed internally and the user cannot
> tweak it from cfg file.
> >
> 
> 
> Hmm, then at least that line  is wrong here:
> https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdoc.dpdk
> .org%2Fguides%2Fsample_app_ug%2Fipsec_secgw.html&amp;data=02%7C01%
> 7Cakhil.goyal%40nxp.com%7Cb1e931a9967943887a0c08d6c7f49d28%7C686ea
> 1d3bc2b4c6fa92cd99c5c301635%7C0%7C0%7C636916250754452306&amp;sda
> ta=tNjHCO0O2rfh8shhQXu93qrM0Hr0OZVXUVhMcsg53dw%3D&amp;reserved=
> 0
> 
> sa out 5 cipher_algo aes-128-cbc cipher_key 0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0 \
> auth_algo sha1-hmac auth_key 0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0 \
> mode ipv4-tunnel src 172.16.1.5 dst 172.16.2.5 \
> type lookaside-protocol-offload port_id 4
> 
> And probably:
> "Port/device ID of the ethernet/crypto accelerator for which the SA is
> configured."
> Need to be rephrased to remove crypto accelerator notice.
Yes.

> 
> Another question - why you guys don't consider using portid for lookaside-proto?
> As I can see add_mapping(function that  fills cdev_id_qp) doesn't bother to
> check
> which rte_security protocols are supported (only crypto capabilities are checked).
> So not sure does current code will work ok with a mix of lookaside-
> none/lookaside-proto devices.
> Forcing user to specify crypto-dev id for lookaside-proto (via portid or so)
> will simplify things significantly.
I will think about it. Actually initially when portid was introduced, the intent was same but it did not work well.
Because there may be a case of a cryptodev with multiple queues, so there will be a mapping done internally in the application and matching it with the user provided portid will be difficult.

I believe that this could be done but would need some rework.

> 
> Konstantin

Re: [dpdk-dev] [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Akhil Goyal]

> -----Original Message-----
> From: Ananyev, Konstantin <konstantin.ananyev@intel.com>
> Sent: Tuesday, April 23, 2019 7:35 PM
> To: Akhil Goyal <akhil.goyal@nxp.com>; Iremonger, Bernard
> <bernard.iremonger@intel.com>; dev@dpdk.org
> Cc: stable@dpdk.org
> Subject: RE: [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for
> inline crypto
> 
> 
> 
> > >
> > > >
> > > > > -----Original Message-----
> > > > > From: Akhil Goyal
> > > > > Sent: Thursday, April 18, 2019 7:21 PM
> > > > > To: Bernard Iremonger <bernard.iremonger@intel.com>; dev@dpdk.org;
> > > > > konstantin.ananyev@intel.com
> > > > > Cc: stable@dpdk.org
> > > > > Subject: RE: [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet
> dropped
> > > for
> > > > > inline crypto
> > > > >
> > > > > Hi Bernard,
> > > > >
> > > > > > -       RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on
> > > cryptodev "
> > > > > > -                       "%u qp %u\n", sa->spi,
> > > > > > -                       ipsec_ctx->tbl[cdev_id_qp].id,
> > > > > > -                       ipsec_ctx->tbl[cdev_id_qp].qp);
> > > > > > +       if ((sa == NULL) || (pool == NULL))
> > > > > > +               return -EINVAL;
> > > > > >
> > > > > > -       if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
> > > > > > -               struct rte_security_session_conf sess_conf = {
> > > > > > +       struct rte_security_session_conf sess_conf = {
> > > > > >                         .action_type = sa->type,
> > > > > >                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
> > > > > >                         {.ipsec = {
> > > > > > @@ -90,247 +65,340 @@ create_session(struct ipsec_ctx *ipsec_ctx,
> > > struct
> > > > > > ipsec_sa *sa)
> > > > > >                         } },
> > > > > >                         .crypto_xform = sa->xforms,
> > > > > >                         .userdata = NULL,
> > > > > > -
> > > > > >                 };
> > > > > >
> > > > > > -               if (sa->type ==
> > > > > RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL)
> > > > > > {
> > > > > > -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > > > > -                                                       rte_cryptodev_get_sec_ctx(
> > > > > > -                                                       ipsec_ctx->tbl[cdev_id_qp].id);
> > > > > > -
> > > > > > -                       /* Set IPsec parameters in conf */
> > > > > > -                       set_ipsec_conf(sa, &(sess_conf.ipsec));
> > > > > > -
> > > > > > -                       sa->sec_session = rte_security_session_create(ctx,
> > > > > > -                                       &sess_conf, ipsec_ctx->session_pool);
> > > > > > -                       if (sa->sec_session == NULL) {
> > > > > > -                               RTE_LOG(ERR, IPSEC,
> > > > > > -                               "SEC Session init failed: err: %d\n", ret);
> > > > > > -                               return -1;
> > > > > > -                       }
> > > > > > -               } else if (sa->type ==
> > > RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
> > > > > {
> > > > > > -                       struct rte_flow_error err;
> > > > > > -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > > > > -                                                       rte_eth_dev_get_sec_ctx(
> > > > > > -                                                       sa->portid);
> > > > > > -                       const struct rte_security_capability *sec_cap;
> > > > > > -                       int ret = 0;
> > > > > > -
> > > > > > -                       sa->sec_session = rte_security_session_create(ctx,
> > > > > > -                                       &sess_conf, ipsec_ctx->session_pool);
> > > > > > -                       if (sa->sec_session == NULL) {
> > > > > > -                               RTE_LOG(ERR, IPSEC,
> > > > > > -                               "SEC Session init failed: err: %d\n", ret);
> > > > > > -                               return -1;
> > > > > > -                       }
> > > > > > +       if (sa->type ==
> > > RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > > > > > +               ctx = (struct rte_security_ctx *)
> > > > > > +                               rte_eth_dev_get_sec_ctx(sa->portid);
> > > > >
> > > > > This is breaking the lookaside mode. Ctx was retrieved using the
> ipsec_ctx-
> > > >tbl
> > > > > struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > > > 				rte_cryptodev_get_sec_ctx(
> > > > > 				ipsec_ctx->tbl[cdev_id_qp].id);
> > > > >
> > > > > I am looking into it, but I don't have time left to get it integrated in RC2.
> So
> > > this
> > > > > has to be pushed to RC3
> > > >
> > > > It looks like there are multiple issues in this patch wrt lookaside and none
> cases.
> > > Only the inline cases seem to be working.
> > > >
> > > > 1. the patch removes the cdev_mapping concept completely. Cdev_id_qp is
> > > not getting used.
> > >
> > > Not exactly.
> > > cdev_id_qp is still setup, and is still used to decide to which crypto-dev to
> > > enqueuer the crypto-op:
> > > ipsec_enqueue(...)
> > > {
> > >    ...
> > >    enqueue_cop(&ipsec_ctx->tbl[sa->cdev_id_qp], &priv->cop);
> >
> > I don't see anybody filling "sa->cdev_id_qp". Please let me know if I have
> missed it somewhere.
> > It is memset to 0 I guess.
> 
> 
> Yep, true we lost it somewhere during the rework.
> 
> >
> > >
> > >
> > > Same in ipsec_process().
> > >
> > > For initialization, yes cdev_id_qp is not used anymore.
> > > As discussed here:
> > >
> https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fmails.dp
> > > dk.org%2Farchives%2Fdev%2F2019-
> > >
> March%2F127725.html&amp;data=02%7C01%7Cakhil.goyal%40nxp.com%7C04
> > >
> 194193cfc04c0b629008d6c7eea247%7C686ea1d3bc2b4c6fa92cd99c5c301635%
> > >
> 7C0%7C0%7C636916225072561313&amp;sdata=ga9IiqhYRWOz9QkRDIXNiigInk
> > > soVGgu1E5EetqvE%2FA%3D&amp;reserved=0
> > >
> > > I think the problem you are hitting with lookaside-proto is that for it
> > > we use 2 different values here:
> > > a) In create_sec_session we use portid (it also should be
> > > rte_cryptodev_get_sec_ctx() here)
> > >     if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > >                 ctx = (struct rte_security_ctx *)
> > >                                 rte_eth_dev_get_sec_ctx(sa->portid);
> > It should be rte_cryptodev_get_sec_ctx in the first place. And it needs a
> cdev_id as input based on the cdev mapping done while initializing
> > the cryptodev and neither the portid and nor cdev_id_qp.
> > > b) in enqueue() we use cdev_id_qp
> > >
> > > Right now these values could be different.
> > > As I understand we need to make sure that fro lookaside-proto cdev_id_qp
> ==
> > > portid provided by user, correct?
> > No it is not the case. Right now for lookaside there is no use of portid in case
> of lookaside case.
> > As the cdev/qp/core mappings are managed internally and the user cannot
> tweak it from cfg file.
> >
> 
> 
> Hmm, then at least that line  is wrong here:
> https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdoc.dpdk
> .org%2Fguides%2Fsample_app_ug%2Fipsec_secgw.html&amp;data=02%7C01%
> 7Cakhil.goyal%40nxp.com%7Cb1e931a9967943887a0c08d6c7f49d28%7C686ea
> 1d3bc2b4c6fa92cd99c5c301635%7C0%7C0%7C636916250754452306&amp;sda
> ta=tNjHCO0O2rfh8shhQXu93qrM0Hr0OZVXUVhMcsg53dw%3D&amp;reserved=
> 0
> 
> sa out 5 cipher_algo aes-128-cbc cipher_key 0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0 \
> auth_algo sha1-hmac auth_key 0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0 \
> mode ipv4-tunnel src 172.16.1.5 dst 172.16.2.5 \
> type lookaside-protocol-offload port_id 4
> 
> And probably:
> "Port/device ID of the ethernet/crypto accelerator for which the SA is
> configured."
> Need to be rephrased to remove crypto accelerator notice.
Yes.

> 
> Another question - why you guys don't consider using portid for lookaside-proto?
> As I can see add_mapping(function that  fills cdev_id_qp) doesn't bother to
> check
> which rte_security protocols are supported (only crypto capabilities are checked).
> So not sure does current code will work ok with a mix of lookaside-
> none/lookaside-proto devices.
> Forcing user to specify crypto-dev id for lookaside-proto (via portid or so)
> will simplify things significantly.
I will think about it. Actually initially when portid was introduced, the intent was same but it did not work well.
Because there may be a case of a cryptodev with multiple queues, so there will be a mapping done internally in the application and matching it with the user provided portid will be difficult.

I believe that this could be done but would need some rework.

> 
> Konstantin

Re: [dpdk-dev] [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Ananyev, Konstantin]

> >
> > >
> > > > -----Original Message-----
> > > > From: Akhil Goyal
> > > > Sent: Thursday, April 18, 2019 7:21 PM
> > > > To: Bernard Iremonger <bernard.iremonger@intel.com>; dev@dpdk.org;
> > > > konstantin.ananyev@intel.com
> > > > Cc: stable@dpdk.org
> > > > Subject: RE: [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped
> > for
> > > > inline crypto
> > > >
> > > > Hi Bernard,
> > > >
> > > > > -       RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on
> > cryptodev "
> > > > > -                       "%u qp %u\n", sa->spi,
> > > > > -                       ipsec_ctx->tbl[cdev_id_qp].id,
> > > > > -                       ipsec_ctx->tbl[cdev_id_qp].qp);
> > > > > +       if ((sa == NULL) || (pool == NULL))
> > > > > +               return -EINVAL;
> > > > >
> > > > > -       if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
> > > > > -               struct rte_security_session_conf sess_conf = {
> > > > > +       struct rte_security_session_conf sess_conf = {
> > > > >                         .action_type = sa->type,
> > > > >                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
> > > > >                         {.ipsec = {
> > > > > @@ -90,247 +65,340 @@ create_session(struct ipsec_ctx *ipsec_ctx,
> > struct
> > > > > ipsec_sa *sa)
> > > > >                         } },
> > > > >                         .crypto_xform = sa->xforms,
> > > > >                         .userdata = NULL,
> > > > > -
> > > > >                 };
> > > > >
> > > > > -               if (sa->type ==
> > > > RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL)
> > > > > {
> > > > > -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > > > -                                                       rte_cryptodev_get_sec_ctx(
> > > > > -                                                       ipsec_ctx->tbl[cdev_id_qp].id);
> > > > > -
> > > > > -                       /* Set IPsec parameters in conf */
> > > > > -                       set_ipsec_conf(sa, &(sess_conf.ipsec));
> > > > > -
> > > > > -                       sa->sec_session = rte_security_session_create(ctx,
> > > > > -                                       &sess_conf, ipsec_ctx->session_pool);
> > > > > -                       if (sa->sec_session == NULL) {
> > > > > -                               RTE_LOG(ERR, IPSEC,
> > > > > -                               "SEC Session init failed: err: %d\n", ret);
> > > > > -                               return -1;
> > > > > -                       }
> > > > > -               } else if (sa->type ==
> > RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
> > > > {
> > > > > -                       struct rte_flow_error err;
> > > > > -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > > > -                                                       rte_eth_dev_get_sec_ctx(
> > > > > -                                                       sa->portid);
> > > > > -                       const struct rte_security_capability *sec_cap;
> > > > > -                       int ret = 0;
> > > > > -
> > > > > -                       sa->sec_session = rte_security_session_create(ctx,
> > > > > -                                       &sess_conf, ipsec_ctx->session_pool);
> > > > > -                       if (sa->sec_session == NULL) {
> > > > > -                               RTE_LOG(ERR, IPSEC,
> > > > > -                               "SEC Session init failed: err: %d\n", ret);
> > > > > -                               return -1;
> > > > > -                       }
> > > > > +       if (sa->type ==
> > RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > > > > +               ctx = (struct rte_security_ctx *)
> > > > > +                               rte_eth_dev_get_sec_ctx(sa->portid);
> > > >
> > > > This is breaking the lookaside mode. Ctx was retrieved using the ipsec_ctx-
> > >tbl
> > > > struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > > 				rte_cryptodev_get_sec_ctx(
> > > > 				ipsec_ctx->tbl[cdev_id_qp].id);
> > > >
> > > > I am looking into it, but I don't have time left to get it integrated in RC2. So
> > this
> > > > has to be pushed to RC3
> > >
> > > It looks like there are multiple issues in this patch wrt lookaside and none cases.
> > Only the inline cases seem to be working.
> > >
> > > 1. the patch removes the cdev_mapping concept completely. Cdev_id_qp is
> > not getting used.
> >
> > Not exactly.
> > cdev_id_qp is still setup, and is still used to decide to which crypto-dev to
> > enqueuer the crypto-op:
> > ipsec_enqueue(...)
> > {
> >    ...
> >    enqueue_cop(&ipsec_ctx->tbl[sa->cdev_id_qp], &priv->cop);
> 
> I don't see anybody filling "sa->cdev_id_qp". Please let me know if I have missed it somewhere.
> It is memset to 0 I guess.


Yep, true we lost it somewhere during the rework.  

> 
> >
> >
> > Same in ipsec_process().
> >
> > For initialization, yes cdev_id_qp is not used anymore.
> > As discussed here:
> > https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fmails.dp
> > dk.org%2Farchives%2Fdev%2F2019-
> > March%2F127725.html&amp;data=02%7C01%7Cakhil.goyal%40nxp.com%7C04
> > 194193cfc04c0b629008d6c7eea247%7C686ea1d3bc2b4c6fa92cd99c5c301635%
> > 7C0%7C0%7C636916225072561313&amp;sdata=ga9IiqhYRWOz9QkRDIXNiigInk
> > soVGgu1E5EetqvE%2FA%3D&amp;reserved=0
> >
> > I think the problem you are hitting with lookaside-proto is that for it
> > we use 2 different values here:
> > a) In create_sec_session we use portid (it also should be
> > rte_cryptodev_get_sec_ctx() here)
> >     if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> >                 ctx = (struct rte_security_ctx *)
> >                                 rte_eth_dev_get_sec_ctx(sa->portid);
> It should be rte_cryptodev_get_sec_ctx in the first place. And it needs a cdev_id as input based on the cdev mapping done while initializing
> the cryptodev and neither the portid and nor cdev_id_qp.
> > b) in enqueue() we use cdev_id_qp
> >
> > Right now these values could be different.
> > As I understand we need to make sure that fro lookaside-proto cdev_id_qp ==
> > portid provided by user, correct?
> No it is not the case. Right now for lookaside there is no use of portid in case of lookaside case.
> As the cdev/qp/core mappings are managed internally and the user cannot tweak it from cfg file.
> 


Hmm, then at least that line  is wrong here:
https://doc.dpdk.org/guides/sample_app_ug/ipsec_secgw.html

sa out 5 cipher_algo aes-128-cbc cipher_key 0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0 \
auth_algo sha1-hmac auth_key 0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0 \
mode ipv4-tunnel src 172.16.1.5 dst 172.16.2.5 \
type lookaside-protocol-offload port_id 4

And probably:
"Port/device ID of the ethernet/crypto accelerator for which the SA is configured."
Need to be rephrased to remove crypto accelerator notice.

Another question - why you guys don't consider using portid for lookaside-proto?
As I can see add_mapping(function that  fills cdev_id_qp) doesn't bother to check
which rte_security protocols are supported (only crypto capabilities are checked).
So not sure does current code will work ok with a mix of lookaside-none/lookaside-proto devices.
Forcing user to specify crypto-dev id for lookaside-proto (via portid or so)
will simplify things significantly. 

Konstantin

Re: [dpdk-dev] [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Ananyev, Konstantin]

> >
> > >
> > > > -----Original Message-----
> > > > From: Akhil Goyal
> > > > Sent: Thursday, April 18, 2019 7:21 PM
> > > > To: Bernard Iremonger <bernard.iremonger@intel.com>; dev@dpdk.org;
> > > > konstantin.ananyev@intel.com
> > > > Cc: stable@dpdk.org
> > > > Subject: RE: [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped
> > for
> > > > inline crypto
> > > >
> > > > Hi Bernard,
> > > >
> > > > > -       RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on
> > cryptodev "
> > > > > -                       "%u qp %u\n", sa->spi,
> > > > > -                       ipsec_ctx->tbl[cdev_id_qp].id,
> > > > > -                       ipsec_ctx->tbl[cdev_id_qp].qp);
> > > > > +       if ((sa == NULL) || (pool == NULL))
> > > > > +               return -EINVAL;
> > > > >
> > > > > -       if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
> > > > > -               struct rte_security_session_conf sess_conf = {
> > > > > +       struct rte_security_session_conf sess_conf = {
> > > > >                         .action_type = sa->type,
> > > > >                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
> > > > >                         {.ipsec = {
> > > > > @@ -90,247 +65,340 @@ create_session(struct ipsec_ctx *ipsec_ctx,
> > struct
> > > > > ipsec_sa *sa)
> > > > >                         } },
> > > > >                         .crypto_xform = sa->xforms,
> > > > >                         .userdata = NULL,
> > > > > -
> > > > >                 };
> > > > >
> > > > > -               if (sa->type ==
> > > > RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL)
> > > > > {
> > > > > -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > > > -                                                       rte_cryptodev_get_sec_ctx(
> > > > > -                                                       ipsec_ctx->tbl[cdev_id_qp].id);
> > > > > -
> > > > > -                       /* Set IPsec parameters in conf */
> > > > > -                       set_ipsec_conf(sa, &(sess_conf.ipsec));
> > > > > -
> > > > > -                       sa->sec_session = rte_security_session_create(ctx,
> > > > > -                                       &sess_conf, ipsec_ctx->session_pool);
> > > > > -                       if (sa->sec_session == NULL) {
> > > > > -                               RTE_LOG(ERR, IPSEC,
> > > > > -                               "SEC Session init failed: err: %d\n", ret);
> > > > > -                               return -1;
> > > > > -                       }
> > > > > -               } else if (sa->type ==
> > RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
> > > > {
> > > > > -                       struct rte_flow_error err;
> > > > > -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > > > -                                                       rte_eth_dev_get_sec_ctx(
> > > > > -                                                       sa->portid);
> > > > > -                       const struct rte_security_capability *sec_cap;
> > > > > -                       int ret = 0;
> > > > > -
> > > > > -                       sa->sec_session = rte_security_session_create(ctx,
> > > > > -                                       &sess_conf, ipsec_ctx->session_pool);
> > > > > -                       if (sa->sec_session == NULL) {
> > > > > -                               RTE_LOG(ERR, IPSEC,
> > > > > -                               "SEC Session init failed: err: %d\n", ret);
> > > > > -                               return -1;
> > > > > -                       }
> > > > > +       if (sa->type ==
> > RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > > > > +               ctx = (struct rte_security_ctx *)
> > > > > +                               rte_eth_dev_get_sec_ctx(sa->portid);
> > > >
> > > > This is breaking the lookaside mode. Ctx was retrieved using the ipsec_ctx-
> > >tbl
> > > > struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > > 				rte_cryptodev_get_sec_ctx(
> > > > 				ipsec_ctx->tbl[cdev_id_qp].id);
> > > >
> > > > I am looking into it, but I don't have time left to get it integrated in RC2. So
> > this
> > > > has to be pushed to RC3
> > >
> > > It looks like there are multiple issues in this patch wrt lookaside and none cases.
> > Only the inline cases seem to be working.
> > >
> > > 1. the patch removes the cdev_mapping concept completely. Cdev_id_qp is
> > not getting used.
> >
> > Not exactly.
> > cdev_id_qp is still setup, and is still used to decide to which crypto-dev to
> > enqueuer the crypto-op:
> > ipsec_enqueue(...)
> > {
> >    ...
> >    enqueue_cop(&ipsec_ctx->tbl[sa->cdev_id_qp], &priv->cop);
> 
> I don't see anybody filling "sa->cdev_id_qp". Please let me know if I have missed it somewhere.
> It is memset to 0 I guess.


Yep, true we lost it somewhere during the rework.  

> 
> >
> >
> > Same in ipsec_process().
> >
> > For initialization, yes cdev_id_qp is not used anymore.
> > As discussed here:
> > https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fmails.dp
> > dk.org%2Farchives%2Fdev%2F2019-
> > March%2F127725.html&amp;data=02%7C01%7Cakhil.goyal%40nxp.com%7C04
> > 194193cfc04c0b629008d6c7eea247%7C686ea1d3bc2b4c6fa92cd99c5c301635%
> > 7C0%7C0%7C636916225072561313&amp;sdata=ga9IiqhYRWOz9QkRDIXNiigInk
> > soVGgu1E5EetqvE%2FA%3D&amp;reserved=0
> >
> > I think the problem you are hitting with lookaside-proto is that for it
> > we use 2 different values here:
> > a) In create_sec_session we use portid (it also should be
> > rte_cryptodev_get_sec_ctx() here)
> >     if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> >                 ctx = (struct rte_security_ctx *)
> >                                 rte_eth_dev_get_sec_ctx(sa->portid);
> It should be rte_cryptodev_get_sec_ctx in the first place. And it needs a cdev_id as input based on the cdev mapping done while initializing
> the cryptodev and neither the portid and nor cdev_id_qp.
> > b) in enqueue() we use cdev_id_qp
> >
> > Right now these values could be different.
> > As I understand we need to make sure that fro lookaside-proto cdev_id_qp ==
> > portid provided by user, correct?
> No it is not the case. Right now for lookaside there is no use of portid in case of lookaside case.
> As the cdev/qp/core mappings are managed internally and the user cannot tweak it from cfg file.
> 


Hmm, then at least that line  is wrong here:
https://doc.dpdk.org/guides/sample_app_ug/ipsec_secgw.html

sa out 5 cipher_algo aes-128-cbc cipher_key 0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0 \
auth_algo sha1-hmac auth_key 0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0 \
mode ipv4-tunnel src 172.16.1.5 dst 172.16.2.5 \
type lookaside-protocol-offload port_id 4

And probably:
"Port/device ID of the ethernet/crypto accelerator for which the SA is configured."
Need to be rephrased to remove crypto accelerator notice.

Another question - why you guys don't consider using portid for lookaside-proto?
As I can see add_mapping(function that  fills cdev_id_qp) doesn't bother to check
which rte_security protocols are supported (only crypto capabilities are checked).
So not sure does current code will work ok with a mix of lookaside-none/lookaside-proto devices.
Forcing user to specify crypto-dev id for lookaside-proto (via portid or so)
will simplify things significantly. 

Konstantin

Re: [dpdk-dev] [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Akhil Goyal]

Hi Konstantin,

> Hi Akhil,
> 
> >
> > > -----Original Message-----
> > > From: Akhil Goyal
> > > Sent: Thursday, April 18, 2019 7:21 PM
> > > To: Bernard Iremonger <bernard.iremonger@intel.com>; dev@dpdk.org;
> > > konstantin.ananyev@intel.com
> > > Cc: stable@dpdk.org
> > > Subject: RE: [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped
> for
> > > inline crypto
> > >
> > > Hi Bernard,
> > >
> > > > -       RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on
> cryptodev "
> > > > -                       "%u qp %u\n", sa->spi,
> > > > -                       ipsec_ctx->tbl[cdev_id_qp].id,
> > > > -                       ipsec_ctx->tbl[cdev_id_qp].qp);
> > > > +       if ((sa == NULL) || (pool == NULL))
> > > > +               return -EINVAL;
> > > >
> > > > -       if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
> > > > -               struct rte_security_session_conf sess_conf = {
> > > > +       struct rte_security_session_conf sess_conf = {
> > > >                         .action_type = sa->type,
> > > >                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
> > > >                         {.ipsec = {
> > > > @@ -90,247 +65,340 @@ create_session(struct ipsec_ctx *ipsec_ctx,
> struct
> > > > ipsec_sa *sa)
> > > >                         } },
> > > >                         .crypto_xform = sa->xforms,
> > > >                         .userdata = NULL,
> > > > -
> > > >                 };
> > > >
> > > > -               if (sa->type ==
> > > RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL)
> > > > {
> > > > -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > > -                                                       rte_cryptodev_get_sec_ctx(
> > > > -                                                       ipsec_ctx->tbl[cdev_id_qp].id);
> > > > -
> > > > -                       /* Set IPsec parameters in conf */
> > > > -                       set_ipsec_conf(sa, &(sess_conf.ipsec));
> > > > -
> > > > -                       sa->sec_session = rte_security_session_create(ctx,
> > > > -                                       &sess_conf, ipsec_ctx->session_pool);
> > > > -                       if (sa->sec_session == NULL) {
> > > > -                               RTE_LOG(ERR, IPSEC,
> > > > -                               "SEC Session init failed: err: %d\n", ret);
> > > > -                               return -1;
> > > > -                       }
> > > > -               } else if (sa->type ==
> RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
> > > {
> > > > -                       struct rte_flow_error err;
> > > > -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > > -                                                       rte_eth_dev_get_sec_ctx(
> > > > -                                                       sa->portid);
> > > > -                       const struct rte_security_capability *sec_cap;
> > > > -                       int ret = 0;
> > > > -
> > > > -                       sa->sec_session = rte_security_session_create(ctx,
> > > > -                                       &sess_conf, ipsec_ctx->session_pool);
> > > > -                       if (sa->sec_session == NULL) {
> > > > -                               RTE_LOG(ERR, IPSEC,
> > > > -                               "SEC Session init failed: err: %d\n", ret);
> > > > -                               return -1;
> > > > -                       }
> > > > +       if (sa->type ==
> RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > > > +               ctx = (struct rte_security_ctx *)
> > > > +                               rte_eth_dev_get_sec_ctx(sa->portid);
> > >
> > > This is breaking the lookaside mode. Ctx was retrieved using the ipsec_ctx-
> >tbl
> > > struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > 				rte_cryptodev_get_sec_ctx(
> > > 				ipsec_ctx->tbl[cdev_id_qp].id);
> > >
> > > I am looking into it, but I don't have time left to get it integrated in RC2. So
> this
> > > has to be pushed to RC3
> >
> > It looks like there are multiple issues in this patch wrt lookaside and none cases.
> Only the inline cases seem to be working.
> >
> > 1. the patch removes the cdev_mapping concept completely. Cdev_id_qp is
> not getting used.
> 
> Not exactly.
> cdev_id_qp is still setup, and is still used to decide to which crypto-dev to
> enqueuer the crypto-op:
> ipsec_enqueue(...)
> {
>    ...
>    enqueue_cop(&ipsec_ctx->tbl[sa->cdev_id_qp], &priv->cop);

I don't see anybody filling "sa->cdev_id_qp". Please let me know if I have missed it somewhere.
It is memset to 0 I guess.

> 
> 
> Same in ipsec_process().
> 
> For initialization, yes cdev_id_qp is not used anymore.
> As discussed here:
> https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fmails.dp
> dk.org%2Farchives%2Fdev%2F2019-
> March%2F127725.html&amp;data=02%7C01%7Cakhil.goyal%40nxp.com%7C04
> 194193cfc04c0b629008d6c7eea247%7C686ea1d3bc2b4c6fa92cd99c5c301635%
> 7C0%7C0%7C636916225072561313&amp;sdata=ga9IiqhYRWOz9QkRDIXNiigInk
> soVGgu1E5EetqvE%2FA%3D&amp;reserved=0
> 
> I think the problem you are hitting with lookaside-proto is that for it
> we use 2 different values here:
> a) In create_sec_session we use portid (it also should be
> rte_cryptodev_get_sec_ctx() here)
>     if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
>                 ctx = (struct rte_security_ctx *)
>                                 rte_eth_dev_get_sec_ctx(sa->portid);
It should be rte_cryptodev_get_sec_ctx in the first place. And it needs a cdev_id as input based on the cdev mapping done while initializing the cryptodev and neither the portid and nor cdev_id_qp.
> b) in enqueue() we use cdev_id_qp
> 
> Right now these values could be different.
> As I understand we need to make sure that fro lookaside-proto cdev_id_qp ==
> portid provided by user, correct?
No it is not the case. Right now for lookaside there is no use of portid in case of lookaside case.
As the cdev/qp/core mappings are managed internally and the user cannot tweak it from cfg file.


> 
> 
> >     The port_id cannot be used in case of crypto, the mapping of cdev/qp/core
> is done differently for inbound and outbound ports which is
> > missed in this patch.
> >
> > 2. crypto sessions are created using the session mempool and the private data
> is allocated using the session priv_mempool which is
> > removed in this patch. This will break cases where the priv data is more than
> the size of sess_mp element size.
> >     Also the security sessions need to be allocated using the session_priv_mp
> instead of the session_mp.
> > Please check this one.
> >
> https://eur01.safelinks.protection.outlook.com/?url=http%3A%2F%2Fpatches.d
> pdk.org%2Fpatch%2F52981%2F&amp;data=02%7C01%7Cakhil.goyal%40nxp.co
> m%7C04194193cfc04c0b629008d6c7eea247%7C686ea1d3bc2b4c6fa92cd99c5c
> 301635%7C0%7C0%7C636916225072561313&amp;sdata=libTy%2F%2Bj23gGru
> vxhdlVUGIOeVq%2BlM2PIF1ZsgN%2FaSY%3D&amp;reserved=0
> 
> Yes, I think you right, we need to use sess_private_pool here.
> 
> >
> > Ideally this issue should be resolved by adding another parameter in
> rte_security_session_create which can take another mempool pointer
> > for private data allocation. But this cannot be done in this release as it would
> need a deprecation notice.
> >
> > With the above issues I don't see your patch going in 19.05 release cycle.
> >
> > Regards,
> > Akhil
> >
> > >
> > >
> > >
> > > >
> > > > -                       sec_cap = rte_security_capabilities_get(ctx);
> > > > +               /* Set IPsec parameters in conf */
> > > > +               set_ipsec_conf(sa, &(sess_conf.ipsec));
> > > >
> > > > -                       /* iterate until ESP tunnel*/
> > > > -                       while (sec_cap->action !=
> > > > -                                       RTE_SECURITY_ACTION_TYPE_NONE) {
> > > > +               sa->sec_session = rte_security_session_create(ctx,
> > > > +                               &sess_conf, pool);
> > > > +               if (sa->sec_session == NULL) {
> > > > +                       RTE_LOG(ERR, IPSEC,
> > > > +                               "SEC Session init failed: err: %d\n",
> > > > +                               ret);
> > > > +                       return -1;
> > > > +               }

Re: [dpdk-dev] [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Akhil Goyal]

Hi Konstantin,

> Hi Akhil,
> 
> >
> > > -----Original Message-----
> > > From: Akhil Goyal
> > > Sent: Thursday, April 18, 2019 7:21 PM
> > > To: Bernard Iremonger <bernard.iremonger@intel.com>; dev@dpdk.org;
> > > konstantin.ananyev@intel.com
> > > Cc: stable@dpdk.org
> > > Subject: RE: [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped
> for
> > > inline crypto
> > >
> > > Hi Bernard,
> > >
> > > > -       RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on
> cryptodev "
> > > > -                       "%u qp %u\n", sa->spi,
> > > > -                       ipsec_ctx->tbl[cdev_id_qp].id,
> > > > -                       ipsec_ctx->tbl[cdev_id_qp].qp);
> > > > +       if ((sa == NULL) || (pool == NULL))
> > > > +               return -EINVAL;
> > > >
> > > > -       if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
> > > > -               struct rte_security_session_conf sess_conf = {
> > > > +       struct rte_security_session_conf sess_conf = {
> > > >                         .action_type = sa->type,
> > > >                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
> > > >                         {.ipsec = {
> > > > @@ -90,247 +65,340 @@ create_session(struct ipsec_ctx *ipsec_ctx,
> struct
> > > > ipsec_sa *sa)
> > > >                         } },
> > > >                         .crypto_xform = sa->xforms,
> > > >                         .userdata = NULL,
> > > > -
> > > >                 };
> > > >
> > > > -               if (sa->type ==
> > > RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL)
> > > > {
> > > > -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > > -                                                       rte_cryptodev_get_sec_ctx(
> > > > -                                                       ipsec_ctx->tbl[cdev_id_qp].id);
> > > > -
> > > > -                       /* Set IPsec parameters in conf */
> > > > -                       set_ipsec_conf(sa, &(sess_conf.ipsec));
> > > > -
> > > > -                       sa->sec_session = rte_security_session_create(ctx,
> > > > -                                       &sess_conf, ipsec_ctx->session_pool);
> > > > -                       if (sa->sec_session == NULL) {
> > > > -                               RTE_LOG(ERR, IPSEC,
> > > > -                               "SEC Session init failed: err: %d\n", ret);
> > > > -                               return -1;
> > > > -                       }
> > > > -               } else if (sa->type ==
> RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
> > > {
> > > > -                       struct rte_flow_error err;
> > > > -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > > -                                                       rte_eth_dev_get_sec_ctx(
> > > > -                                                       sa->portid);
> > > > -                       const struct rte_security_capability *sec_cap;
> > > > -                       int ret = 0;
> > > > -
> > > > -                       sa->sec_session = rte_security_session_create(ctx,
> > > > -                                       &sess_conf, ipsec_ctx->session_pool);
> > > > -                       if (sa->sec_session == NULL) {
> > > > -                               RTE_LOG(ERR, IPSEC,
> > > > -                               "SEC Session init failed: err: %d\n", ret);
> > > > -                               return -1;
> > > > -                       }
> > > > +       if (sa->type ==
> RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > > > +               ctx = (struct rte_security_ctx *)
> > > > +                               rte_eth_dev_get_sec_ctx(sa->portid);
> > >
> > > This is breaking the lookaside mode. Ctx was retrieved using the ipsec_ctx-
> >tbl
> > > struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > 				rte_cryptodev_get_sec_ctx(
> > > 				ipsec_ctx->tbl[cdev_id_qp].id);
> > >
> > > I am looking into it, but I don't have time left to get it integrated in RC2. So
> this
> > > has to be pushed to RC3
> >
> > It looks like there are multiple issues in this patch wrt lookaside and none cases.
> Only the inline cases seem to be working.
> >
> > 1. the patch removes the cdev_mapping concept completely. Cdev_id_qp is
> not getting used.
> 
> Not exactly.
> cdev_id_qp is still setup, and is still used to decide to which crypto-dev to
> enqueuer the crypto-op:
> ipsec_enqueue(...)
> {
>    ...
>    enqueue_cop(&ipsec_ctx->tbl[sa->cdev_id_qp], &priv->cop);

I don't see anybody filling "sa->cdev_id_qp". Please let me know if I have missed it somewhere.
It is memset to 0 I guess.

> 
> 
> Same in ipsec_process().
> 
> For initialization, yes cdev_id_qp is not used anymore.
> As discussed here:
> https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fmails.dp
> dk.org%2Farchives%2Fdev%2F2019-
> March%2F127725.html&amp;data=02%7C01%7Cakhil.goyal%40nxp.com%7C04
> 194193cfc04c0b629008d6c7eea247%7C686ea1d3bc2b4c6fa92cd99c5c301635%
> 7C0%7C0%7C636916225072561313&amp;sdata=ga9IiqhYRWOz9QkRDIXNiigInk
> soVGgu1E5EetqvE%2FA%3D&amp;reserved=0
> 
> I think the problem you are hitting with lookaside-proto is that for it
> we use 2 different values here:
> a) In create_sec_session we use portid (it also should be
> rte_cryptodev_get_sec_ctx() here)
>     if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
>                 ctx = (struct rte_security_ctx *)
>                                 rte_eth_dev_get_sec_ctx(sa->portid);
It should be rte_cryptodev_get_sec_ctx in the first place. And it needs a cdev_id as input based on the cdev mapping done while initializing the cryptodev and neither the portid and nor cdev_id_qp.
> b) in enqueue() we use cdev_id_qp
> 
> Right now these values could be different.
> As I understand we need to make sure that fro lookaside-proto cdev_id_qp ==
> portid provided by user, correct?
No it is not the case. Right now for lookaside there is no use of portid in case of lookaside case.
As the cdev/qp/core mappings are managed internally and the user cannot tweak it from cfg file.


> 
> 
> >     The port_id cannot be used in case of crypto, the mapping of cdev/qp/core
> is done differently for inbound and outbound ports which is
> > missed in this patch.
> >
> > 2. crypto sessions are created using the session mempool and the private data
> is allocated using the session priv_mempool which is
> > removed in this patch. This will break cases where the priv data is more than
> the size of sess_mp element size.
> >     Also the security sessions need to be allocated using the session_priv_mp
> instead of the session_mp.
> > Please check this one.
> >
> https://eur01.safelinks.protection.outlook.com/?url=http%3A%2F%2Fpatches.d
> pdk.org%2Fpatch%2F52981%2F&amp;data=02%7C01%7Cakhil.goyal%40nxp.co
> m%7C04194193cfc04c0b629008d6c7eea247%7C686ea1d3bc2b4c6fa92cd99c5c
> 301635%7C0%7C0%7C636916225072561313&amp;sdata=libTy%2F%2Bj23gGru
> vxhdlVUGIOeVq%2BlM2PIF1ZsgN%2FaSY%3D&amp;reserved=0
> 
> Yes, I think you right, we need to use sess_private_pool here.
> 
> >
> > Ideally this issue should be resolved by adding another parameter in
> rte_security_session_create which can take another mempool pointer
> > for private data allocation. But this cannot be done in this release as it would
> need a deprecation notice.
> >
> > With the above issues I don't see your patch going in 19.05 release cycle.
> >
> > Regards,
> > Akhil
> >
> > >
> > >
> > >
> > > >
> > > > -                       sec_cap = rte_security_capabilities_get(ctx);
> > > > +               /* Set IPsec parameters in conf */
> > > > +               set_ipsec_conf(sa, &(sess_conf.ipsec));
> > > >
> > > > -                       /* iterate until ESP tunnel*/
> > > > -                       while (sec_cap->action !=
> > > > -                                       RTE_SECURITY_ACTION_TYPE_NONE) {
> > > > +               sa->sec_session = rte_security_session_create(ctx,
> > > > +                               &sess_conf, pool);
> > > > +               if (sa->sec_session == NULL) {
> > > > +                       RTE_LOG(ERR, IPSEC,
> > > > +                               "SEC Session init failed: err: %d\n",
> > > > +                               ret);
> > > > +                       return -1;
> > > > +               }

Re: [dpdk-dev] [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Ananyev, Konstantin]

Hi Akhil,

> 
> > -----Original Message-----
> > From: Akhil Goyal
> > Sent: Thursday, April 18, 2019 7:21 PM
> > To: Bernard Iremonger <bernard.iremonger@intel.com>; dev@dpdk.org;
> > konstantin.ananyev@intel.com
> > Cc: stable@dpdk.org
> > Subject: RE: [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for
> > inline crypto
> >
> > Hi Bernard,
> >
> > > -       RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on cryptodev "
> > > -                       "%u qp %u\n", sa->spi,
> > > -                       ipsec_ctx->tbl[cdev_id_qp].id,
> > > -                       ipsec_ctx->tbl[cdev_id_qp].qp);
> > > +       if ((sa == NULL) || (pool == NULL))
> > > +               return -EINVAL;
> > >
> > > -       if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
> > > -               struct rte_security_session_conf sess_conf = {
> > > +       struct rte_security_session_conf sess_conf = {
> > >                         .action_type = sa->type,
> > >                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
> > >                         {.ipsec = {
> > > @@ -90,247 +65,340 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct
> > > ipsec_sa *sa)
> > >                         } },
> > >                         .crypto_xform = sa->xforms,
> > >                         .userdata = NULL,
> > > -
> > >                 };
> > >
> > > -               if (sa->type ==
> > RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL)
> > > {
> > > -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > -                                                       rte_cryptodev_get_sec_ctx(
> > > -                                                       ipsec_ctx->tbl[cdev_id_qp].id);
> > > -
> > > -                       /* Set IPsec parameters in conf */
> > > -                       set_ipsec_conf(sa, &(sess_conf.ipsec));
> > > -
> > > -                       sa->sec_session = rte_security_session_create(ctx,
> > > -                                       &sess_conf, ipsec_ctx->session_pool);
> > > -                       if (sa->sec_session == NULL) {
> > > -                               RTE_LOG(ERR, IPSEC,
> > > -                               "SEC Session init failed: err: %d\n", ret);
> > > -                               return -1;
> > > -                       }
> > > -               } else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
> > {
> > > -                       struct rte_flow_error err;
> > > -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > -                                                       rte_eth_dev_get_sec_ctx(
> > > -                                                       sa->portid);
> > > -                       const struct rte_security_capability *sec_cap;
> > > -                       int ret = 0;
> > > -
> > > -                       sa->sec_session = rte_security_session_create(ctx,
> > > -                                       &sess_conf, ipsec_ctx->session_pool);
> > > -                       if (sa->sec_session == NULL) {
> > > -                               RTE_LOG(ERR, IPSEC,
> > > -                               "SEC Session init failed: err: %d\n", ret);
> > > -                               return -1;
> > > -                       }
> > > +       if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > > +               ctx = (struct rte_security_ctx *)
> > > +                               rte_eth_dev_get_sec_ctx(sa->portid);
> >
> > This is breaking the lookaside mode. Ctx was retrieved using the ipsec_ctx->tbl
> > struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > 				rte_cryptodev_get_sec_ctx(
> > 				ipsec_ctx->tbl[cdev_id_qp].id);
> >
> > I am looking into it, but I don't have time left to get it integrated in RC2. So this
> > has to be pushed to RC3
> 
> It looks like there are multiple issues in this patch wrt lookaside and none cases. Only the inline cases seem to be working.
> 
> 1. the patch removes the cdev_mapping concept completely. Cdev_id_qp is not getting used.

Not exactly.
cdev_id_qp is still setup, and is still used to decide to which crypto-dev to enqueuer the crypto-op:
ipsec_enqueue(...)
{
   ...
   enqueue_cop(&ipsec_ctx->tbl[sa->cdev_id_qp], &priv->cop);


Same in ipsec_process().

For initialization, yes cdev_id_qp is not used anymore.
As discussed here:
https://mails.dpdk.org/archives/dev/2019-March/127725.html

I think the problem you are hitting with lookaside-proto is that for it
we use 2 different values here: 
a) In create_sec_session we use portid (it also should be rte_cryptodev_get_sec_ctx() here)
    if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
                ctx = (struct rte_security_ctx *)
                                rte_eth_dev_get_sec_ctx(sa->portid);
b) in enqueue() we use cdev_id_qp

Right now these values could be different.
As I understand we need to make sure that fro lookaside-proto cdev_id_qp == portid provided by user, correct?


>     The port_id cannot be used in case of crypto, the mapping of cdev/qp/core is done differently for inbound and outbound ports which is
> missed in this patch.
> 
> 2. crypto sessions are created using the session mempool and the private data is allocated using the session priv_mempool which is
> removed in this patch. This will break cases where the priv data is more than the size of sess_mp element size.
>     Also the security sessions need to be allocated using the session_priv_mp instead of the session_mp.
> Please check this one.
> http://patches.dpdk.org/patch/52981/

Yes, I think you right, we need to use sess_private_pool here.

> 
> Ideally this issue should be resolved by adding another parameter in rte_security_session_create which can take another mempool pointer
> for private data allocation. But this cannot be done in this release as it would need a deprecation notice.
> 
> With the above issues I don't see your patch going in 19.05 release cycle.
> 
> Regards,
> Akhil
> 
> >
> >
> >
> > >
> > > -                       sec_cap = rte_security_capabilities_get(ctx);
> > > +               /* Set IPsec parameters in conf */
> > > +               set_ipsec_conf(sa, &(sess_conf.ipsec));
> > >
> > > -                       /* iterate until ESP tunnel*/
> > > -                       while (sec_cap->action !=
> > > -                                       RTE_SECURITY_ACTION_TYPE_NONE) {
> > > +               sa->sec_session = rte_security_session_create(ctx,
> > > +                               &sess_conf, pool);
> > > +               if (sa->sec_session == NULL) {
> > > +                       RTE_LOG(ERR, IPSEC,
> > > +                               "SEC Session init failed: err: %d\n",
> > > +                               ret);
> > > +                       return -1;
> > > +               }

Re: [dpdk-dev] [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Ananyev, Konstantin]

Hi Akhil,

> 
> > -----Original Message-----
> > From: Akhil Goyal
> > Sent: Thursday, April 18, 2019 7:21 PM
> > To: Bernard Iremonger <bernard.iremonger@intel.com>; dev@dpdk.org;
> > konstantin.ananyev@intel.com
> > Cc: stable@dpdk.org
> > Subject: RE: [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for
> > inline crypto
> >
> > Hi Bernard,
> >
> > > -       RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on cryptodev "
> > > -                       "%u qp %u\n", sa->spi,
> > > -                       ipsec_ctx->tbl[cdev_id_qp].id,
> > > -                       ipsec_ctx->tbl[cdev_id_qp].qp);
> > > +       if ((sa == NULL) || (pool == NULL))
> > > +               return -EINVAL;
> > >
> > > -       if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
> > > -               struct rte_security_session_conf sess_conf = {
> > > +       struct rte_security_session_conf sess_conf = {
> > >                         .action_type = sa->type,
> > >                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
> > >                         {.ipsec = {
> > > @@ -90,247 +65,340 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct
> > > ipsec_sa *sa)
> > >                         } },
> > >                         .crypto_xform = sa->xforms,
> > >                         .userdata = NULL,
> > > -
> > >                 };
> > >
> > > -               if (sa->type ==
> > RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL)
> > > {
> > > -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > -                                                       rte_cryptodev_get_sec_ctx(
> > > -                                                       ipsec_ctx->tbl[cdev_id_qp].id);
> > > -
> > > -                       /* Set IPsec parameters in conf */
> > > -                       set_ipsec_conf(sa, &(sess_conf.ipsec));
> > > -
> > > -                       sa->sec_session = rte_security_session_create(ctx,
> > > -                                       &sess_conf, ipsec_ctx->session_pool);
> > > -                       if (sa->sec_session == NULL) {
> > > -                               RTE_LOG(ERR, IPSEC,
> > > -                               "SEC Session init failed: err: %d\n", ret);
> > > -                               return -1;
> > > -                       }
> > > -               } else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
> > {
> > > -                       struct rte_flow_error err;
> > > -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > > -                                                       rte_eth_dev_get_sec_ctx(
> > > -                                                       sa->portid);
> > > -                       const struct rte_security_capability *sec_cap;
> > > -                       int ret = 0;
> > > -
> > > -                       sa->sec_session = rte_security_session_create(ctx,
> > > -                                       &sess_conf, ipsec_ctx->session_pool);
> > > -                       if (sa->sec_session == NULL) {
> > > -                               RTE_LOG(ERR, IPSEC,
> > > -                               "SEC Session init failed: err: %d\n", ret);
> > > -                               return -1;
> > > -                       }
> > > +       if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > > +               ctx = (struct rte_security_ctx *)
> > > +                               rte_eth_dev_get_sec_ctx(sa->portid);
> >
> > This is breaking the lookaside mode. Ctx was retrieved using the ipsec_ctx->tbl
> > struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > 				rte_cryptodev_get_sec_ctx(
> > 				ipsec_ctx->tbl[cdev_id_qp].id);
> >
> > I am looking into it, but I don't have time left to get it integrated in RC2. So this
> > has to be pushed to RC3
> 
> It looks like there are multiple issues in this patch wrt lookaside and none cases. Only the inline cases seem to be working.
> 
> 1. the patch removes the cdev_mapping concept completely. Cdev_id_qp is not getting used.

Not exactly.
cdev_id_qp is still setup, and is still used to decide to which crypto-dev to enqueuer the crypto-op:
ipsec_enqueue(...)
{
   ...
   enqueue_cop(&ipsec_ctx->tbl[sa->cdev_id_qp], &priv->cop);


Same in ipsec_process().

For initialization, yes cdev_id_qp is not used anymore.
As discussed here:
https://mails.dpdk.org/archives/dev/2019-March/127725.html

I think the problem you are hitting with lookaside-proto is that for it
we use 2 different values here: 
a) In create_sec_session we use portid (it also should be rte_cryptodev_get_sec_ctx() here)
    if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
                ctx = (struct rte_security_ctx *)
                                rte_eth_dev_get_sec_ctx(sa->portid);
b) in enqueue() we use cdev_id_qp

Right now these values could be different.
As I understand we need to make sure that fro lookaside-proto cdev_id_qp == portid provided by user, correct?


>     The port_id cannot be used in case of crypto, the mapping of cdev/qp/core is done differently for inbound and outbound ports which is
> missed in this patch.
> 
> 2. crypto sessions are created using the session mempool and the private data is allocated using the session priv_mempool which is
> removed in this patch. This will break cases where the priv data is more than the size of sess_mp element size.
>     Also the security sessions need to be allocated using the session_priv_mp instead of the session_mp.
> Please check this one.
> http://patches.dpdk.org/patch/52981/

Yes, I think you right, we need to use sess_private_pool here.

> 
> Ideally this issue should be resolved by adding another parameter in rte_security_session_create which can take another mempool pointer
> for private data allocation. But this cannot be done in this release as it would need a deprecation notice.
> 
> With the above issues I don't see your patch going in 19.05 release cycle.
> 
> Regards,
> Akhil
> 
> >
> >
> >
> > >
> > > -                       sec_cap = rte_security_capabilities_get(ctx);
> > > +               /* Set IPsec parameters in conf */
> > > +               set_ipsec_conf(sa, &(sess_conf.ipsec));
> > >
> > > -                       /* iterate until ESP tunnel*/
> > > -                       while (sec_cap->action !=
> > > -                                       RTE_SECURITY_ACTION_TYPE_NONE) {
> > > +               sa->sec_session = rte_security_session_create(ctx,
> > > +                               &sess_conf, pool);
> > > +               if (sa->sec_session == NULL) {
> > > +                       RTE_LOG(ERR, IPSEC,
> > > +                               "SEC Session init failed: err: %d\n",
> > > +                               ret);
> > > +                       return -1;
> > > +               }

Re: [dpdk-dev] [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Akhil Goyal]

Hi Bernard,


> -----Original Message-----
> From: Akhil Goyal
> Sent: Thursday, April 18, 2019 7:21 PM
> To: Bernard Iremonger <bernard.iremonger@intel.com>; dev@dpdk.org;
> konstantin.ananyev@intel.com
> Cc: stable@dpdk.org
> Subject: RE: [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for
> inline crypto
> 
> Hi Bernard,
> 
> > -       RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on cryptodev "
> > -                       "%u qp %u\n", sa->spi,
> > -                       ipsec_ctx->tbl[cdev_id_qp].id,
> > -                       ipsec_ctx->tbl[cdev_id_qp].qp);
> > +       if ((sa == NULL) || (pool == NULL))
> > +               return -EINVAL;
> >
> > -       if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
> > -               struct rte_security_session_conf sess_conf = {
> > +       struct rte_security_session_conf sess_conf = {
> >                         .action_type = sa->type,
> >                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
> >                         {.ipsec = {
> > @@ -90,247 +65,340 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct
> > ipsec_sa *sa)
> >                         } },
> >                         .crypto_xform = sa->xforms,
> >                         .userdata = NULL,
> > -
> >                 };
> >
> > -               if (sa->type ==
> RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL)
> > {
> > -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > -                                                       rte_cryptodev_get_sec_ctx(
> > -                                                       ipsec_ctx->tbl[cdev_id_qp].id);
> > -
> > -                       /* Set IPsec parameters in conf */
> > -                       set_ipsec_conf(sa, &(sess_conf.ipsec));
> > -
> > -                       sa->sec_session = rte_security_session_create(ctx,
> > -                                       &sess_conf, ipsec_ctx->session_pool);
> > -                       if (sa->sec_session == NULL) {
> > -                               RTE_LOG(ERR, IPSEC,
> > -                               "SEC Session init failed: err: %d\n", ret);
> > -                               return -1;
> > -                       }
> > -               } else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
> {
> > -                       struct rte_flow_error err;
> > -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > -                                                       rte_eth_dev_get_sec_ctx(
> > -                                                       sa->portid);
> > -                       const struct rte_security_capability *sec_cap;
> > -                       int ret = 0;
> > -
> > -                       sa->sec_session = rte_security_session_create(ctx,
> > -                                       &sess_conf, ipsec_ctx->session_pool);
> > -                       if (sa->sec_session == NULL) {
> > -                               RTE_LOG(ERR, IPSEC,
> > -                               "SEC Session init failed: err: %d\n", ret);
> > -                               return -1;
> > -                       }
> > +       if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > +               ctx = (struct rte_security_ctx *)
> > +                               rte_eth_dev_get_sec_ctx(sa->portid);
> 
> This is breaking the lookaside mode. Ctx was retrieved using the ipsec_ctx->tbl
> struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> 				rte_cryptodev_get_sec_ctx(
> 				ipsec_ctx->tbl[cdev_id_qp].id);
> 
> I am looking into it, but I don't have time left to get it integrated in RC2. So this
> has to be pushed to RC3

It looks like there are multiple issues in this patch wrt lookaside and none cases. Only the inline cases seem to be working.

1. the patch removes the cdev_mapping concept completely. Cdev_id_qp is not getting used.
    The port_id cannot be used in case of crypto, the mapping of cdev/qp/core is done differently for inbound and outbound ports which is missed in this patch.

2. crypto sessions are created using the session mempool and the private data is allocated using the session priv_mempool which is removed in this patch. This will break cases where the priv data is more than the size of sess_mp element size.
    Also the security sessions need to be allocated using the session_priv_mp instead of the session_mp.
Please check this one.
http://patches.dpdk.org/patch/52981/

Ideally this issue should be resolved by adding another parameter in rte_security_session_create which can take another mempool pointer for private data allocation. But this cannot be done in this release as it would need a deprecation notice.

With the above issues I don't see your patch going in 19.05 release cycle.

Regards,
Akhil

> 
> 
> 
> >
> > -                       sec_cap = rte_security_capabilities_get(ctx);
> > +               /* Set IPsec parameters in conf */
> > +               set_ipsec_conf(sa, &(sess_conf.ipsec));
> >
> > -                       /* iterate until ESP tunnel*/
> > -                       while (sec_cap->action !=
> > -                                       RTE_SECURITY_ACTION_TYPE_NONE) {
> > +               sa->sec_session = rte_security_session_create(ctx,
> > +                               &sess_conf, pool);
> > +               if (sa->sec_session == NULL) {
> > +                       RTE_LOG(ERR, IPSEC,
> > +                               "SEC Session init failed: err: %d\n",
> > +                               ret);
> > +                       return -1;
> > +               }

Re: [dpdk-dev] [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Akhil Goyal]

Hi Bernard,


> -----Original Message-----
> From: Akhil Goyal
> Sent: Thursday, April 18, 2019 7:21 PM
> To: Bernard Iremonger <bernard.iremonger@intel.com>; dev@dpdk.org;
> konstantin.ananyev@intel.com
> Cc: stable@dpdk.org
> Subject: RE: [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for
> inline crypto
> 
> Hi Bernard,
> 
> > -       RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on cryptodev "
> > -                       "%u qp %u\n", sa->spi,
> > -                       ipsec_ctx->tbl[cdev_id_qp].id,
> > -                       ipsec_ctx->tbl[cdev_id_qp].qp);
> > +       if ((sa == NULL) || (pool == NULL))
> > +               return -EINVAL;
> >
> > -       if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
> > -               struct rte_security_session_conf sess_conf = {
> > +       struct rte_security_session_conf sess_conf = {
> >                         .action_type = sa->type,
> >                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
> >                         {.ipsec = {
> > @@ -90,247 +65,340 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct
> > ipsec_sa *sa)
> >                         } },
> >                         .crypto_xform = sa->xforms,
> >                         .userdata = NULL,
> > -
> >                 };
> >
> > -               if (sa->type ==
> RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL)
> > {
> > -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > -                                                       rte_cryptodev_get_sec_ctx(
> > -                                                       ipsec_ctx->tbl[cdev_id_qp].id);
> > -
> > -                       /* Set IPsec parameters in conf */
> > -                       set_ipsec_conf(sa, &(sess_conf.ipsec));
> > -
> > -                       sa->sec_session = rte_security_session_create(ctx,
> > -                                       &sess_conf, ipsec_ctx->session_pool);
> > -                       if (sa->sec_session == NULL) {
> > -                               RTE_LOG(ERR, IPSEC,
> > -                               "SEC Session init failed: err: %d\n", ret);
> > -                               return -1;
> > -                       }
> > -               } else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
> {
> > -                       struct rte_flow_error err;
> > -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > -                                                       rte_eth_dev_get_sec_ctx(
> > -                                                       sa->portid);
> > -                       const struct rte_security_capability *sec_cap;
> > -                       int ret = 0;
> > -
> > -                       sa->sec_session = rte_security_session_create(ctx,
> > -                                       &sess_conf, ipsec_ctx->session_pool);
> > -                       if (sa->sec_session == NULL) {
> > -                               RTE_LOG(ERR, IPSEC,
> > -                               "SEC Session init failed: err: %d\n", ret);
> > -                               return -1;
> > -                       }
> > +       if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > +               ctx = (struct rte_security_ctx *)
> > +                               rte_eth_dev_get_sec_ctx(sa->portid);
> 
> This is breaking the lookaside mode. Ctx was retrieved using the ipsec_ctx->tbl
> struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> 				rte_cryptodev_get_sec_ctx(
> 				ipsec_ctx->tbl[cdev_id_qp].id);
> 
> I am looking into it, but I don't have time left to get it integrated in RC2. So this
> has to be pushed to RC3

It looks like there are multiple issues in this patch wrt lookaside and none cases. Only the inline cases seem to be working.

1. the patch removes the cdev_mapping concept completely. Cdev_id_qp is not getting used.
    The port_id cannot be used in case of crypto, the mapping of cdev/qp/core is done differently for inbound and outbound ports which is missed in this patch.

2. crypto sessions are created using the session mempool and the private data is allocated using the session priv_mempool which is removed in this patch. This will break cases where the priv data is more than the size of sess_mp element size.
    Also the security sessions need to be allocated using the session_priv_mp instead of the session_mp.
Please check this one.
http://patches.dpdk.org/patch/52981/

Ideally this issue should be resolved by adding another parameter in rte_security_session_create which can take another mempool pointer for private data allocation. But this cannot be done in this release as it would need a deprecation notice.

With the above issues I don't see your patch going in 19.05 release cycle.

Regards,
Akhil

> 
> 
> 
> >
> > -                       sec_cap = rte_security_capabilities_get(ctx);
> > +               /* Set IPsec parameters in conf */
> > +               set_ipsec_conf(sa, &(sess_conf.ipsec));
> >
> > -                       /* iterate until ESP tunnel*/
> > -                       while (sec_cap->action !=
> > -                                       RTE_SECURITY_ACTION_TYPE_NONE) {
> > +               sa->sec_session = rte_security_session_create(ctx,
> > +                               &sess_conf, pool);
> > +               if (sa->sec_session == NULL) {
> > +                       RTE_LOG(ERR, IPSEC,
> > +                               "SEC Session init failed: err: %d\n",
> > +                               ret);
> > +                       return -1;
> > +               }

Re: [dpdk-dev] [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Akhil Goyal]

Hi Bernard,

> 
> Hi Akhil,
> 
> <snip>
> 
> > Subject: RE: [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped
> > for inline crypto
> 
> <snip>
> > > +       if (sa->type ==
> > RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > > +               ctx = (struct rte_security_ctx *)
> > > +                               rte_eth_dev_get_sec_ctx(sa->portid);
> >
> > This is breaking the lookaside mode. Ctx was retrieved using the ipsec_ctx-
> > >tbl struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> >                               rte_cryptodev_get_sec_ctx(
> >                               ipsec_ctx->tbl[cdev_id_qp].id);
> >
> > I am looking into it, but I don't have time left to get it integrated in RC2. So
> > this has to be pushed to RC3
> 
> <snip>
> 
> Unfortunately we do not have the HW to test this feature.
> What HW are you using to test this?
> 
> Having looked at the code previously
> ipsec_ctx->tbl[cdev_id_qp].id   turned out to be the port_id.
> 
> So we had expected it to work.
> 
> We will need your help with this.

I am looking into this. Will let you know when I get the fix.
> 
> Regards,
> 
> Bernard.

Re: [dpdk-dev] [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Akhil Goyal]

Hi Bernard,

> 
> Hi Akhil,
> 
> <snip>
> 
> > Subject: RE: [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped
> > for inline crypto
> 
> <snip>
> > > +       if (sa->type ==
> > RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > > +               ctx = (struct rte_security_ctx *)
> > > +                               rte_eth_dev_get_sec_ctx(sa->portid);
> >
> > This is breaking the lookaside mode. Ctx was retrieved using the ipsec_ctx-
> > >tbl struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> >                               rte_cryptodev_get_sec_ctx(
> >                               ipsec_ctx->tbl[cdev_id_qp].id);
> >
> > I am looking into it, but I don't have time left to get it integrated in RC2. So
> > this has to be pushed to RC3
> 
> <snip>
> 
> Unfortunately we do not have the HW to test this feature.
> What HW are you using to test this?
> 
> Having looked at the code previously
> ipsec_ctx->tbl[cdev_id_qp].id   turned out to be the port_id.
> 
> So we had expected it to work.
> 
> We will need your help with this.

I am looking into this. Will let you know when I get the fix.
> 
> Regards,
> 
> Bernard.

Re: [dpdk-dev] [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Iremonger, Bernard]

Hi Akhil,

> -----Original Message-----
> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Iremonger, Bernard
> Sent: Thursday, April 18, 2019 3:59 PM
> To: Akhil Goyal <akhil.goyal@nxp.com>; dev@dpdk.org; Ananyev,
> Konstantin <konstantin.ananyev@intel.com>
> Cc: stable@dpdk.org
> Subject: Re: [dpdk-dev] [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet
> dropped for inline crypto
> 
> Hi Akhil,
> 
> <snip>
> 
> > Subject: RE: [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet
> > dropped for inline crypto
> 
> <snip>
> > > +       if (sa->type ==
> > RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > > +               ctx = (struct rte_security_ctx *)
> > > +                               rte_eth_dev_get_sec_ctx(sa->portid);
> >
> > This is breaking the lookaside mode. Ctx was retrieved using the
> > ipsec_ctx-
> > >tbl struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > 				rte_cryptodev_get_sec_ctx(
> > 				ipsec_ctx->tbl[cdev_id_qp].id);
> >
> > I am looking into it, but I don't have time left to get it integrated
> > in RC2. So this has to be pushed to RC3
> 
> <snip>
> 
> Unfortunately we do not have the HW to test this feature.
> What HW are you using to test this?
> 
> Having looked at the code previously
> ipsec_ctx->tbl[cdev_id_qp].id   turned out to be the port_id.
> 
> So we had expected it to work.
> 
> We will need your help with this.
> 
> Regards,
> 
> Bernard.

Just had another look at the 19.05.rc1 code
Line 1546 in ipsec-secgw.c: 
ipsec_ctx->tbl[i].id = cdev_id;

The id is the cryptodev id, not the port_id

Regards,

Bernard.

Re: [dpdk-dev] [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Iremonger, Bernard]

Hi Akhil,

> -----Original Message-----
> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Iremonger, Bernard
> Sent: Thursday, April 18, 2019 3:59 PM
> To: Akhil Goyal <akhil.goyal@nxp.com>; dev@dpdk.org; Ananyev,
> Konstantin <konstantin.ananyev@intel.com>
> Cc: stable@dpdk.org
> Subject: Re: [dpdk-dev] [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet
> dropped for inline crypto
> 
> Hi Akhil,
> 
> <snip>
> 
> > Subject: RE: [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet
> > dropped for inline crypto
> 
> <snip>
> > > +       if (sa->type ==
> > RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > > +               ctx = (struct rte_security_ctx *)
> > > +                               rte_eth_dev_get_sec_ctx(sa->portid);
> >
> > This is breaking the lookaside mode. Ctx was retrieved using the
> > ipsec_ctx-
> > >tbl struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > 				rte_cryptodev_get_sec_ctx(
> > 				ipsec_ctx->tbl[cdev_id_qp].id);
> >
> > I am looking into it, but I don't have time left to get it integrated
> > in RC2. So this has to be pushed to RC3
> 
> <snip>
> 
> Unfortunately we do not have the HW to test this feature.
> What HW are you using to test this?
> 
> Having looked at the code previously
> ipsec_ctx->tbl[cdev_id_qp].id   turned out to be the port_id.
> 
> So we had expected it to work.
> 
> We will need your help with this.
> 
> Regards,
> 
> Bernard.

Just had another look at the 19.05.rc1 code
Line 1546 in ipsec-secgw.c: 
ipsec_ctx->tbl[i].id = cdev_id;

The id is the cryptodev id, not the port_id

Regards,

Bernard.

Re: [dpdk-dev] [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Iremonger, Bernard]

Hi Akhil,

<snip>

> Subject: RE: [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped
> for inline crypto
 
<snip>
> > +       if (sa->type ==
> RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > +               ctx = (struct rte_security_ctx *)
> > +                               rte_eth_dev_get_sec_ctx(sa->portid);
> 
> This is breaking the lookaside mode. Ctx was retrieved using the ipsec_ctx-
> >tbl struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> 				rte_cryptodev_get_sec_ctx(
> 				ipsec_ctx->tbl[cdev_id_qp].id);
> 
> I am looking into it, but I don't have time left to get it integrated in RC2. So
> this has to be pushed to RC3

<snip>

Unfortunately we do not have the HW to test this feature.
What HW are you using to test this?

Having looked at the code previously
ipsec_ctx->tbl[cdev_id_qp].id   turned out to be the port_id.

So we had expected it to work.

We will need your help with this.

Regards,

Bernard.

Re: [dpdk-dev] [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Iremonger, Bernard]

Hi Akhil,

<snip>

> Subject: RE: [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped
> for inline crypto
 
<snip>
> > +       if (sa->type ==
> RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > +               ctx = (struct rte_security_ctx *)
> > +                               rte_eth_dev_get_sec_ctx(sa->portid);
> 
> This is breaking the lookaside mode. Ctx was retrieved using the ipsec_ctx-
> >tbl struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> 				rte_cryptodev_get_sec_ctx(
> 				ipsec_ctx->tbl[cdev_id_qp].id);
> 
> I am looking into it, but I don't have time left to get it integrated in RC2. So
> this has to be pushed to RC3

<snip>

Unfortunately we do not have the HW to test this feature.
What HW are you using to test this?

Having looked at the code previously
ipsec_ctx->tbl[cdev_id_qp].id   turned out to be the port_id.

So we had expected it to work.

We will need your help with this.

Regards,

Bernard.

Re: [dpdk-dev] [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Akhil Goyal]

Hi Bernard,

> -       RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on cryptodev "
> -                       "%u qp %u\n", sa->spi,
> -                       ipsec_ctx->tbl[cdev_id_qp].id,
> -                       ipsec_ctx->tbl[cdev_id_qp].qp);
> +       if ((sa == NULL) || (pool == NULL))
> +               return -EINVAL;
> 
> -       if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
> -               struct rte_security_session_conf sess_conf = {
> +       struct rte_security_session_conf sess_conf = {
>                         .action_type = sa->type,
>                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
>                         {.ipsec = {
> @@ -90,247 +65,340 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct
> ipsec_sa *sa)
>                         } },
>                         .crypto_xform = sa->xforms,
>                         .userdata = NULL,
> -
>                 };
> 
> -               if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL)
> {
> -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> -                                                       rte_cryptodev_get_sec_ctx(
> -                                                       ipsec_ctx->tbl[cdev_id_qp].id);
> -
> -                       /* Set IPsec parameters in conf */
> -                       set_ipsec_conf(sa, &(sess_conf.ipsec));
> -
> -                       sa->sec_session = rte_security_session_create(ctx,
> -                                       &sess_conf, ipsec_ctx->session_pool);
> -                       if (sa->sec_session == NULL) {
> -                               RTE_LOG(ERR, IPSEC,
> -                               "SEC Session init failed: err: %d\n", ret);
> -                               return -1;
> -                       }
> -               } else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
> -                       struct rte_flow_error err;
> -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> -                                                       rte_eth_dev_get_sec_ctx(
> -                                                       sa->portid);
> -                       const struct rte_security_capability *sec_cap;
> -                       int ret = 0;
> -
> -                       sa->sec_session = rte_security_session_create(ctx,
> -                                       &sess_conf, ipsec_ctx->session_pool);
> -                       if (sa->sec_session == NULL) {
> -                               RTE_LOG(ERR, IPSEC,
> -                               "SEC Session init failed: err: %d\n", ret);
> -                               return -1;
> -                       }
> +       if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> +               ctx = (struct rte_security_ctx *)
> +                               rte_eth_dev_get_sec_ctx(sa->portid);

This is breaking the lookaside mode. Ctx was retrieved using the ipsec_ctx->tbl
struct rte_security_ctx *ctx = (struct rte_security_ctx *)
				rte_cryptodev_get_sec_ctx(
				ipsec_ctx->tbl[cdev_id_qp].id);

I am looking into it, but I don't have time left to get it integrated in RC2. So this has to be pushed to RC3



> 
> -                       sec_cap = rte_security_capabilities_get(ctx);
> +               /* Set IPsec parameters in conf */
> +               set_ipsec_conf(sa, &(sess_conf.ipsec));
> 
> -                       /* iterate until ESP tunnel*/
> -                       while (sec_cap->action !=
> -                                       RTE_SECURITY_ACTION_TYPE_NONE) {
> +               sa->sec_session = rte_security_session_create(ctx,
> +                               &sess_conf, pool);
> +               if (sa->sec_session == NULL) {
> +                       RTE_LOG(ERR, IPSEC,
> +                               "SEC Session init failed: err: %d\n",
> +                               ret);
> +                       return -1;
> +               }

Re: [dpdk-dev] [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Akhil Goyal]

Hi Bernard,

> -       RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on cryptodev "
> -                       "%u qp %u\n", sa->spi,
> -                       ipsec_ctx->tbl[cdev_id_qp].id,
> -                       ipsec_ctx->tbl[cdev_id_qp].qp);
> +       if ((sa == NULL) || (pool == NULL))
> +               return -EINVAL;
> 
> -       if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
> -               struct rte_security_session_conf sess_conf = {
> +       struct rte_security_session_conf sess_conf = {
>                         .action_type = sa->type,
>                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
>                         {.ipsec = {
> @@ -90,247 +65,340 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct
> ipsec_sa *sa)
>                         } },
>                         .crypto_xform = sa->xforms,
>                         .userdata = NULL,
> -
>                 };
> 
> -               if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL)
> {
> -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> -                                                       rte_cryptodev_get_sec_ctx(
> -                                                       ipsec_ctx->tbl[cdev_id_qp].id);
> -
> -                       /* Set IPsec parameters in conf */
> -                       set_ipsec_conf(sa, &(sess_conf.ipsec));
> -
> -                       sa->sec_session = rte_security_session_create(ctx,
> -                                       &sess_conf, ipsec_ctx->session_pool);
> -                       if (sa->sec_session == NULL) {
> -                               RTE_LOG(ERR, IPSEC,
> -                               "SEC Session init failed: err: %d\n", ret);
> -                               return -1;
> -                       }
> -               } else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
> -                       struct rte_flow_error err;
> -                       struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> -                                                       rte_eth_dev_get_sec_ctx(
> -                                                       sa->portid);
> -                       const struct rte_security_capability *sec_cap;
> -                       int ret = 0;
> -
> -                       sa->sec_session = rte_security_session_create(ctx,
> -                                       &sess_conf, ipsec_ctx->session_pool);
> -                       if (sa->sec_session == NULL) {
> -                               RTE_LOG(ERR, IPSEC,
> -                               "SEC Session init failed: err: %d\n", ret);
> -                               return -1;
> -                       }
> +       if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> +               ctx = (struct rte_security_ctx *)
> +                               rte_eth_dev_get_sec_ctx(sa->portid);

This is breaking the lookaside mode. Ctx was retrieved using the ipsec_ctx->tbl
struct rte_security_ctx *ctx = (struct rte_security_ctx *)
				rte_cryptodev_get_sec_ctx(
				ipsec_ctx->tbl[cdev_id_qp].id);

I am looking into it, but I don't have time left to get it integrated in RC2. So this has to be pushed to RC3



> 
> -                       sec_cap = rte_security_capabilities_get(ctx);
> +               /* Set IPsec parameters in conf */
> +               set_ipsec_conf(sa, &(sess_conf.ipsec));
> 
> -                       /* iterate until ESP tunnel*/
> -                       while (sec_cap->action !=
> -                                       RTE_SECURITY_ACTION_TYPE_NONE) {
> +               sa->sec_session = rte_security_session_create(ctx,
> +                               &sess_conf, pool);
> +               if (sa->sec_session == NULL) {
> +                       RTE_LOG(ERR, IPSEC,
> +                               "SEC Session init failed: err: %d\n",
> +                               ret);
> +                       return -1;
> +               }